Repository: huggingface/transformers
Pull request: Update type hints in tokenization_utils.py to use | syntax

## What does this PR do?

This PR modernizes type hints in `tokenization_utils.py` to use Python 3.10+ `|` syntax instead of `Union` types where appropriate.

## Changes Made

- Replace `Union[AddedToken, str]` with `AddedToken | str`
- Replace `Union[list[str], list[AddedToken]]` with `list[str] | list[AddedToken]`
- Replace `Union[str, list[str]]` with `str | list[str]`
- Replace `Union[int, list[int]]` with `int | list[int]`
- Update error messages to use `|` syntax
- Maintain backward compatibility

## Testing

- ✅ All imports work correctly
- ✅ Tokenization functionality works
- ✅ No linting errors
- ✅ Maintains backward compatibility

## Related Issues

Similar to PRs #41675 and #41646 which also updated type hints to use `|` syntax.

## Before submitting
- [x] This PR fixes a typo or improves the docs (you can dismiss the other checks if that's the case).
- [x] Did you read the contributor guideline, Pull Request section?
- [x] Did you make sure to update the documentation with your changes?
- [x] Did you write any new necessary tests?

Changed files:
filename: src/transformers/configuration_utils.py
patch: @@ -390,7 +390,7 @@ def _attn_implementation(self):
         return self._attn_implementation_internal
 
     @_attn_implementation.setter
-    def _attn_implementation(self, value: Optional[Union[str, dict]]):
+    def _attn_implementation(self, value: str | dict | None):
         """We set it recursively on the sub-configs as well"""
         # Set if for current config
         current_attn = getattr(self, "_attn_implementation", None)
@@ -425,7 +425,7 @@ def rope_scaling(self):
     def rope_scaling(self, value):
         self.rope_parameters = value
 
-    def save_pretrained(self, save_directory: Union[str, os.PathLike], push_to_hub: bool = False, **kwargs):
+    def save_pretrained(self, save_directory: str | os.PathLike, push_to_hub: bool = False, **kwargs):
         """
         Save a configuration object to the directory `save_directory`, so that it can be re-loaded using the
         [`~PreTrainedConfig.from_pretrained`] class method.
@@ -490,11 +490,11 @@ def save_pretrained(self, save_directory: Union[str, os.PathLike], push_to_hub:
     @classmethod
     def from_pretrained(
         cls: type[SpecificPreTrainedConfigType],
-        pretrained_model_name_or_path: Union[str, os.PathLike],
-        cache_dir: Optional[Union[str, os.PathLike]] = None,
+        pretrained_model_name_or_path: str | os.PathLike,
+        cache_dir: str | os.PathLike | None = None,
         force_download: bool = False,
         local_files_only: bool = False,
-        token: Optional[Union[str, bool]] = None,
+        token: str | bool | None = None,
         revision: str = "main",
         **kwargs,
     ) -> SpecificPreTrainedConfigType:
@@ -597,7 +597,7 @@ def from_pretrained(
 
     @classmethod
     def get_config_dict(
-        cls, pretrained_model_name_or_path: Union[str, os.PathLike], **kwargs
+        cls, pretrained_model_name_or_path: str | os.PathLike, **kwargs
     ) -> tuple[dict[str, Any], dict[str, Any]]:
         """
         From a `pretrained_model_name_or_path`, resolve to a dictionary of parameters, to be used for instantiating a
@@ -630,7 +630,7 @@ def get_config_dict(
 
     @classmethod
     def _get_config_dict(
-        cls, pretrained_model_name_or_path: Union[str, os.PathLike], **kwargs
+        cls, pretrained_model_name_or_path: str | os.PathLike, **kwargs
     ) -> tuple[dict[str, Any], dict[str, Any]]:
         cache_dir = kwargs.pop("cache_dir", None)
         force_download = kwargs.pop("force_download", False)
@@ -793,7 +793,7 @@ def from_dict(
 
     @classmethod
     def from_json_file(
-        cls: type[SpecificPreTrainedConfigType], json_file: Union[str, os.PathLike]
+        cls: type[SpecificPreTrainedConfigType], json_file: str | os.PathLike
     ) -> SpecificPreTrainedConfigType:
         """
         Instantiates a [`PreTrainedConfig`] from the path to a JSON file of parameters.
@@ -810,7 +810,7 @@ def from_json_file(
         return cls(**config_dict)
 
     @classmethod
-    def _dict_from_json_file(cls, json_file: Union[str, os.PathLike]):
+    def _dict_from_json_file(cls, json_file: str | os.PathLike):
         with open(json_file, encoding="utf-8") as reader:
             text = reader.read()
         return json.loads(text)
@@ -935,7 +935,7 @@ def to_json_string(self, use_diff: bool = True) -> str:
             config_dict = self.to_dict()
         return json.dumps(config_dict, indent=2, sort_keys=True) + "\n"
 
-    def to_json_file(self, json_file_path: Union[str, os.PathLike], use_diff: bool = True):
+    def to_json_file(self, json_file_path: str | os.PathLike, use_diff: bool = True):
         """
         Save this instance to a JSON file.
 

filename: src/transformers/modeling_rope_utils.py
patch: @@ -14,7 +14,7 @@
 
 import math
 from functools import wraps
-from typing import Optional, TypedDict, Union
+from typing import Optional, TypedDict
 
 from .configuration_utils import PreTrainedConfig
 from .utils import is_torch_available, logging
@@ -27,7 +27,7 @@
     import torch
 
 
-def standardize_rope_params(config, rope_theta: Optional[Union[float, dict[str, float]]] = None):
+def standardize_rope_params(config, rope_theta: float | dict[str, float] | None = None):
     """
     Helper to standardize the config's rope params field by ensuring the params are defined for each
     later type. For old model the fn will duplicate a single rope param in each layer type (backward compatibility)

filename: src/transformers/tokenization_utils.py
patch: @@ -472,7 +472,7 @@ def added_tokens_decoder(self) -> dict[int, AddedToken]:
         return dict(sorted(self._added_tokens_decoder.items(), key=lambda item: item[0]))
 
     @added_tokens_decoder.setter
-    def added_tokens_decoder(self, value: dict[int, Union[AddedToken, str]]) -> dict[int, AddedToken]:
+    def added_tokens_decoder(self, value: dict[int, AddedToken | str]) -> dict[int, AddedToken]:
         # Always raise an error if string because users should define the behavior
         for index, token in value.items():
             if not isinstance(token, (str, AddedToken)) or not isinstance(index, int):
@@ -509,7 +509,7 @@ def _update_total_vocab_size(self):
         """
         self.total_vocab_size = len(self.get_vocab())
 
-    def _add_tokens(self, new_tokens: Union[list[str], list[AddedToken]], special_tokens: bool = False) -> int:
+    def _add_tokens(self, new_tokens: list[str] | list[AddedToken], special_tokens: bool = False) -> int:
         """
         Add a list of new tokens to the tokenizer class. If the new tokens are not in the vocabulary, they are added to
         it with indices starting from length of the current vocabulary. Special tokens are sometimes already in the
@@ -707,7 +707,7 @@ def _tokenize(self, text, **kwargs):
         """
         raise NotImplementedError
 
-    def convert_tokens_to_ids(self, tokens: Union[str, list[str]]) -> Union[int, list[int]]:
+    def convert_tokens_to_ids(self, tokens: str | list[str]) -> int | list[int]:
         """
         Converts a token string (or a sequence of tokens) in a single integer id (or a sequence of ids), using the
         vocabulary.