Repository: huggingface/transformers
Pull request: Patch Mistral Common Tokenizer 2

# What does this PR do?

1. Fixes the behavior of add_special_tokens to match PretrainedTokenizer goals:
- if mode is finetuning: add both bos and eos tokens
- if mode is test: add only bos tokens so that the model can generate freely.

2. Add `ValidationMode` as a string support to avoid import from mistral-common for the user.

3. Fix decode when user passes an int instead of a sequence of ints

4. Add support to batch for `decode`.

5. Add special tokens properties to list ids and tokens string representation.


## Before submitting
- [ ] This PR fixes a typo or improves the docs (you can dismiss the other checks if that's the case).
- [x] Did you read the [contributor guideline](https://github.com/huggingface/transformers/blob/main/CONTRIBUTING.md#create-a-pull-request),
      Pull Request section?
- [ ] Was this discussed/approved via a Github issue or the [forum](https://discuss.huggingface.co/)? Please add a link
      to it if that's the case.
- [ ] Did you make sure to update the documentation with your changes? Here are the
      [documentation guidelines](https://github.com/huggingface/transformers/tree/main/docs), and
      [here are tips on formatting docstrings](https://github.com/huggingface/transformers/tree/main/docs#writing-source-documentation).
- [x] Did you write any new necessary tests?


## Who can review?

cc @patrickvonplaten for viz

@ArthurZucker and @itazap


Changed files:
filename: src/transformers/tokenization_mistral_common.py
patch: @@ -61,7 +61,7 @@
                 Whether or not to add special tokens when encoding the sequences. This will use the underlying
                 `PretrainedTokenizerBase.build_inputs_with_special_tokens` function, which defines which tokens are
                 automatically added to the input ids. This is useful if you want to add `bos` or `eos` tokens
-                automatically.
+                automatically. When Tokenizer is loading with `finetuning` mode it adds both `bos` and `eos`. Else, for "test" mode it only adds `bos`.
             padding (`bool`, `str` or [`~utils.PaddingStrategy`], *optional*, defaults to `False`):
                 Activates and controls padding. Accepts the following values:
 
@@ -215,7 +215,7 @@ def __init__(
         """
         Constructs a `MistralCommonBackend`.
 
-        - **model_input_names** (`List[str]`) -- A list of inputs expected in the forward pass of the model.
+        - **model_input_names** (`list[str]`) -- A list of inputs expected in the forward pass of the model.
         - **padding_side** (`str`) -- The default value for the side on which the model should have padding applied.
             Should be `'right'` or `'left'`.
         - **truncation_side** (`str`) -- The default value for the side on which the model should have truncation
@@ -224,8 +224,11 @@ def __init__(
         Args:
             tokenizer_path (`str` or `os.PathLike` or `Path`):
                 Path to the tokenizer file to load the `MistralTokenizer`.
-            mode (`ValidationMode`, *optional*, defaults to `ValidationMode.test`):
-                The mode to use for the tokenizer. This will be passed to the `MistralTokenizer` constructor.
+            mode (`Union[str, ValidationMode]`, *optional*, defaults to `ValidationMode.test`):
+                The mode to use for the tokenizer. This will be passed to the `MistralTokenizer` constructor. Possible values are:
+                - `"finetuning"` or `ValidationMode.finetuning`: The finetuning mode.
+                - `"test"` or `ValidationMode.test`: The test mode.
+                It changes how the tokenizer validates the input and prepares the request to the model.
             model_max_length (`int`, *optional*):
                 The maximum length (in number of tokens) for the inputs to the transformer model. When the tokenizer is
                 loaded with [`~tokenization_utils_base.PreTrainedTokenizerBase.from_pretrained`], this will be set to the
@@ -248,7 +251,8 @@ def __init__(
             raise ValueError(f"Kwargs {list(kwargs.keys())} are not supported to init `MistralCommonBackend`.")
 
         self._tokenizer_path = Path(tokenizer_path)
-        self.tokenizer: MistralTokenizer = MistralTokenizer.from_file(str(self._tokenizer_path), mode=mode)
+        self._mode = self._get_validation_mode(mode)
+        self.tokenizer: MistralTokenizer = MistralTokenizer.from_file(str(self._tokenizer_path), mode=self._mode)
         self._tokenizer_type = (
             MistralTokenizerType.tekken
             if isinstance(self.tokenizer.instruct_tokenizer.tokenizer, Tekkenizer)
@@ -259,6 +263,7 @@ def __init__(
         self.model_max_length = model_max_length
         self.cleanup_tokenization_spaces = clean_up_tokenization_spaces
         self.deprecation_warnings = {}  # Use to store when we have already noticed a deprecation warning (avoid overlogging).
+        self._all_special_tokens_ids = self._get_all_special_ids()
 
         if model_input_names is not None:
             if (
@@ -291,6 +296,16 @@ def clean_up_tokenization(text: str) -> str:
             .replace(" 're", "'re")
         )
 
+    @property
+    def mode(self) -> ValidationMode:
+        """
+        `ValidationMode`: The mode used by the tokenizer. Possible values are:
+            - `"finetuning"` or `ValidationMode.finetuning`: The finetuning mode.
+            - `"test"` or `ValidationMode.test`: The test mode.
+            It changes how the tokenizer validates the input and prepares the request to the model.
+        """
+        return self._mode
+
     @property
     def bos_token_id(self) -> int:
         """
@@ -347,6 +362,20 @@ def pad_token(self) -> str:
         """
         return self.convert_ids_to_tokens(self.pad_token_id)
 
+    @property
+    def all_special_ids(self) -> list[int]:
+        """
+        `list[int]`: List the ids of the special tokens(`'<unk>'`, `'<cls>'`, etc.).
+        """
+        return sorted(self._all_special_tokens_ids)
+
+    @property
+    def all_special_tokens(self) -> list[str]:
+        """
+        `list[str]`: A list of all unique special tokens.
+        """
+        return self.convert_ids_to_tokens(self.all_special_ids)
+
     @property
     def vocab_size(self) -> int:
         """
@@ -390,7 +419,7 @@ def __len__(self):
         """,
         """
         Returns:
-            `List[int]`, `torch.Tensor`: The tokenized ids of the text.
+            `list[int]`, `torch.Tensor`: The tokenized ids of the text.
         """,
     )
     def encode(
@@ -412,7 +441,7 @@ def encode(
         Converts a string to a sequence of ids (integer), using the tokenizer and vocabulary.
 
         Args:
-            text (`str` or `List[int]`):
+            text (`str` or `list[int]`):
                 The first sequence to be encoded. This can be a string or a list of integers (tokenized string ids).
             text_pair (`None`, *optional*):
                 Not supported by `MistralCommonBackend.encode`. Kept to match `PreTrainedTokenizerBase.encode` signature.
@@ -451,18 +480,19 @@ def encode(
 
     def decode(
         self,
-        token_ids: Union[int, list[int], np.ndarray, "torch.Tensor"],
+        token_ids: Union[int, list[int], list[list[int]], np.ndarray, "torch.Tensor"],
         skip_special_tokens: bool = False,
         clean_up_tokenization_spaces: bool | None = None,
         **kwargs,
-    ) -> str:
+    ) -> Union[str, list[str]]:
         """
         Converts a sequence of ids in a string, using the tokenizer and vocabulary with options to remove special
         tokens and clean up tokenization spaces.
 
         Args:
-            token_ids (`Union[int, List[int], np.ndarray, torch.Tensor]`):
-                List of tokenized input ids. Can be obtained using the `__call__` method.
+            token_ids (`Union[int, list[int], list[list[int]], np.ndarray, torch.Tensor]`):
+                A single sequence or a batch (list of sequences) of tokenized input ids. Can be obtained using the
+                `__call__` method.
             skip_special_tokens (`bool`, *optional*, defaults to `False`):
                 Whether or not to remove special tokens in the decoding.
             clean_up_tokenization_spaces (`bool`, *optional*):
@@ -473,29 +503,26 @@ def decode(
                 Will raise an error if used.
 
         Returns:
-            `str`: The decoded sentence.
+            `Union[str, list[str]]`: The decoded string for a single sequence, or a list of decoded strings for a
+            batch of sequences.
         """
         if kwargs:
             raise ValueError(f"Kwargs {list(kwargs.keys())} are not supported by `MistralCommonBackend.decode`.")
 
-        clean_up_tokenization_spaces = clean_up_tokenization_spaces or self.cleanup_tokenization_spaces
-
-        # Convert inputs to python lists
         token_ids = to_py_obj(token_ids)
 
-        special_token_policy = SpecialTokenPolicy.IGNORE if skip_special_tokens else SpecialTokenPolicy.KEEP
-
-        decoded_string = self.tokenizer.decode(token_ids, special_token_policy=special_token_policy)
-        if clean_up_tokenization_spaces:
-            decoded_string = self.clean_up_tokenization(decoded_string)
-
-        # in the specific case of Voxtral, the added f"lang:xx" (always a two char language code since it follows ISO 639-1 alpha-2 format)
-        # is not considered as a special token by mistral-common and is encoded/ decoded as normal text.
-        # Nevertheless we should remove it to ease users life.
-        if skip_special_tokens:
-            decoded_string = re.sub(r"^lang:[a-z]{2}", "", decoded_string)
+        if isinstance(token_ids, (list, tuple)) and len(token_ids) > 0 and isinstance(token_ids[0], (list, tuple)):
+            return self._batch_decode(
+                sequences=token_ids,
+                skip_special_tokens=skip_special_tokens,
+                clean_up_tokenization_spaces=clean_up_tokenization_spaces,
+            )
 
-        return decoded_string
+        return self._decode(
+            token_ids=token_ids,
+            skip_special_tokens=skip_special_tokens,
+            clean_up_tokenization_spaces=clean_up_tokenization_spaces,
+        )
 
     def batch_decode(
         self,
@@ -507,8 +534,11 @@ def batch_decode(
         """
         Convert a list of lists of token ids into a list of strings by calling decode.
 
+        This method is provided for backwards compatibility. The `decode` method now handles batched input natively,
+        so you can use `decode` directly instead of `batch_decode`.
+
         Args:
-            sequences (`Union[List[int], List[List[int]], np.ndarray, torch.Tensor]`):
+            sequences (`Union[list[int], list[list[int]], np.ndarray, torch.Tensor]`):
                 List of tokenized input ids. Can be obtained using the `__call__` method.
             skip_special_tokens (`bool`, *optional*, defaults to `False`):
                 Whether or not to remove special tokens in the decoding.
@@ -520,14 +550,56 @@ def batch_decode(
                 Will raise an error if used.
 
         Returns:
-            `List[str]`: The list of decoded sentences.
+            `list[str]`: The list of decoded sentences.
         """
+        if kwargs:
+            raise ValueError(f"Kwargs {list(kwargs.keys())} are not supported by `MistralCommonBackend.batch_decode`.")
+
+        return self._batch_decode(
+            sequences=sequences,
+            skip_special_tokens=skip_special_tokens,
+            clean_up_tokenization_spaces=clean_up_tokenization_spaces,
+        )
+
+    def _decode(
+        self,
+        token_ids: Union[int, list[int], list[list[int]], np.ndarray, "torch.Tensor"],
+        skip_special_tokens: bool = False,
+        clean_up_tokenization_spaces: bool | None = None,
+    ) -> str:
+        clean_up_tokenization_spaces = clean_up_tokenization_spaces or self.cleanup_tokenization_spaces
+
+        # Convert inputs to python lists
+        if isinstance(token_ids, int):
+            token_ids = [token_ids]
+
+        token_ids = to_py_obj(token_ids)
+
+        special_token_policy = SpecialTokenPolicy.IGNORE if skip_special_tokens else SpecialTokenPolicy.KEEP
+
+        decoded_string = self.tokenizer.decode(token_ids, special_token_policy=special_token_policy)
+        if clean_up_tokenization_spaces:
+            decoded_string = self.clean_up_tokenization(decoded_string)
+
+        # in the specific case of Voxtral, the added f"lang:xx" (always a two char language code since it follows ISO 639-1 alpha-2 format)
+        # is not considered as a special token by mistral-common and is encoded/ decoded as normal text.
+        # Nevertheless we should remove it to ease users life.
+        if skip_special_tokens:
+            decoded_string = re.sub(r"^lang:[a-z]{2}", "", decoded_string)
+
+        return decoded_string
+
+    def _batch_decode(
+        self,
+        sequences: Union[list[int], list[list[int]], np.ndarray, "torch.Tensor"],
+        skip_special_tokens: bool = False,
+        clean_up_tokenization_spaces: bool | None = None,
+    ) -> list[str]:
         return [
-            self.decode(
+            self._decode(
                 seq,
                 skip_special_tokens=skip_special_tokens,
                 clean_up_tokenization_spaces=clean_up_tokenization_spaces,
-                **kwargs,
             )
             for seq in sequences
         ]
@@ -550,13 +622,13 @@ def convert_ids_to_tokens(self, ids: int | list[int], skip_special_tokens: bool
         added tokens.
 
         Args:
-            ids (`int` or `List[int]`):
+            ids (`int` or `list[int]`):
                 The token id (or token ids) to convert to tokens.
             skip_special_tokens (`bool`, *optional*, defaults to `False`):
                 Whether or not to remove special tokens in the decoding.
 
         Returns:
-            `str` or `List[str]`: The decoded token(s).
+            `str` or `list[str]`: The decoded token(s).
         """
 
         if isinstance(ids, int):
@@ -608,10 +680,10 @@ def convert_tokens_to_ids(self, tokens: str | list[str]) -> int | list[int]:
         vocabulary.
 
         Args:
-            tokens (`str` or `List[str]`): One or several token(s) to convert to token id(s).
+            tokens (`str` or `list[str]`): One or several token(s) to convert to token id(s).
 
         Returns:
-            `int` or `List[int]`: The token id or list of token ids.
+            `int` or `list[int]`: The token id or list of token ids.
         """
 
         if isinstance(tokens, str):
@@ -632,9 +704,8 @@ def _text_to_ids(self, text: TextInput, add_special_tokens: bool) -> list[int]:
         """
         Converts a string into a sequence of tokens ids, using the tokenizer.
         """
-        tokens_ids = self.tokenizer.instruct_tokenizer.tokenizer.encode(
-            text, bos=add_special_tokens, eos=add_special_tokens
-        )
+        add_eos = add_special_tokens and self._mode == ValidationMode.finetuning
+        tokens_ids = self.tokenizer.instruct_tokenizer.tokenizer.encode(text, bos=add_special_tokens, eos=add_eos)
         return tokens_ids
 
     def tokenize(self, text: TextInput, **kwargs) -> list[str]:
@@ -651,7 +722,7 @@ def tokenize(self, text: TextInput, **kwargs) -> list[str]:
                 Will raise an error if used.
 
         Returns:
-            `List[str]`: The list of tokens.
+            `list[str]`: The list of tokens.
         """
         if kwargs:
             raise ValueError(f"Kwargs {list(kwargs.keys())} are not supported by `MistralCommonBackend.tokenize`.")
@@ -751,7 +822,7 @@ def get_input_ids(text):
 
         return BatchEncoding(batch_outputs)
 
-    def _all_special_ids(self) -> set[int]:
+    def _get_all_special_ids(self) -> set[int]:
         if self._tokenizer_type == MistralTokenizerType.tekken:
             return {t["rank"] for t in self.tokenizer.instruct_tokenizer.tokenizer._all_special_tokens}
         elif self._tokenizer_type == MistralTokenizerType.spm:
@@ -767,9 +838,9 @@ def get_special_tokens_mask(
         special tokens using the tokenizer `prepare_for_model` or `encode_plus` methods.
 
         Args:
-            token_ids_0 (`List[int]`):
+            token_ids_0 (`list[int]`):
                 List of ids of the sequence.
-            token_ids_1 (`List[int]`, *optional*):
+            token_ids_1 (`list[int]`, *optional*):
                 Not supported by `MistralCommonBackend`. Kept to match the interface of `PreTrainedTokenizerBase`.
             already_has_special_tokens (`bool`, *optional*, defaults to `False`):
                 Whether or not the token list is already formatted with special tokens for the model.
@@ -786,9 +857,7 @@ def get_special_tokens_mask(
                 "`already_has_special_tokens` is not supported by `MistralCommonBackend` and should be `False`."
             )
 
-        all_special_ids = self._all_special_ids()  # cache the ids
-
-        special_tokens_mask = [1 if token in all_special_ids else 0 for token in token_ids_0]
+        special_tokens_mask = [1 if token in self._all_special_tokens_ids else 0 for token in token_ids_0]
         return special_tokens_mask
 
     def _batch_prepare_for_model(
@@ -882,7 +951,7 @@ def prepare_for_model(
         manages a moving window (with user defined stride) for overflowing tokens.
 
         Args:
-            ids (`List[int]`):
+            ids (`list[int]`):
                 Tokenized input ids of the first sequence.
             pair_ids (`None`, *optional*):
                 Not supported by `MistralCommonBackend`. Kept to match the interface of `PreTrainedTokenizerBase`.
@@ -1081,7 +1150,7 @@ def _pad(
 
         Args:
             encoded_inputs:
-                Dictionary of tokenized inputs (`List[int]`) or batch of tokenized inputs (`List[List[int]]`).
+                Dictionary of tokenized inputs (`list[int]`) or batch of tokenized inputs (`list[list[int]]`).
             max_length: maximum length of the returned list and optionally padding length (see below).
                 Will truncate by taking into account the special tokens.
             padding_strategy: PaddingStrategy to use for padding.
@@ -1171,13 +1240,13 @@ def pad(
         </Tip>
 
         Args:
-            encoded_inputs ([`BatchEncoding`], list of [`BatchEncoding`], `Dict[str, List[int]]`, `Dict[str, List[List[int]]` or `List[Dict[str, List[int]]]`):
-                Tokenized inputs. Can represent one input ([`BatchEncoding`] or `Dict[str, List[int]]`) or a batch of
-                tokenized inputs (list of [`BatchEncoding`], *Dict[str, List[List[int]]]* or *List[Dict[str,
-                List[int]]]*) so you can use this method during preprocessing as well as in a PyTorch Dataloader
+            encoded_inputs ([`BatchEncoding`], list of [`BatchEncoding`], `Dict[str, list[int]]`, `Dict[str, list[list[int]]` or `List[Dict[str, list[int]]]`):
+                Tokenized inputs. Can represent one input ([`BatchEncoding`] or `Dict[str, list[int]]`) or a batch of
+                tokenized inputs (list of [`BatchEncoding`], *Dict[str, list[list[int]]]* or *List[Dict[str,
+                list[int]]]*) so you can use this method during preprocessing as well as in a PyTorch Dataloader
                 collate function.
 
-                Instead of `List[int]` you can have tensors (numpy arrays, PyTorch tensors), see
+                Instead of `list[int]` you can have tensors (numpy arrays, PyTorch tensors), see
                 the note above for the return type.
             padding (`bool`, `str` or [`~utils.PaddingStrategy`], *optional*, defaults to `True`):
                  Select a strategy to pad the returned sequences (according to the model's padding side and padding
@@ -1316,7 +1385,7 @@ def truncate_sequences(
         Truncates a sequence pair in-place following the strategy.
 
         Args:
-            ids (`List[int]`):
+            ids (`list[int]`):
                 Tokenized input ids. Can be obtained from a string by chaining the `tokenize` and
                 `convert_tokens_to_ids` methods.
             pair_ids (`None`, *optional*):
@@ -1335,7 +1404,7 @@ def truncate_sequences(
                 sequence returned. The value of this argument defines the number of additional tokens.
 
         Returns:
-            `Tuple[List[int], None, List[int]]`: The truncated `ids` and the list of
+            `Tuple[list[int], None, list[int]]`: The truncated `ids` and the list of
             overflowing tokens. `None` is returned to match Transformers signature.
         """
         if kwargs:
@@ -1442,7 +1511,7 @@ def apply_chat_template(
                 Will raise an error if used.
 
         Returns:
-            `Union[str, List[int], List[str], List[List[int]], BatchEncoding]`: A list of token ids representing the tokenized chat so far, including control
+            `Union[str, list[int], list[str], list[list[int]], BatchEncoding]`: A list of token ids representing the tokenized chat so far, including control
             tokens. This output is ready to pass to the model, either directly or via methods like `generate()`.
         """
         if kwargs:
@@ -1617,7 +1686,7 @@ def __call__(
         sequences.
 
         Args:
-            text (`str`, `List[str]`, `List[List[str]]`, *optional*):
+            text (`str`, `list[str]`, `list[list[str]]`, *optional*):
                 The sequence or batch of sequences to be encoded. Each sequence can be a string or a list of int
                 (encoded strings).
             text_pair (`None`, *optional*):
@@ -1657,8 +1726,8 @@ def _is_valid_text_input(t):
 
         if not _is_valid_text_input(text):
             raise ValueError(
-                "text input must be of type `str` (single example), `List[str]` (batch or single encoded example) "
-                "or `List[List[int]]` (batch of encoded examples)."
+                "text input must be of type `str` (single example), `list[str]` (batch or single encoded example) "
+                "or `list[list[int]]` (batch of encoded examples)."
             )
 
         is_batched = isinstance(text, (list, tuple)) and isinstance(text[0], (str, list, tuple))
@@ -1712,7 +1781,7 @@ def from_pretrained(
         cls,
         pretrained_model_name_or_path: str | os.PathLike,
         *init_inputs,
-        mode: ValidationMode = ValidationMode.test,
+        mode: Union[str, ValidationMode] = ValidationMode.test,
         cache_dir: str | os.PathLike | None = None,
         force_download: bool = False,
         local_files_only: bool = False,
@@ -1737,8 +1806,11 @@ def from_pretrained(
                 - A path to a *directory* containing the tokenizer config, for instance saved
                   using the [`MistralCommonBackend.tokenization_mistral_common.save_pretrained`] method, e.g.,
                   `./my_model_directory/`.
-            mode (`ValidationMode`, *optional*, defaults to `ValidationMode.test`):
-                Validation mode for the `MistralTokenizer` tokenizer.
+            mode (`Union[str, ValidationMode]`, *optional*, defaults to `ValidationMode.test`):
+                Validation mode for the `MistralTokenizer` tokenizer. Possible values are:
+                - `"finetuning"` or `ValidationMode.finetuning`: The finetuning mode.
+                - `"test"` or `ValidationMode.test`: The test mode.
+                It changes how the tokenizer validates the input and prepare the request to the model.
             cache_dir (`str` or `os.PathLike`, *optional*):
                 Path to a directory in which a downloaded predefined tokenizer vocabulary files should be cached if the
                 standard cache should not be used.
@@ -1784,6 +1856,8 @@ def from_pretrained(
         ):
             raise ValueError(f"Some kwargs in {kwargs} are not supported by `MistralCommonBackend.from_pretrained`.")
 
+        mode = cls._get_validation_mode(mode)
+
         if not os.path.isdir(pretrained_model_name_or_path):
             tokenizer_path = download_tokenizer_from_hf_hub(
                 repo_id=pretrained_model_name_or_path,
@@ -1894,3 +1968,21 @@ def save_pretrained(
             )
 
         return (str(save_directory / self._tokenizer_path.name),)
+
+    @staticmethod
+    def _get_validation_mode(mode: Union[str, ValidationMode]) -> ValidationMode:
+        """Get the validation mode from a string or a ValidationMode."""
+        _invalid_mode_msg = (
+            f"Invalid `mistral-common` tokenizer mode: {mode}. Possible values are 'finetuning' or 'test'."
+        )
+        if isinstance(mode, str):
+            try:
+                mode = ValidationMode[mode]
+            except KeyError:
+                raise ValueError(_invalid_mode_msg)
+        elif not isinstance(mode, (str, ValidationMode)):
+            raise ValueError(_invalid_mode_msg)
+
+        if mode not in [ValidationMode.finetuning, ValidationMode.test]:
+            raise ValueError(_invalid_mode_msg)
+        return mode

filename: tests/test_tokenization_mistral_common.py
patch: @@ -17,6 +17,7 @@
 import io
 import tempfile
 import unittest
+from unittest.mock import patch
 
 import numpy as np
 import torch
@@ -33,6 +34,9 @@
     import mistral_common.tokens.tokenizers
     from mistral_common.exceptions import InvalidMessageStructureException
     from mistral_common.protocol.instruct.request import ChatCompletionRequest
+    from mistral_common.protocol.instruct.validator import (
+        ValidationMode,
+    )
     from mistral_common.protocol.transcription.request import TranscriptionRequest
     from mistral_common.tokens.tokenizers.base import SpecialTokenPolicy
     from mistral_common.tokens.tokenizers.mistral import MistralTokenizer
@@ -206,7 +210,7 @@ def test_encode(self):
 
         # Test 1:
         # encode with add_special_tokens
-        expected_with_special = self.ref_tokenizer.instruct_tokenizer.tokenizer.encode(string, bos=True, eos=True)
+        expected_with_special = self.ref_tokenizer.instruct_tokenizer.tokenizer.encode(string, bos=True, eos=False)
         tokens_with_special = self.tokenizer.encode(string, add_special_tokens=True)
         self.assertEqual(tokens_with_special, expected_with_special)
 
@@ -216,6 +220,13 @@ def test_encode(self):
         tokens_without_special = self.tokenizer.encode(string, add_special_tokens=False)
         self.assertEqual(tokens_without_special, expected_without_special)
 
+        # Test 3:
+        # encode with add_special_tokens and mode finetuning
+        expected_with_special_ft = self.ref_tokenizer.instruct_tokenizer.tokenizer.encode(string, bos=True, eos=True)
+        with patch.object(self.tokenizer, "_mode", ValidationMode.finetuning):
+            tokens_with_special_ft = self.tokenizer.encode(string, add_special_tokens=True)
+        self.assertEqual(tokens_with_special_ft, expected_with_special_ft)
+
         # Test 3:
         # encode with return_tensors
         tokens_with_return_tensors = self.tokenizer.encode(string, add_special_tokens=False, return_tensors="pt")
@@ -273,7 +284,10 @@ def test_encode(self):
         ) + expected_long
         self.assertEqual(tokens_with_padding_and_truncation, expected_long_padding)
 
-        # Test encode with unsupported kwargs
+        # Test 8:
+        # encode empty string
+        self.assertEqual(self.tokenizer.encode("", add_special_tokens=False), [])
+
         with self.assertRaises(
             ValueError, msg="Kwargs [unk_args] are not supported by `MistralCommonBackend.encode`."
         ):
@@ -302,12 +316,57 @@ def test_decode(self):
         )
 
         # Test 3:
+        # decode one token
+        self.assertEqual(self.tokenizer.decode(tokens_ids[0], skip_special_tokens=False), "
")
+
+        # Test 4:
+        # decode numpy
+        self.assertEqual(self.tokenizer.decode(np.array(tokens_ids), skip_special_tokens=True), string)
+
+        # Test 5:
+        # decode empty string
+        self.assertEqual(self.tokenizer.decode([], skip_special_tokens=True), "")
+
+        # Test 6:
         # decode with unsupported kwargs
         with self.assertRaises(
             ValueError, msg="Kwargs [unk_args] are not supported by `MistralCommonBackend.decode`."
         ):
             self.tokenizer.decode(tokens_ids, skip_special_tokens=False, unk_args="")
 
+    def test_decode_on_batch(self):
+        string = "Hello, world!"
+        string_with_space = "Hello, world !"
+
+        batch_tokens_ids = [
+            self.ref_tokenizer.instruct_tokenizer.tokenizer.encode(string, bos=True, eos=True),
+            self.ref_tokenizer.instruct_tokenizer.tokenizer.encode(string_with_space, bos=True, eos=True),
+        ]
+
+        # Test 1:
+        # batch_decode with and without skip_special_tokens
+        self.assertEqual(
+            self.tokenizer.decode(batch_tokens_ids, skip_special_tokens=True),
+            [string, string_with_space],
+        )
+        self.assertEqual(
+            self.tokenizer.decode(batch_tokens_ids, skip_special_tokens=False),
+            ["
" + string + "
", "
" + string_with_space + "
"],
+        )
+        self.assertEqual(
+            self.tokenizer.decode(batch_tokens_ids, skip_special_tokens=True, clean_up_tokenization_spaces=True),
+            ["Hello, world!", "Hello, world!"],
+        )
+
+        # Test 3:
+        # decode numpy
+        self.assertEqual(
+            self.tokenizer.decode(
+                np.array(batch_tokens_ids), skip_special_tokens=True, clean_up_tokenization_spaces=True
+            ),
+            ["Hello, world!", "Hello, world!"],
+        )
+
     def test_decode_transcription_mode(self):
         # in the specific case of Voxtral, the added f"lang:xx" (always a two char language code since it follows ISO 639-1 alpha-2 format)
         # is not considered as a special token by mistral-common and is encoded/ decoded as normal text.
@@ -357,7 +416,23 @@ def test_batch_decode(self):
             ["Hello, world!", "Hello, world!"],
         )
 
-        # Test 2:
+        # Test 3:
+        # decode numpy
+        self.assertEqual(
+            self.tokenizer.batch_decode(
+                np.array(batch_tokens_ids), skip_special_tokens=True, clean_up_tokenization_spaces=True
+            ),
+            ["Hello, world!", "Hello, world!"],
+        )
+
+        # Test 4:
+        # decode empty list
+        self.assertEqual(self.tokenizer.batch_decode([], skip_special_tokens=True), [])
+        self.assertEqual(
+            self.tokenizer.batch_decode([batch_tokens_ids[0], []], skip_special_tokens=True), [string, ""]
+        )
+
+        # Test 5:
         # batch_decode with unsupported kwargs
         with self.assertRaises(
             ValueError, msg="Kwargs [unk_args] are not supported by `MistralCommonBackend.batch_decode`."
@@ -382,6 +457,11 @@ def test_convert_ids_to_tokens(self):
         tokens = self.tokenizer.convert_ids_to_tokens(ids, skip_special_tokens=True)
         self.assertEqual(tokens, expected_tokens)
 
+        # Test 3:
+        # with empty list
+        tokens = self.tokenizer.convert_ids_to_tokens([])
+        self.assertEqual(tokens, [])
+
         with self.assertRaises(ValueError):
             self.tokenizer.convert_ids_to_tokens(ids[0], skip_special_tokens=True)
         token = self.tokenizer.convert_ids_to_tokens(ids[1], skip_special_tokens=True)
@@ -401,15 +481,27 @@ def test_convert_tokens_to_ids(self):
         self.assertEqual(id, expected_ids[0])
         self.assertEqual(id, self.tokenizer.convert_tokens_to_ids(tokens[0]))
 
+        # Test 3:
+        # with empty list
+        ids = self.tokenizer.convert_tokens_to_ids([])
+        self.assertEqual(ids, [])
+
     def test_tokenize(self):
         string = "Hello world!"
+        # Test 1:
+        # with string
         expected_tokens = [
             self.ref_tokenizer.instruct_tokenizer.tokenizer.id_to_piece(id)
             for id in self.ref_tokenizer.instruct_tokenizer.tokenizer.encode(string, bos=False, eos=False)
         ]
         tokens = self.tokenizer.tokenize(string)
         self.assertEqual(tokens, expected_tokens)
 
+        # Test 2:
+        # with empty string
+        tokens = self.tokenizer.tokenize("")
+        self.assertEqual(tokens, [])
+
         with self.assertRaises(
             ValueError, msg="Kwargs [add_special_tokens] are not supported by `MistralCommonBackend.tokenize`."
         ):
@@ -1473,7 +1565,7 @@ def test_call(self):
         # Test 1:
         # default case
         text = "Hello world!"
-        expected_tokens = self.ref_tokenizer.instruct_tokenizer.tokenizer.encode(text, bos=True, eos=True)
+        expected_tokens = self.ref_tokenizer.instruct_tokenizer.tokenizer.encode(text, bos=True, eos=False)
         tokens = self.tokenizer(text)
         self.assertIsInstance(tokens, BatchEncoding)
         self.assertEqual(tokens["input_ids"], expected_tokens)
@@ -1498,7 +1590,7 @@ def test_call(self):
         tokens = self.tokenizer(text, return_special_tokens_mask=True)
         self.assertEqual(tokens["input_ids"], expected_tokens)
         self.assertEqual(tokens["attention_mask"], [1] * len(expected_tokens))
-        self.assertEqual(tokens["special_tokens_mask"], [1] + [0] * (len(expected_tokens) - 2) + [1])
+        self.assertEqual(tokens["special_tokens_mask"], [1] + [0] * (len(expected_tokens) - 1))
 
         # Test 5:
         # add_special_tokens=False
@@ -1509,6 +1601,24 @@ def test_call(self):
         self.assertEqual(tokens["attention_mask"], [1] * len(expected_tokens))
         self.assertEqual(tokens["special_tokens_mask"], [0] * len(expected_tokens))
 
+        # Test 6:
+        # add_special_tokens=False and mode finetuning
+        text = "Hello world!"
+        expected_tokens = self.ref_tokenizer.instruct_tokenizer.tokenizer.encode(text, bos=True, eos=True)
+        with patch.object(self.tokenizer, "_mode", ValidationMode.finetuning):
+            tokens = self.tokenizer(text, return_special_tokens_mask=True)
+        self.assertIsInstance(tokens, BatchEncoding)
+        self.assertEqual(tokens["input_ids"], expected_tokens)
+        self.assertEqual(tokens["attention_mask"], [1] * len(expected_tokens))
+        self.assertEqual(tokens["special_tokens_mask"], [1] + [0] * (len(expected_tokens) - 2) + [1])
+
+        # Test 7:
+        # empty string
+        tokens = self.tokenizer("", add_special_tokens=False)
+        self.assertIsInstance(tokens, BatchEncoding)
+        self.assertEqual(tokens["input_ids"], [])
+        self.assertEqual(tokens["attention_mask"], [])
+
         with self.assertRaises(
             ValueError, msg="Kwargs [wrong_kwarg] are not supported by `MistralCommonBackend.__call__`."
         ):
@@ -1534,7 +1644,7 @@ def test_call_with_truncation(self):
         # Test 1:
         # truncation=True or "longest_first" or TruncationStrategy.LONGEST_FIRST
         text = "Hello world!" * 10
-        expected_tokens = self.ref_tokenizer.instruct_tokenizer.tokenizer.encode(text, bos=True, eos=True)
+        expected_tokens = self.ref_tokenizer.instruct_tokenizer.tokenizer.encode(text, bos=True, eos=False)
 
         for truncation in [True, "longest_first", TruncationStrategy.LONGEST_FIRST]:
             tokens = self.tokenizer(text, truncation=True, max_length=10, return_special_tokens_mask=True)
@@ -1550,7 +1660,7 @@ def test_call_with_truncation(self):
             self.assertIsInstance(tokens, BatchEncoding)
             self.assertEqual(tokens["input_ids"], expected_tokens)
             self.assertEqual(tokens["attention_mask"], [1] * len(expected_tokens))
-            self.assertEqual(tokens["special_tokens_mask"], [1] + [0] * (len(expected_tokens) - 2) + [1])
+            self.assertEqual(tokens["special_tokens_mask"], [1] + [0] * (len(expected_tokens) - 1))
 
         # Test 3:
         # truncation=True or "longest_first" or TruncationStrategy.LONGEST_FIRST with return_overflowing_tokens=True and stride
@@ -1583,7 +1693,7 @@ def test_call_with_truncation(self):
 
     def test_call_with_padding(self):
         text = "Hello world!"
-        expected_tokens = self.ref_tokenizer.instruct_tokenizer.tokenizer.encode(text, bos=True, eos=True)
+        expected_tokens = self.ref_tokenizer.instruct_tokenizer.tokenizer.encode(text, bos=True, eos=False)
 
         # Test 1:
         # padding=False or padding=True or "do_not_pad" or PaddingStrategy.DO_NOT_PAD or padding="longest" or PaddingStrategy.LONGEST
@@ -1592,7 +1702,7 @@ def test_call_with_padding(self):
             self.assertIsInstance(tokens, BatchEncoding)
             self.assertEqual(tokens["input_ids"], expected_tokens)
             self.assertEqual(tokens["attention_mask"], [1] * len(expected_tokens))
-            self.assertEqual(tokens["special_tokens_mask"], [1] + [0] * (len(expected_tokens) - 2) + [1])
+            self.assertEqual(tokens["special_tokens_mask"], [1] + [0] * (len(expected_tokens) - 1))
 
         # Test 2:
         # padding="max_length" or PaddingStrategy.MAX_LENGTH
@@ -1602,9 +1712,7 @@ def test_call_with_padding(self):
             num_padding = 20 - len(expected_tokens)
             self.assertEqual(tokens["input_ids"], num_padding * [self.tokenizer.pad_token_id] + expected_tokens)
             self.assertEqual(tokens["attention_mask"], num_padding * [0] + [1] * len(expected_tokens))
-            self.assertEqual(
-                tokens["special_tokens_mask"], num_padding * [1] + [1] + [0] * (len(expected_tokens) - 2) + [1]
-            )
+            self.assertEqual(tokens["special_tokens_mask"], num_padding * [1] + [1] + [0] * (len(expected_tokens) - 1))
 
         # Test 3:
         # pad_to_multiple_of
@@ -1615,9 +1723,7 @@ def test_call_with_padding(self):
         num_padding = 16 - len(expected_tokens)
         self.assertEqual(tokens["input_ids"], num_padding * [self.tokenizer.pad_token_id] + expected_tokens)
         self.assertEqual(tokens["attention_mask"], num_padding * [0] + [1] * len(expected_tokens))
-        self.assertEqual(
-            tokens["special_tokens_mask"], num_padding * [1] + [1] + [0] * (len(expected_tokens) - 2) + [1]
-        )
+        self.assertEqual(tokens["special_tokens_mask"], num_padding * [1] + [1] + [0] * (len(expected_tokens) - 1))
 
         # Test 4:
         # padding="max_length" and padding_side="right"
@@ -1628,15 +1734,15 @@ def test_call_with_padding(self):
         num_padding = 20 - len(expected_tokens)
         self.assertEqual(tokens["input_ids"], expected_tokens + num_padding * [self.tokenizer.pad_token_id])
         self.assertEqual(tokens["attention_mask"], [1] * len(expected_tokens) + num_padding * [0])
-        self.assertEqual(
-            tokens["special_tokens_mask"], [1] + [0] * (len(expected_tokens) - 2) + [1] + num_padding * [1]
-        )
+        self.assertEqual(tokens["special_tokens_mask"], [1] + [0] * (len(expected_tokens) - 1) + num_padding * [1])
 
     def test_batch_call(self):
         # Test 1:
         # default case
         text = ["Hello world!", "Hello world! Longer"]
-        expected_tokens = [self.ref_tokenizer.instruct_tokenizer.tokenizer.encode(t, bos=True, eos=True) for t in text]
+        expected_tokens = [
+            self.ref_tokenizer.instruct_tokenizer.tokenizer.encode(t, bos=True, eos=False) for t in text
+        ]
         tokens = self.tokenizer(text)
         self.assertIsInstance(tokens, BatchEncoding)
         self.assertEqual(tokens["input_ids"], expected_tokens)
@@ -1682,15 +1788,12 @@ def test_batch_call(self):
                 torch.Tensor(
                     (len(expected_tokens[1]) - len(expected_tokens[0])) * [1]
                     + [1]
-                    + [0] * (len(expected_tokens[0]) - 2)
-                    + [1]
+                    + [0] * (len(expected_tokens[0]) - 1)
                 ),
             )
         )
         self.assertTrue(
-            torch.equal(
-                tokens["special_tokens_mask"][1], torch.Tensor([1] + [0] * (len(expected_tokens[1]) - 2) + [1])
-            )
+            torch.equal(tokens["special_tokens_mask"][1], torch.Tensor([1] + [0] * (len(expected_tokens[1]) - 1)))
         )
 
         # Test 4:
@@ -1704,11 +1807,45 @@ def test_batch_call(self):
         self.assertEqual(tokens["attention_mask"], [[1] * len(t) for t in expected_tokens])
         self.assertEqual(tokens["special_tokens_mask"], [[0] * len(t) for t in expected_tokens])
 
+        # Test 5:
+        # add_special_tokens=True and mode = finetuning
+        expected_tokens = [self.ref_tokenizer.instruct_tokenizer.tokenizer.encode(t, bos=True, eos=True) for t in text]
+        with patch.object(self.tokenizer, "_mode", ValidationMode.finetuning):
+            tokens = self.tokenizer(text, add_special_tokens=True, return_special_tokens_mask=True)
+        self.assertIsInstance(tokens, BatchEncoding)
+        self.assertEqual(tokens["input_ids"], expected_tokens)
+        self.assertEqual(
+            tokens["special_tokens_mask"],
+            [[1] + [0] * (len(expected_tokens[0]) - 2) + [1], [1] + [0] * (len(expected_tokens[1]) - 2) + [1]],
+        )
+
+        # Test 6:
+        # empty string in batch
+        expected_tokens = [
+            self.ref_tokenizer.instruct_tokenizer.tokenizer.encode(t, bos=False, eos=False) for t in text
+        ]
+        expected_tokens.append([])
+        tokens = self.tokenizer(text + [""], add_special_tokens=False, return_special_tokens_mask=True)
+        self.assertIsInstance(tokens, BatchEncoding)
+        self.assertEqual(tokens["input_ids"], expected_tokens)
+        self.assertEqual(tokens["attention_mask"], [[1] * len(t) for t in expected_tokens])
+        self.assertEqual(tokens["special_tokens_mask"], [[0] * len(t) for t in expected_tokens])
+
+        # Test 7:
+        # empty batch
+        tokens = self.tokenizer([""], add_special_tokens=False, return_special_tokens_mask=True)
+        self.assertIsInstance(tokens, BatchEncoding)
+        self.assertEqual(tokens["input_ids"], [[]])
+        self.assertEqual(tokens["attention_mask"], [[]])
+        self.assertEqual(tokens["special_tokens_mask"], [[]])
+
     def test_batch_call_with_truncation(self):
         # Test 1:
         # truncation=True
         text = ["Hello world!", "Hello world! Longer" * 10]
-        expected_tokens = [self.ref_tokenizer.instruct_tokenizer.tokenizer.encode(t, bos=True, eos=True) for t in text]
+        expected_tokens = [
+            self.ref_tokenizer.instruct_tokenizer.tokenizer.encode(t, bos=True, eos=False) for t in text
+        ]
 
         for truncation in [True, "longest_first", TruncationStrategy.LONGEST_FIRST]:
             tokens = self.tokenizer(text, truncation=True, max_length=10, return_special_tokens_mask=True)
@@ -1729,7 +1866,7 @@ def test_batch_call_with_truncation(self):
             self.assertEqual(tokens["attention_mask"], [[1] * len(t) for t in expected_tokens])
             self.assertEqual(
                 tokens["special_tokens_mask"],
-                [[1] + [0] * (len(t) - 2) + [1] for t in expected_tokens],
+                [[1] + [0] * (len(t) - 1) for t in expected_tokens],
             )
 
         # Test 3:
@@ -1764,15 +1901,17 @@ def test_batch_call_with_padding(self):
         # Test 1:
         # padding=False or padding=True or "do_not_pad" or PaddingStrategy.DO_NOT_PAD or padding="longest" or PaddingStrategy.LONGEST
         text = ["Hello world!", "Hello world! Longer"]
-        expected_tokens = [self.ref_tokenizer.instruct_tokenizer.tokenizer.encode(t, bos=True, eos=True) for t in text]
+        expected_tokens = [
+            self.ref_tokenizer.instruct_tokenizer.tokenizer.encode(t, bos=True, eos=False) for t in text
+        ]
         for padding in [False, "do_not_pad", PaddingStrategy.DO_NOT_PAD]:
             tokens = self.tokenizer(text, padding=padding, return_special_tokens_mask=True)
             self.assertIsInstance(tokens, BatchEncoding)
             self.assertEqual(tokens["input_ids"], expected_tokens)
             self.assertEqual(tokens["attention_mask"], [[1] * len(t) for t in expected_tokens])
             self.assertEqual(
                 tokens["special_tokens_mask"],
-                [[1] + [0] * (len(t) - 2) + [1] for t in expected_tokens],
+                [[1] + [0] * (len(t) - 1) for t in expected_tokens],
             )
 
         # Test 2:
@@ -1798,8 +1937,8 @@ def test_batch_call_with_padding(self):
             self.assertEqual(
                 tokens["special_tokens_mask"],
                 [
-                    num_padding[0] * [1] + [1] + [0] * (len(expected_tokens[0]) - 2) + [1],
-                    num_padding[1] * [1] + [1] + [0] * (len(expected_tokens[1]) - 2) + [1],
+                    num_padding[0] * [1] + [1] + [0] * (len(expected_tokens[0]) - 1),
+                    num_padding[1] * [1] + [1] + [0] * (len(expected_tokens[1]) - 1),
                 ],
             )
 
@@ -1826,8 +1965,8 @@ def test_batch_call_with_padding(self):
             self.assertEqual(
                 tokens["special_tokens_mask"],
                 [
-                    num_padding[0] * [1] + [1] + [0] * (len(expected_tokens[0]) - 2) + [1],
-                    num_padding[1] * [1] + [1] + [0] * (len(expected_tokens[1]) - 2) + [1],
+                    num_padding[0] * [1] + [1] + [0] * (len(expected_tokens[0]) - 1),
+                    num_padding[1] * [1] + [1] + [0] * (len(expected_tokens[1]) - 1),
                 ],
             )
 
@@ -1855,8 +1994,8 @@ def test_batch_call_with_padding(self):
         self.assertEqual(
             tokens["special_tokens_mask"],
             [
-                num_padding[0] * [1] + [1] + [0] * (len(expected_tokens[0]) - 2) + [1],
-                num_padding[1] * [1] + [1] + [0] * (len(expected_tokens[1]) - 2) + [1],
+                num_padding[0] * [1] + [1] + [0] * (len(expected_tokens[0]) - 1),
+                num_padding[1] * [1] + [1] + [0] * (len(expected_tokens[1]) - 1),
             ],
         )
 
@@ -1885,8 +2024,8 @@ def test_batch_call_with_padding(self):
             self.assertEqual(
                 tokens["special_tokens_mask"],
                 [
-                    [1] + [0] * (len(expected_tokens[0]) - 2) + [1] + num_padding[0] * [1],
-                    [1] + [0] * (len(expected_tokens[1]) - 2) + [1] + num_padding[1] * [1],
+                    [1] + [0] * (len(expected_tokens[0]) - 1) + num_padding[0] * [1],
+                    [1] + [0] * (len(expected_tokens[1]) - 1) + num_padding[1] * [1],
                 ],
             )
 
@@ -1896,7 +2035,9 @@ def test_batch_call_with_padding_and_truncation(self):
         # and truncation=True or "longest_first" or TruncationStrategy.LONGEST_FIRST
         # and max_length
         text = ["Hello world!", "Hello world! Longer" * 10]
-        expected_tokens = [self.ref_tokenizer.instruct_tokenizer.tokenizer.encode(t, bos=True, eos=True) for t in text]
+        expected_tokens = [
+            self.ref_tokenizer.instruct_tokenizer.tokenizer.encode(t, bos=True, eos=False) for t in text
+        ]
         for padding in [True, "longest", PaddingStrategy.LONGEST, "max_length", PaddingStrategy.MAX_LENGTH]:
             for truncation in [True, "longest_first", TruncationStrategy.LONGEST_FIRST]:
                 tokens = self.tokenizer(
@@ -1956,3 +2097,32 @@ def test_get_vocab(self):
             self.assertEqual(
                 self.ref_tokenizer.decode([id_token], special_token_policy=SpecialTokenPolicy.KEEP), token
             )
+
+    def test_get_validation_mode(self):
+        for mode, expected in [
+            ("test", ValidationMode.test),
+            (ValidationMode.test, ValidationMode.test),
+            ("finetuning", ValidationMode.finetuning),
+            (ValidationMode.finetuning, ValidationMode.finetuning),
+        ]:
+            self.assertEqual(MistralCommonBackend._get_validation_mode(mode), expected)
+
+        for invalid_mode in [("serving", ValidationMode.serving, "invalid", 1)]:
+            with self.assertRaises(ValueError):
+                MistralCommonBackend._get_validation_mode(invalid_mode)
+
+    def test_all_special_ids(self):
+        with patch.object(self.tokenizer, "_all_special_tokens_ids", {1, 0}):
+            self.assertEqual(self.tokenizer.all_special_ids, [0, 1])
+
+        spm_tokenizer = self._get_spm_tokenizer()
+        with patch.object(spm_tokenizer, "_all_special_tokens_ids", {1, 0}):
+            self.assertEqual(spm_tokenizer.all_special_ids, [0, 1])
+
+    def test_all_special_tokens(self):
+        with patch.object(self.tokenizer, "_all_special_tokens_ids", {1, 0}):
+            self.assertEqual(self.tokenizer.all_special_tokens, ["<unk>", "
"])
+
+        spm_tokenizer = self._get_spm_tokenizer()
+        with patch.object(spm_tokenizer, "_all_special_tokens_ids", {1, 0}):
+            self.assertEqual(spm_tokenizer.all_special_tokens, ["<unk>", "
"])