Repository: huggingface/transformers
Pull request: Fix convert_tekken_tokenizer

# What does this PR do?

Right now the `convert_tekken_tokenizer` does not add `bos_tokens`, `eos_token`  to the special tokens via the `add_special_tokens` method.

This prevents the chat templates that expect `eos_token` and `bos_token` to work properly.

Previously this was working as when saving the tokenizer a `special_tokens_map.json` was created which is no longer the case. Unknown to me why but I'd assume this is due to the V5 refactoring ?

This PR fixes that by adding explicitly these tokens to the tokenizer and when saving they're now stored in `tokenizer_config.json`.

## Before submitting
- [ ] This PR fixes a typo or improves the docs (you can dismiss the other checks if that's the case).
- [ ] Did you read the [contributor guideline](https://github.com/huggingface/transformers/blob/main/CONTRIBUTING.md#create-a-pull-request),
      Pull Request section?
- [ ] Was this discussed/approved via a Github issue or the [forum](https://discuss.huggingface.co/)? Please add a link
      to it if that's the case.
- [ ] Did you make sure to update the documentation with your changes? Here are the
      [documentation guidelines](https://github.com/huggingface/transformers/tree/main/docs), and
      [here are tips on formatting docstrings](https://github.com/huggingface/transformers/tree/main/docs#writing-source-documentation).
- [ ] Did you write any new necessary tests?


## Who can review?

@ArthurZucker 

Changed files:
filename: src/transformers/integrations/mistral.py
patch: @@ -77,6 +77,7 @@ def convert_tekken_tokenizer(tokenizer_file: str):
     """Convert a "tekken" tokenizer to a fast Tokenizer."""
     # Tekken format -- need to use the Converter
 
+    from mistral_common.tokens.tokenizers.base import SpecialTokens
     from mistral_common.tokens.tokenizers.mistral import MistralTokenizer
 
     # Load directly using their lib
@@ -106,4 +107,15 @@ def convert_tekken_tokenizer(tokenizer_file: str):
     # Post-process
     tokenizer.add_special_tokens({"additional_special_tokens": all_special})
 
+    MAP_SPECAL = {
+        "bos_token": SpecialTokens.bos.value,
+        "eos_token": SpecialTokens.eos.value,
+        "pad_token": SpecialTokens.pad.value,
+        "unk_token": SpecialTokens.unk.value,
+    }
+
+    for special_key, special_token in MAP_SPECAL.items():
+        if special_token in all_special:
+            tokenizer.add_special_tokens({special_key: special_token})
+
     return tokenizer

filename: src/transformers/models/ministral3/convert_ministral3_weights_to_hf.py
patch: @@ -273,7 +273,6 @@ def convert_and_write_processor_and_tokenizer(
 
     tokenizer_file = os.path.join(input_dir, "tekken.json")
     tokenizer = convert_tekken_tokenizer(tokenizer_file)
-    tokenizer.add_special_tokens({"pad_token": "<pad>"})
 
     # No vision
     if isinstance(model_config, Ministral3Config):

filename: src/transformers/models/mistral3/convert_mistral3_weights_to_hf.py
patch: @@ -192,7 +192,7 @@ def convert_and_write_processor(input_dir: str, output_dir: str):
     """Convert the tokenizer and save it."""
     tokenizer_file = os.path.join(input_dir, "tekken.json")
     tokenizer = convert_tekken_tokenizer(tokenizer_file)
-    tokenizer.add_special_tokens({"pad_token": "<pad>"})
+
     chat_template = '{%- if messages[0]["role"] == "system" %}{%- set system_message = messages[0]["content"] %}{%- set loop_messages = messages[1:] %}\n{%- else %}{%- set loop_messages = messages %}{%- endif %}{{- bos_token }}{%- for message in loop_messages %}{%- if (message[\'role\'] == \'user\') != (loop.index0 % 2 == 0) %}{{- raise_exception(\'After the optional system message, conversation roles must alternate user/assistant/user/assistant/...\') }}{%- endif %}{%- if message["role"] == "user" %}{%- if loop.last and system_message is defined %}{{- "
" + system_message + "\n\n" }}{%- else %}{{ "
" }}{%- endif %}{%- endif %}{%- if message["content"] is not string %}{%- for chunk in message["content"] %}{%- if chunk["type"] == "text" %}{%- if "content" in chunk %}{{- chunk["content"] }}{%- elif "text" in chunk %}{{- chunk["text"] }}{%- endif %}{%- elif chunk["type"] == "image" %}{{- "[IMG]" }}{%- else %}{{- raise_exception("Unrecognized content type!") }}{%- endif %}{%- endfor %}{%- else %}{{- message["content"] }}{%- endif %}{%- if message["role"] == "user" %}{{- "
" }}{%- elif message["role"] == "assistant" %}{{- eos_token}}{%- else %}{{- raise_exception("Only user and assistant roles are supported, with the exception of an initial optional system message!") }}{%- endif %}{%- endfor %}'
 
     config = read_json(os.path.join(input_dir, "params.json"))