Repository: huggingface/transformers
Pull request: [Gemma] Update conversion scripts for Transformers v5 Comaptibility

# What does this PR do?

Updates the weights conversion scripts for Gemma to:

* Use the new `SentencePieceExtractor` class to get the vocab and merges from the SPM
* Always initialize and save the unified `GemmaTokenizer` class 

## Before submitting
- [ ] This PR fixes a typo or improves the docs (you can dismiss the other checks if that's the case).
- [x] Did you read the [contributor guideline](https://github.com/huggingface/transformers/blob/main/CONTRIBUTING.md#create-a-pull-request),
      Pull Request section?
- [ ] Was this discussed/approved via a Github issue or the [forum](https://discuss.huggingface.co/)? Please add a link
      to it if that's the case.
- [x] Did you make sure to update the documentation with your changes? Here are the
      [documentation guidelines](https://github.com/huggingface/transformers/tree/main/docs), and
      [here are tips on formatting docstrings](https://github.com/huggingface/transformers/tree/main/docs#writing-source-documentation).
- [x] Did you write any new necessary tests?


## Who can review?

Anyone in the community is free to review the PR once the tests have passed. Feel free to tag
members/contributors who may be interested in your PR.

cc/ @ArthurZucker @Cyrilvallez

Changed files:
filename: src/transformers/models/gemma/convert_gemma_weights_to_hf.py
patch: @@ -13,22 +13,13 @@
 # limitations under the License.
 import argparse
 import os
-import warnings
 
 import torch
 
 from transformers import GemmaConfig, GemmaForCausalLM, GemmaTokenizer
+from transformers.tokenization_utils_sentencepiece import SentencePieceExtractor
 
 
-try:
-    from transformers import GemmaTokenizerFast
-except ImportError as e:
-    warnings.warn(e)
-    warnings.warn(
-        "The converted tokenizer will be the `slow` tokenizer. To use the fast, update your `tokenizers` library and re-run the tokenizer conversion"
-    )
-    GemmaTokenizerFast = None
-
 """
 Sample usage:
 
@@ -126,9 +117,14 @@ def write_model(save_path, input_base_path, config, push_to_hub=False, dtype=tor
 
 def write_tokenizer(input_tokenizer_path, save_path, push_to_hub=False):
     # Initialize the tokenizer based on the `spm` model
-    tokenizer_class = GemmaTokenizer if GemmaTokenizerFast is None else GemmaTokenizerFast
-    print(f"Saving a {tokenizer_class.__name__} to {save_path}.")
-    tokenizer = tokenizer_class(input_tokenizer_path)
+    print(f"Saving a GemmaTokenizer to {save_path}.")
+    sentencepiece_extractor = SentencePieceExtractor(input_tokenizer_path)
+    vocab, _, merges = sentencepiece_extractor.extract()
+    tokenizer = GemmaTokenizer(
+        vocab=vocab,
+        merges=merges,
+        padding_side="left",
+    )
     if push_to_hub:
         tokenizer.push_to_hub(save_path)
     else:

filename: src/transformers/models/gemma2/convert_gemma2_weights_to_hf.py
patch: @@ -13,22 +13,13 @@
 # limitations under the License.
 import argparse
 import os
-import warnings
 
 import torch
 
 from transformers import Gemma2Config, Gemma2ForCausalLM, GemmaTokenizer
+from transformers.tokenization_utils_sentencepiece import SentencePieceExtractor
 
 
-try:
-    from transformers import GemmaTokenizerFast
-except ImportError as e:
-    warnings.warn(e)
-    warnings.warn(
-        "The converted tokenizer will be the `slow` tokenizer. To use the fast, update your `tokenizers` library and re-run the tokenizer conversion"
-    )
-    GemmaTokenizerFast = None
-
 """
 Sample usage:
 
@@ -159,9 +150,14 @@ def write_model(save_path, input_base_path, config, push_to_hub=False, dtype=tor
 
 def write_tokenizer(input_tokenizer_path, save_path, push_to_hub=False):
     # Initialize the tokenizer based on the `spm` model
-    tokenizer_class = GemmaTokenizer if GemmaTokenizerFast is None else GemmaTokenizerFast
-    print(f"Saving a {tokenizer_class.__name__} to {save_path}.")
-    tokenizer = tokenizer_class(input_tokenizer_path)
+    print(f"Saving a GemmaTokenizer to {save_path}.")
+    sentencepiece_extractor = SentencePieceExtractor(input_tokenizer_path)
+    vocab, _, merges = sentencepiece_extractor.extract()
+    tokenizer = GemmaTokenizer(
+        vocab=vocab,
+        merges=merges,
+        padding_side="left",
+    )
     if push_to_hub:
         tokenizer.push_to_hub(save_path)
     else:

filename: src/transformers/models/gemma3/convert_gemma3_weights.py
patch: @@ -41,11 +41,12 @@
     Gemma3Processor,
     Gemma3TextConfig,
     Gemma3TextModel,
-    GemmaTokenizerFast,
+    GemmaTokenizer,
     GenerationConfig,
     SiglipVisionConfig,
 )
 from transformers.image_utils import PILImageResampling
+from transformers.tokenization_utils_sentencepiece import SentencePieceExtractor
 
 
 # ==== Internal Constants and Classes ====
@@ -642,8 +643,11 @@ def main(*args):
     del model
     del state_tree
 
-    tokenizer = GemmaTokenizerFast(
-        _TOKENIZER_PATH.value,
+    sentencepiece_extractor = SentencePieceExtractor(_TOKENIZER_PATH.value)
+    vocab, _, merges = sentencepiece_extractor.extract()
+    tokenizer = GemmaTokenizer(
+        vocab=vocab,
+        merges=merges,
         add_bos_token=True,
         add_eos_token=variant == _VARIANT_EMBEDDINGGEMMA,
         padding_side="right" if variant == _VARIANT_EMBEDDINGGEMMA else "left",

filename: src/transformers/models/gemma3n/convert_gemma3n_weights.py
patch: @@ -22,8 +22,6 @@
     --output_path="$HOME/checkpoints/gemma-3n-safetensors/"
 """
 
-import json
-import os
 import re
 from collections.abc import Iterable, Mapping
 from typing import Any
@@ -43,11 +41,12 @@
     Gemma3nProcessor,
     Gemma3nTextConfig,
     Gemma3nVisionConfig,
-    GemmaTokenizerFast,
+    GemmaTokenizer,
     GenerationConfig,
     SiglipImageProcessorFast,
 )
 from transformers.image_utils import PILImageResampling
+from transformers.tokenization_utils_sentencepiece import SentencePieceExtractor
 
 
 # ==== Internal Constants and Classes ====
@@ -741,9 +740,13 @@ def main(*args):
 
     chat_template_kwargs = {"chat_template": _CHAT_TEMPLATE} if _INCLUDE_CHAT_TEMPLATE.value else {}
 
-    tokenizer = GemmaTokenizerFast(
-        _TOKENIZER_PATH.value,
+    sentencepiece_extractor = SentencePieceExtractor(_TOKENIZER_PATH.value)
+    vocab, _, merges = sentencepiece_extractor.extract()
+    tokenizer = GemmaTokenizer(
+        vocab=vocab,
+        merges=merges,
         add_bos_token=True,
+        padding_side="left",
         extra_special_tokens={
             "image_token": "<image_soft_token>",  # Should be ID=262_145
             "boi_token": "<start_of_image>",  # Should be ID=255_999
@@ -776,17 +779,6 @@ def main(*args):
 
     logging.info("Saved Gemma3nProcessor for %s to %s", variant, output_path)
 
-    # NOTE: feature_extractor and image_processor both use the same filename, preprocessor_config.json, when saved to
-    # disk, but the files are overwritten by processor.save_pretrained(). However, the configs can be unioned, saved,
-    # and loaded from the same preprocessor_config.json file, so we do that explicitly here.
-    feature_extractor_config = json.loads(feature_extractor.to_json_string())
-    image_processor_config = json.loads(image_processor.to_json_string())
-    preprocessor_config = {**feature_extractor_config, **image_processor_config}
-    with open(os.path.join(output_path, "preprocessor_config.json"), "w", encoding="utf-8") as writer:
-        writer.write(json.dumps(preprocessor_config, indent=2, sort_keys=True) + "\n")
-
-    logging.info("Saved joint preprocessor_config.json for %s to %s", variant, output_path)
-
     del feature_extractor, image_processor, processor, tokenizer
 
     generation_config = GenerationConfig(