Repository: huggingface/transformers
File: tests/models/auto/test_tokenization_auto.py (Python)

Code before:
                try:
                    # this should fail because we don't trust and we're not at a terminal for interactive response
                    _ = AutoTokenizer.from_pretrained(fake_model_id, local_files_only=True, trust_remote_code=False)
                    self.fail("AutoTokenizer.from_pretrained with trust_remote_code=False should raise ValueException")
                except ValueError:
                    pass
            finally:
                os.chdir(prev_dir)

    def test_tokenization_class_priority(self):
        from transformers import AutoProcessor

        tok = AutoTokenizer.from_pretrained("mlx-community/MiniMax-M2.1-4bit")
        self.assertTrue(tok.__class__ == TokenizersBackend)

        tok = AutoTokenizer.from_pretrained("allegro/herbert-base-cased")
        self.assertTrue(tok.__class__ == HerbertTokenizer)
        with tempfile.TemporaryDirectory() as tmp_dir:
            tok.save_pretrained(tmp_dir)
            tok2 = AutoTokenizer.from_pretrained(tmp_dir)
            self.assertTrue(tok2.__class__ == HerbertTokenizer)

        # SmolLM2 has tokenizer_class=GPT2Tokenizer explicitly saved, so we should use that
        # instead of falling back to TokenizersBackend
        tok = AutoTokenizer.from_pretrained("HuggingFaceTB/SmolLM2-135M-Instruct")
        self.assertTrue(tok.__class__ == GPT2Tokenizer)

        tok = AutoProcessor.from_pretrained("mistralai/Ministral-3-8B-Instruct-2512-BF16").tokenizer
        self.assertTrue(tok.__class__ == TokenizersBackend)

        tok = AutoTokenizer.from_pretrained("HuggingFaceTB/SmolLM2-135M-Instruct")
        self.assertTrue(tok.__class__ == GPT2Tokenizer)

    def test_custom_tokenizer_with_mismatched_tokenizer_class(self):
        nop_tokenizer_code = """
import transformers

class NopTokenizer(transformers.PreTrainedTokenizer):
    special_attribute_present = True

    def get_vocab(self):
        return {}
"""

        nop_config_code = """
from transformers import PreTrainedConfig

class NopConfig(PreTrainedConfig):
    model_type = "test_unregistered_dynamic"

    def __init__(self, **kwargs):

Review comment (bug):
Isn't that exactly what we wanted to avoid? A lot of configs save the "wrong" class in a sense that they meant to use the tokenizers backend while pointing out to GPT2 or Llama for example

The goal was to avoid these exceptions when we put them into auto ourselves so that we are 100% sure it's that explicit tokenizer 

Code after:
                try:
                    # this should fail because we don't trust and we're not at a terminal for interactive response
                    _ = AutoTokenizer.from_pretrained(fake_model_id, local_files_only=True, trust_remote_code=False)
                    self.fail("AutoTokenizer.from_pretrained with trust_remote_code=False should raise ValueException")
                except ValueError:
                    pass
            finally:
                os.chdir(prev_dir)

    def test_tokenization_class_priority(self):
        from transformers import AutoProcessor

        tok = AutoTokenizer.from_pretrained("mlx-community/MiniMax-M2.1-4bit")
        self.assertTrue(tok.__class__ == TokenizersBackend)

        tok = AutoTokenizer.from_pretrained("allegro/herbert-base-cased")
        self.assertTrue(tok.__class__ == HerbertTokenizer)
        with tempfile.TemporaryDirectory() as tmp_dir:
            tok.save_pretrained(tmp_dir)
            tok2 = AutoTokenizer.from_pretrained(tmp_dir)
            self.assertTrue(tok2.__class__ == HerbertTokenizer)

        tok = AutoTokenizer.from_pretrained("HuggingFaceTB/SmolLM2-135M-Instruct")
        self.assertTrue(tok.__class__ == TokenizersBackend)

        tok = AutoProcessor.from_pretrained("mistralai/Ministral-3-8B-Instruct-2512-BF16").tokenizer
        self.assertTrue(tok.__class__ == TokenizersBackend)

        tok = AutoTokenizer.from_pretrained("HuggingFaceTB/SmolLM2-135M-Instruct")
        self.assertTrue(tok.__class__ == TokenizersBackend)

    def test_custom_tokenizer_with_mismatched_tokenizer_class(self):
        nop_tokenizer_code = """
import transformers

class NopTokenizer(transformers.PreTrainedTokenizer):
    special_attribute_present = True

    def get_vocab(self):
        return {}
"""

        nop_config_code = """
from transformers import PreTrainedConfig

class NopConfig(PreTrainedConfig):
    model_type = "test_unregistered_dynamic"

    def __init__(self, **kwargs):
        super().__init__(**kwargs)
"""