Repository: huggingface/transformers
Pull request: Fix tokenizer auto_map being ignored for custom models

Fixes #43202

PR #42894 introduced an early-exit to `TokenizersBackend` when `tokenizer_class` doesn't match the registered tokenizer for a `model_type`. However, this check was placed before the `auto_map` extraction, causing custom tokenizers (with `trust_remote_code=True`) to be ignored.

Reproduction:
```python
from transformers import AutoTokenizer
tokenizer = AutoTokenizer.from_pretrained("IQuestLab/IQuest-Coder-V1-40B-Loop-Instruct", trust_remote_code=True)
print(tokenizer.decode(tokenizer.encode("This is a test")))
# Expected: "This is a test"
# Actual (bug): "Th is <unk> is <unk> a <unk> te st"
```

For models with unregistered `model_type` (like `iquestloopcoder`), the condition `TOKENIZER_MAPPING_NAMES.get(model_type, "") != tokenizer_config_class` is always True, causing early-exit to `TokenizersBackend` without checking if `auto_map` exists.

This fix moves the `auto_map` extraction before the early-exit check and adds `tokenizer_auto_map is None` to the condition.

Added regression test `test_custom_tokenizer_with_mismatched_tokenizer_class`.

Changed files:
filename: src/transformers/models/auto/tokenization_auto.py
patch: @@ -625,9 +625,21 @@ def from_pretrained(
         # Next, let's try to use the tokenizer_config file to get the tokenizer class.
         tokenizer_config = get_tokenizer_config(pretrained_model_name_or_path, **kwargs)
         tokenizer_config_class = tokenizer_config.get("tokenizer_class", None)
-        # if there is a config, we can check that the tokenizer class != than model class and can thus assume we need to use `TokenizersBackend`
+
+        # Check for auto_map early to handle dynamic tokenizers properly
+        tokenizer_auto_map = None
+        if "auto_map" in tokenizer_config:
+            if isinstance(tokenizer_config["auto_map"], (tuple, list)):
+                # Legacy format for dynamic tokenizers
+                tokenizer_auto_map = tokenizer_config["auto_map"]
+            else:
+                tokenizer_auto_map = tokenizer_config["auto_map"].get("AutoTokenizer", None)
+
+        # if there is a config, we can check that the tokenizer class != than model class and can thus assume we need to use TokenizersBackend
+        # Skip this early exit if auto_map is present (custom tokenizer with trust_remote_code)
         if (
-            tokenizer_config_class is not None
+            tokenizer_auto_map is None
+            and tokenizer_config_class is not None
             and config_model_type is not None
             and config_model_type != ""
             and TOKENIZER_MAPPING_NAMES.get(config_model_type, "").replace("Fast", "")
@@ -644,15 +656,6 @@ def from_pretrained(
         if "_commit_hash" in tokenizer_config:
             kwargs["_commit_hash"] = tokenizer_config["_commit_hash"]
 
-        # Check for auto_map early to handle dynamic tokenizers properly
-        tokenizer_auto_map = None
-        if "auto_map" in tokenizer_config:
-            if isinstance(tokenizer_config["auto_map"], (tuple, list)):
-                # Legacy format for dynamic tokenizers
-                tokenizer_auto_map = tokenizer_config["auto_map"]
-            else:
-                tokenizer_auto_map = tokenizer_config["auto_map"].get("AutoTokenizer", None)
-
         if tokenizer_config_class:
             tokenizer_config_class = tokenizer_config_class.replace("Fast", "")
 

filename: tests/models/auto/test_tokenization_auto.py
patch: @@ -616,3 +616,70 @@ def test_tokenization_class_priority(self):
 
         tok = AutoTokenizer.from_pretrained("HuggingFaceTB/SmolLM2-135M-Instruct")
         self.assertTrue(tok.__class__ == TokenizersBackend)
+
+    def test_custom_tokenizer_with_mismatched_tokenizer_class(self):
+        nop_tokenizer_code = """
+import transformers
+
+class NopTokenizer(transformers.PreTrainedTokenizer):
+    special_attribute_present = True
+
+    def get_vocab(self):
+        return {}
+"""
+
+        nop_config_code = """
+from transformers import PreTrainedConfig
+
+class NopConfig(PreTrainedConfig):
+    model_type = "test_unregistered_dynamic"
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+"""
+
+        with tempfile.TemporaryDirectory() as tmp_dir:
+            fake_model_id = "hf-internal-testing/test_unregistered_dynamic"
+            fake_repo = os.path.join(tmp_dir, fake_model_id)
+            os.makedirs(fake_repo)
+
+            tokenizer_src_file = os.path.join(fake_repo, "tokenizer.py")
+            with open(tokenizer_src_file, "w") as wfp:
+                wfp.write(nop_tokenizer_code)
+
+            model_config_src_file = os.path.join(fake_repo, "config.py")
+            with open(model_config_src_file, "w") as wfp:
+                wfp.write(nop_config_code)
+
+            config = {
+                "model_type": "test_unregistered_dynamic",
+                "auto_map": {"AutoConfig": f"{fake_model_id}--config.NopConfig"},
+            }
+
+            config_file = os.path.join(fake_repo, "config.json")
+            with open(config_file, "w") as wfp:
+                json.dump(config, wfp, indent=2)
+
+            tokenizer_config = {
+                "tokenizer_class": "NopTokenizer",
+                "auto_map": {
+                    "AutoTokenizer": [
+                        f"{fake_model_id}--tokenizer.NopTokenizer",
+                        None,
+                    ]
+                },
+            }
+
+            tokenizer_config_file = os.path.join(fake_repo, "tokenizer_config.json")
+            with open(tokenizer_config_file, "w") as wfp:
+                json.dump(tokenizer_config, wfp, indent=2)
+
+            prev_dir = os.getcwd()
+            try:
+                os.chdir(tmp_dir)
+
+                tokenizer = AutoTokenizer.from_pretrained(fake_model_id, local_files_only=True, trust_remote_code=True)
+                self.assertEqual(tokenizer.__class__.__name__, "NopTokenizer")
+                self.assertTrue(tokenizer.special_attribute_present)
+            finally:
+                os.chdir(prev_dir)