Repository: huggingface/transformers
Pull request: Update from pretrained error when loading

# What does this PR do?
Simplifies the error messaged, by grouping layer numbers: 

```python
'model.language_model.0.self_attn.0.mlp0', 
'model.language_model.0.self_attn.1.mlp.conv1.weight', 
'model.language_model.1.self_attn.0.mlp.conv1.weight', 
'model.language_model.1.self_attn.1.mlp.conv1.weight',
'model.language_model.2.self_attn.0.mlp.conv1.weight', 
'model.language_model.2.self_attn.1.conv1.weight',
'model.language_model.3.self_attn.0.mlp.conv1.weight',
'model.language_model.3.self_attn.1.mlp.conv1.weight', 
'model.language_model.0.self_attn.0.mlp.layer.conv1.weight', 
'model.language_model.1.self_attn.0.mlp.layer.conv1.weight', 
'model.language_model.2.self_attn.0.mlp.layer.conv1.weight', 
'model.language_model.3.self_attn.0.mlp.layer.conv1.weight',
'model.language_model.0.self_attn.0.mlp.layer.1.weight',
'model.language_model.1.self_attn.0.mlp.layer.1.weight', 
'model.language_model.2.self_attn.0.mlp.layer.1.weight', 
'model.language_model.3.self_attn.0.mlp.layer.1.weight'
'model.language_model.3.self_attn.0.mlp.layer.conv2.weight'
```
to 
```python
'model.language_model.{0, 1, 2, 3}.self_attn.{0, 1}.mlp.layer.1.weight', 
'model.language_model.{0, 1, 2, 3}.self_attn.{0, 1}.mlp.conv1.weight', 
'model.language_model.{0, 1, 2, 3}.self_attn.{0, 1}.mlp0', 
'model.language_model.{0, 1, 2, 3}.self_attn.{0, 1}.conv1.weight', 
'model.language_model.{0, 1, 2, 3}.self_attn.{0, 1}.mlp.layer.conv1.weight'
'model.language_model.{0, 1, 2, 3}.self_attn.{0, 1}.mlp.layer.conv2weight'
```

On a recent example, went from 3000 no match to 56.
This does not apply to `layer.my_name_with_digit_0`   and single element are left alone


Changed files:
filename: src/transformers/modeling_utils.py
patch: @@ -875,6 +875,40 @@ def _add_variant(weights_name: str, variant: Optional[str] = None) -> str:
     return weights_name
 
 
+def update_key_name(keys):
+    """
+    Updates a dictionary of keys to pack layers together as layer.{0, 1, 4} instead of layers.0, layers.1, layers.4.
+    """
+    key_dict = defaultdict(list)
+    for key in keys:
+        all_digits = re.findall(r".(\d+).", key)
+        for i, k in enumerate(all_digits):
+            if len(key_dict[re.sub(r".(\d+).", ".*.", key)]) <= i:
+                key_dict[re.sub(r".(\d+).", ".*.", key)].append(set())
+            key_dict[re.sub(r".(\d+).", ".*.", key)][i].add(int(k))
+
+    final_keys = set()
+    for key in keys:
+        text = re.sub(r".(\d+).", ".*.", key)
+        pattern = key_dict[text]
+        final_text = ""
+        for i, part in enumerate(text.split("*")):
+            if len(pattern) <= i:
+                final_text += part
+            else:
+                data = [str(i) for i in sorted(pattern[i])]
+                if len(data) > 10:
+                    result = f"{data[0]}...{data[-1]}"
+                else:
+                    result = ", ".join(data)  # If there are only 1 or 2 elements, show them all
+                if len(data) > 1:
+                    final_text += part + "{" + result + "}"
+                else:
+                    final_text += part + data[0]
+        final_keys.add(final_text)
+    return sorted(final_keys)
+
+
 def _get_resolved_checkpoint_files(
     pretrained_model_name_or_path: Optional[Union[str, os.PathLike]],
     subfolder: str,
@@ -5313,7 +5347,7 @@ def _load_pretrained_model(
             warner = logger.warning if model.__class__.__name__ in archs else logger.info
             warner(
                 f"Some weights of the model checkpoint at {pretrained_model_name_or_path} were not used when"
-                f" initializing {model.__class__.__name__}: {unexpected_keys}\n- This IS expected if you are"
+                f" initializing {model.__class__.__name__}: {update_key_name(unexpected_keys)}\n- This IS expected if you are"
                 f" initializing {model.__class__.__name__} from the checkpoint of a model trained on another task or"
                 " with another architecture (e.g. initializing a BertForSequenceClassification model from a"
                 " BertForPreTraining model).\n- This IS NOT expected if you are initializing"
@@ -5323,7 +5357,7 @@ def _load_pretrained_model(
         if len(missing_keys) > 0:
             logger.warning(
                 f"Some weights of {model.__class__.__name__} were not initialized from the model checkpoint at"
-                f" {pretrained_model_name_or_path} and are newly initialized: {missing_keys}\nYou should probably"
+                f" {pretrained_model_name_or_path} and are newly initialized: {update_key_name(missing_keys)}\nYou should probably"
                 " TRAIN this model on a down-stream task to be able to use it for predictions and inference."
             )
         if len(mismatched_keys) > 0:

filename: src/transformers/models/deepseek_v3/configuration_deepseek_v3.py
patch: @@ -152,7 +152,7 @@ class DeepseekV3Config(PreTrainedConfig):
         "norm": (["hidden_states"], ["hidden_states"]),
     }
     attribute_map = {
-        "num_experts": "n_routed_experts",
+        "num_local_experts": "n_routed_experts",
     }
 
     def __init__(

filename: tests/utils/test_modeling_utils.py
patch: @@ -58,6 +58,7 @@
     logging,
 )
 from transformers.modeling_flash_attention_utils import is_flash_attn_available
+from transformers.modeling_utils import update_key_name
 from transformers.models.mistral.modeling_mistral import MistralModel
 from transformers.testing_utils import (
     TOKEN,
@@ -1689,6 +1690,22 @@ def test_isin_mps_friendly(self):
             torch.equal(torch.isin(random_ids, random_test_tensor), isin_mps_friendly(random_ids, random_test_tensor))
         )
 
+    def test_update_key_name(self):
+        model = AutoModel.from_pretrained("google-t5/t5-base", device_map="auto")
+
+        new_keys = "\n".join(sorted(update_key_name(model.state_dict().keys())))
+
+        EXPECTED_KEYS = """decoder.block.0.layer.0.SelfAttention.relative_attention_bias.weight\ndecoder.block.{0...11}.layer.0.SelfAttention.k.weight\ndecoder.block.{0...11}.layer.0.SelfAttention.o.weight\ndecoder.block.{0...11}.layer.0.SelfAttention.q.weight\ndecoder.block.{0...11}.layer.0.SelfAttention.v.weight\ndecoder.block.{0...11}.layer.1.EncDecAttention.k.weight\ndecoder.block.{0...11}.layer.1.EncDecAttention.o.weight\ndecoder.block.{0...11}.layer.1.EncDecAttention.q.weight\ndecoder.block.{0...11}.layer.1.EncDecAttention.v.weight\ndecoder.block.{0...11}.layer.2.DenseReluDense.wi.weight\ndecoder.block.{0...11}.layer.2.DenseReluDense.wo.weight\ndecoder.block.{0...11}.layer.{0, 1, 2}.layer_norm.weight\ndecoder.embed_tokens.weight\ndecoder.final_layer_norm.weight\nencoder.block.0.layer.0.SelfAttention.relative_attention_bias.weight\nencoder.block.{0...11}.layer.0.SelfAttention.k.weight\nencoder.block.{0...11}.layer.0.SelfAttention.o.weight\nencoder.block.{0...11}.layer.0.SelfAttention.q.weight\nencoder.block.{0...11}.layer.0.SelfAttention.v.weight\nencoder.block.{0...11}.layer.1.DenseReluDense.wi.weight\nencoder.block.{0...11}.layer.1.DenseReluDense.wo.weight\nencoder.block.{0...11}.layer.{0, 1}.layer_norm.weight\nencoder.embed_tokens.weight\nencoder.final_layer_norm.weight\nshared.weight"""
+        self.assertEqual(new_keys, EXPECTED_KEYS)
+
+        EXPECTED_KEYS = """embed_tokens.weight\nlayers.{0, 1, 2}.mlp.down_proj.weight\nlayers.{0, 1, 2}.mlp.gate_proj.weight\nlayers.{0, 1, 2}.mlp.up_proj.weight\nlayers.{0...60}.input_layernorm.weight\nlayers.{0...60}.post_attention_layernorm.weight\nlayers.{0...60}.self_attn.kv_a_layernorm.weight\nlayers.{0...60}.self_attn.kv_a_proj_with_mqa.weight\nlayers.{0...60}.self_attn.kv_b_proj.weight\nlayers.{0...60}.self_attn.o_proj.weight\nlayers.{0...60}.self_attn.q_a_layernorm.weight\nlayers.{0...60}.self_attn.q_a_proj.weight\nlayers.{0...60}.self_attn.q_b_proj.weight\nlayers.{3...60}.mlp.experts.{0...255}.down_proj.weight\nlayers.{3...60}.mlp.experts.{0...255}.gate_proj.weight\nlayers.{3...60}.mlp.experts.{0...255}.up_proj.weight\nlayers.{3...60}.mlp.gate.e_score_correction_bias\nlayers.{3...60}.mlp.gate.weight\nlayers.{3...60}.mlp.shared_experts.down_proj.weight\nlayers.{3...60}.mlp.shared_experts.gate_proj.weight\nlayers.{3...60}.mlp.shared_experts.up_proj.weight\nnorm.weight"""
+        config = AutoConfig.from_pretrained("deepseek-ai/DeepSeek-V3.1")
+        with torch.device("meta"):
+            model = AutoModel.from_config(config)
+
+        new_keys = "\n".join(sorted(update_key_name(model.state_dict().keys())))
+        self.assertEqual(new_keys, EXPECTED_KEYS)
+
     def test_can_generate(self):
         """Tests the behavior of `PreTrainedModel.can_generate` method."""
         logger = logging.get_logger("transformers.modeling_utils")