Repository: huggingface/transformers
Pull request: Fix GlmMoeDsaConfig default mlp_layer_types in modular conversion

## Summary
This PR fixes #43864 by preserving the `GlmMoeDsaConfig` default `mlp_layer_types` from the modular source.

`GlmMoeDsaConfig` should default to dense MLP for the first 3 layers and sparse afterward. During modular conversion, the parent init body was being inlined and overwrote that default with the parent pattern (`["dense"] + ["sparse"] * ...`).

## Changes
- In `modular_glm_moe_dsa.py`, call `PreTrainedConfig.__init__(self, **kwargs)` instead of `super().__init__(**kwargs)` to avoid inlining parent init logic.
- Regenerated `configuration_glm_moe_dsa.py` via modular converter, which removes the duplicated parent default block.
- Added a regression test in `tests/models/glm_moe_dsa/test_configuration_glm_moe_dsa.py` to assert the expected default pattern for `num_hidden_layers=8`.

## Validation
- `PYTHONPATH=src python3 utils/modular_model_converter.py glm_moe_dsa`
- `PYTHONPATH=src python3 utils/check_modular_conversion.py --files src/transformers/models/glm_moe_dsa/modular_glm_moe_dsa.py`
- `PYTHONPATH=src python3 -m pytest tests/models/glm_moe_dsa/test_configuration_glm_moe_dsa.py -q`
- `PYTHONPATH=src python3 -m trace --count --summary --module unittest tests.models.glm_moe_dsa.test_configuration_glm_moe_dsa | grep -E "configuration_glm_moe_dsa|test_configuration_glm_moe_dsa"`
  - output: `configuration_glm_moe_dsa ... 100%`


Changed files:
filename: src/transformers/models/glm_moe_dsa/configuration_glm_moe_dsa.py
patch: @@ -181,80 +181,17 @@ def __init__(
         index_topk: int | None = 2048,
         **kwargs,
     ):
-        self.hidden_size = hidden_size
-        self.intermediate_size = intermediate_size
-        self.num_hidden_layers = num_hidden_layers
-        self.moe_intermediate_size = moe_intermediate_size
-        self.num_attention_heads = num_attention_heads
-        self.n_shared_experts = n_shared_experts
-        self.n_routed_experts = n_routed_experts
-        self.routed_scaling_factor = routed_scaling_factor
-        self.kv_lora_rank = kv_lora_rank
-        self.q_lora_rank = q_lora_rank
-        self.qk_rope_head_dim = qk_rope_head_dim
-        self.v_head_dim = v_head_dim
-        self.qk_nope_head_dim = qk_nope_head_dim
-        self.qk_head_dim = qk_nope_head_dim + qk_rope_head_dim
-        self.head_dim = qk_rope_head_dim
-        self.num_experts_per_tok = num_experts_per_tok
-        self.num_key_value_heads = num_key_value_heads
-        self.initializer_range = initializer_range
-        self.index_topk = index_topk
-        self.mlp_layer_types = mlp_layer_types
         self.vocab_size = vocab_size
         self.max_position_embeddings = max_position_embeddings
         self.hidden_size = hidden_size
         self.intermediate_size = intermediate_size
         self.num_hidden_layers = num_hidden_layers
 
         # Default to MoE from the fourth layer and on
+        if mlp_layer_types is None:
+            mlp_layer_types = ["dense"] * min(3, self.num_hidden_layers) + ["sparse"] * (self.num_hidden_layers - 3)
+        layer_type_validation(mlp_layer_types, self.num_hidden_layers, attention=False)
         self.mlp_layer_types = mlp_layer_types
-        if self.mlp_layer_types is None:
-            self.mlp_layer_types = ["dense"] * min(3, self.num_hidden_layers) + ["sparse"] * (
-                self.num_hidden_layers - 3
-            )
-        layer_type_validation(self.mlp_layer_types, self.num_hidden_layers, attention=False)
-
-        self.moe_intermediate_size = moe_intermediate_size
-        self.num_attention_heads = num_attention_heads
-        self.n_shared_experts = n_shared_experts
-        self.n_routed_experts = n_routed_experts
-        self.routed_scaling_factor = routed_scaling_factor
-        self.kv_lora_rank = kv_lora_rank
-        self.q_lora_rank = q_lora_rank
-        self.qk_rope_head_dim = qk_rope_head_dim
-        self.v_head_dim = v_head_dim
-        self.qk_nope_head_dim = qk_nope_head_dim
-        self.qk_head_dim = qk_nope_head_dim + qk_rope_head_dim
-        self.head_dim = qk_rope_head_dim
-        self.n_group = n_group
-        self.topk_group = topk_group
-        self.num_experts_per_tok = num_experts_per_tok
-        self.norm_topk_prob = norm_topk_prob
-        self.rope_interleave = rope_interleave
-        self.num_key_value_heads = num_key_value_heads
-        self.hidden_act = hidden_act
-        self.initializer_range = initializer_range
-        self.rms_norm_eps = rms_norm_eps
-        self.use_cache = use_cache
-        self.attention_bias = attention_bias
-        self.attention_dropout = attention_dropout
-        self.rope_parameters = rope_parameters
-        self.pad_token_id = pad_token_id
-        self.bos_token_id = bos_token_id
-        self.eos_token_id = eos_token_id
-        self.tie_word_embeddings = tie_word_embeddings
-        self.vocab_size = vocab_size
-        self.max_position_embeddings = max_position_embeddings
-        self.hidden_size = hidden_size
-        self.intermediate_size = intermediate_size
-        self.num_hidden_layers = num_hidden_layers
-
-        # Default to MoE from the second layer and on
-        self.mlp_layer_types = mlp_layer_types
-        if self.mlp_layer_types is None:
-            self.mlp_layer_types = ["dense"] + ["sparse"] * (self.num_hidden_layers - 1)
-        layer_type_validation(self.mlp_layer_types, self.num_hidden_layers, attention=False)
 
         self.moe_intermediate_size = moe_intermediate_size
         self.num_attention_heads = num_attention_heads
@@ -276,6 +213,7 @@ def __init__(
         self.num_key_value_heads = num_key_value_heads
         self.hidden_act = hidden_act
         self.initializer_range = initializer_range
+        self.index_topk = index_topk
         self.rms_norm_eps = rms_norm_eps
         self.use_cache = use_cache
         self.attention_bias = attention_bias

filename: src/transformers/models/glm_moe_dsa/modular_glm_moe_dsa.py
patch: @@ -20,7 +20,7 @@
 import torch.nn.functional as F
 
 from ...cache_utils import Cache
-from ...configuration_utils import layer_type_validation
+from ...configuration_utils import PreTrainedConfig, layer_type_validation
 from ...modeling_flash_attention_utils import FlashAttentionKwargs
 from ...modeling_rope_utils import RopeParameters
 from ...modeling_utils import ALL_ATTENTION_FUNCTIONS
@@ -183,39 +183,17 @@ def __init__(
         index_topk: int | None = 2048,
         **kwargs,
     ):
-        self.hidden_size = hidden_size
-        self.intermediate_size = intermediate_size
-        self.num_hidden_layers = num_hidden_layers
-        self.moe_intermediate_size = moe_intermediate_size
-        self.num_attention_heads = num_attention_heads
-        self.n_shared_experts = n_shared_experts
-        self.n_routed_experts = n_routed_experts
-        self.routed_scaling_factor = routed_scaling_factor
-        self.kv_lora_rank = kv_lora_rank
-        self.q_lora_rank = q_lora_rank
-        self.qk_rope_head_dim = qk_rope_head_dim
-        self.v_head_dim = v_head_dim
-        self.qk_nope_head_dim = qk_nope_head_dim
-        self.qk_head_dim = qk_nope_head_dim + qk_rope_head_dim
-        self.head_dim = qk_rope_head_dim
-        self.num_experts_per_tok = num_experts_per_tok
-        self.num_key_value_heads = num_key_value_heads
-        self.initializer_range = initializer_range
-        self.index_topk = index_topk
-        self.mlp_layer_types = mlp_layer_types
         self.vocab_size = vocab_size
         self.max_position_embeddings = max_position_embeddings
         self.hidden_size = hidden_size
         self.intermediate_size = intermediate_size
         self.num_hidden_layers = num_hidden_layers
 
         # Default to MoE from the fourth layer and on
+        if mlp_layer_types is None:
+            mlp_layer_types = ["dense"] * min(3, self.num_hidden_layers) + ["sparse"] * (self.num_hidden_layers - 3)
+        layer_type_validation(mlp_layer_types, self.num_hidden_layers, attention=False)
         self.mlp_layer_types = mlp_layer_types
-        if self.mlp_layer_types is None:
-            self.mlp_layer_types = ["dense"] * min(3, self.num_hidden_layers) + ["sparse"] * (
-                self.num_hidden_layers - 3
-            )
-        layer_type_validation(self.mlp_layer_types, self.num_hidden_layers, attention=False)
 
         self.moe_intermediate_size = moe_intermediate_size
         self.num_attention_heads = num_attention_heads
@@ -237,6 +215,7 @@ def __init__(
         self.num_key_value_heads = num_key_value_heads
         self.hidden_act = hidden_act
         self.initializer_range = initializer_range
+        self.index_topk = index_topk
         self.rms_norm_eps = rms_norm_eps
         self.use_cache = use_cache
         self.attention_bias = attention_bias
@@ -247,8 +226,7 @@ def __init__(
         self.eos_token_id = eos_token_id
         self.tie_word_embeddings = tie_word_embeddings
 
-        super().__init__(**kwargs)
-        del self.pretraining_tp
+        PreTrainedConfig.__init__(self, **kwargs)
 
 
 class GlmMoeDsaRMSNorm(Glm4MoeRMSNorm):

filename: tests/models/glm_moe_dsa/test_modeling_glm_moe_dsa.py
patch: @@ -15,7 +15,7 @@
 
 import unittest
 
-from transformers import Cache, is_torch_available
+from transformers import Cache, GlmMoeDsaConfig, is_torch_available
 from transformers.testing_utils import (
     require_torch,
     require_torch_accelerator,
@@ -79,6 +79,12 @@ def _check_past_key_values_for_generate(self, batch_size, past_key_values, seq_l
             self.assertEqual(layer.keys.shape, expected_key_shape)
             self.assertEqual(layer.values.shape, expected_value_shape)
 
+    def test_default_mlp_layer_types(self):
+        config = GlmMoeDsaConfig(num_hidden_layers=8)
+        self.assertEqual(
+            config.mlp_layer_types, ["dense", "dense", "dense", "sparse", "sparse", "sparse", "sparse", "sparse"]
+        )
+
 
 @require_torch_accelerator
 @slow