Repository: huggingface/transformers
Pull request: Fix a typo in MoE models

# What does this PR do?

As per title, reported by glm-authors. Can be tested with `tests/models/deepseek_v3/test_modeling_deepseek_v3.py::DeepseekV3IntegrationTest::test_compile_static_cache`, not failing with mismatch shape anymore

Changed files:
filename: src/transformers/models/deepseek_v3/modeling_deepseek_v3.py
patch: @@ -157,7 +157,7 @@ def __init__(self, config):
         super().__init__()
         self.num_experts = config.num_local_experts
         self.hidden_dim = config.hidden_size
-        self.intermediate_dim = config.intermediate_size
+        self.intermediate_dim = config.moe_intermediate_size
         self.gate_up_proj = nn.Parameter(torch.empty(self.num_experts, 2 * self.intermediate_dim, self.hidden_dim))
         self.down_proj = nn.Parameter(torch.empty(self.num_experts, self.hidden_dim, self.intermediate_dim))
         self.act_fn = ACT2FN[config.hidden_act]

filename: src/transformers/models/deepseek_v3/modular_deepseek_v3.py
patch: @@ -107,6 +107,7 @@ class DeepseekV3NaiveMoe(MixtralExperts):
     def __init__(self, config):
         super().__init__(config)
         self.num_experts = config.num_local_experts
+        self.intermediate_dim = config.moe_intermediate_size
 
 
 class DeepseekV3MoE(nn.Module):

filename: src/transformers/models/dots1/modeling_dots1.py
patch: @@ -315,7 +315,7 @@ def __init__(self, config):
         super().__init__()
         self.num_experts = config.num_local_experts
         self.hidden_dim = config.hidden_size
-        self.intermediate_dim = config.intermediate_size
+        self.intermediate_dim = config.moe_intermediate_size
         self.gate_up_proj = nn.Parameter(torch.empty(self.num_experts, 2 * self.intermediate_dim, self.hidden_dim))
         self.down_proj = nn.Parameter(torch.empty(self.num_experts, self.hidden_dim, self.intermediate_dim))
         self.act_fn = ACT2FN[config.hidden_act]

filename: src/transformers/models/glm4_moe/modeling_glm4_moe.py
patch: @@ -339,7 +339,7 @@ def __init__(self, config):
         super().__init__()
         self.num_experts = config.num_local_experts
         self.hidden_dim = config.hidden_size
-        self.intermediate_dim = config.intermediate_size
+        self.intermediate_dim = config.moe_intermediate_size
         self.gate_up_proj = nn.Parameter(torch.empty(self.num_experts, 2 * self.intermediate_dim, self.hidden_dim))
         self.down_proj = nn.Parameter(torch.empty(self.num_experts, self.hidden_dim, self.intermediate_dim))
         self.act_fn = ACT2FN[config.hidden_act]

filename: src/transformers/models/glm4v_moe/modeling_glm4v_moe.py
patch: @@ -402,7 +402,7 @@ def __init__(self, config):
         super().__init__()
         self.num_experts = config.num_local_experts
         self.hidden_dim = config.hidden_size
-        self.intermediate_dim = config.intermediate_size
+        self.intermediate_dim = config.moe_intermediate_size
         self.gate_up_proj = nn.Parameter(torch.empty(self.num_experts, 2 * self.intermediate_dim, self.hidden_dim))
         self.down_proj = nn.Parameter(torch.empty(self.num_experts, self.hidden_dim, self.intermediate_dim))
         self.act_fn = ACT2FN[config.hidden_act]