paulilioaica
/

MixtureOfPhi3

Text Generation

Mixture of Experts

microsoft/Phi-3-mini-128k-instruct

text-generation-inference

Inference Endpoints

Model card Files Files and versions Community

paulilioaica commited on Apr 26

Commit

0ca1903

•

1 Parent(s): 04265cf

Update modeling_phi3.py

Files changed (1) hide show

modeling_phi3.py +2 -2

modeling_phi3.py CHANGED Viewed

@@ -262,8 +262,8 @@ class Phi3MLP(nn.Module):
         self.config = config
         self.gate = nn.Linear(config.hidden_size, self.config.num_experts, bias=False)
-        self.gate_up_proj = nn.ModuleList([nn.Linear(config.hidden_size, 2 * config.intermediate_size, bias=False) for i in range(2)])
-        self.down_proj = nn.ModuleList([nn.Linear(config.intermediate_size, config.hidden_size, bias=False) for i in range(2)])
         self.activation_fn = ACT2FN[config.hidden_act]
     def forward(self, hidden_states: torch.FloatTensor) -> torch.FloatTensor:

         self.config = config
         self.gate = nn.Linear(config.hidden_size, self.config.num_experts, bias=False)
+        self.gate_up_proj = nn.ModuleList([nn.Linear(config.hidden_size, 2 * config.intermediate_size, bias=False) for i in range(self.config.num_experts)])
+        self.down_proj = nn.ModuleList([nn.Linear(config.intermediate_size, config.hidden_size, bias=False) for i in range(self.config.num_experts)])
         self.activation_fn = ACT2FN[config.hidden_act]
     def forward(self, hidden_states: torch.FloatTensor) -> torch.FloatTensor: