Update configuration_gemmoe.py

configuration_gemmoe.py

@@ -118,7 +118,7 @@ class GemmoeConfig(PretrainedConfig):
         num_attention_heads=16,
         num_key_value_heads=16,
         head_dim=256,
-        hidden_act="gelu",
+        hidden_act="gelu_pytorch_tanh",
         max_position_embeddings=8192,
         initializer_range=0.02,
         rms_norm_eps=1e-6,
@@ -126,7 +126,6 @@ class GemmoeConfig(PretrainedConfig):
         pad_token_id=0,
         eos_token_id=1,
         bos_token_id=2,
-        hidden_act="gelu_pytorch_tanh",
         hidden_activation=None,
         tie_word_embeddings=True,
         rope_theta=10000.0,
@@ -148,7 +147,6 @@ class GemmoeConfig(PretrainedConfig):
         self.hidden_act = hidden_act
         self.hidden_activation = hidden_activation
         self.num_key_value_heads = num_key_value_heads
-        self.hidden_act = hidden_act
         self.initializer_range = initializer_range
         self.rms_norm_eps = rms_norm_eps
         self.use_cache = use_cache