Update configuration_minicpm.py

configuration_minicpm.py

@@ -58,8 +58,10 @@ class MiniCPMConfig(PretrainedConfig):
             by meanpooling all the original heads within that group. For more details checkout [this
             paper](https://arxiv.org/pdf/2305.13245.pdf). If it is not specified, will default to
             `num_attention_heads`.
-        hidden_act (`str` or `function`, *optional*, defaults to `"silu"`):
+        hidden_act (`str` or `function`, *optional*, defaults to `"relu"`):
             The non-linear activation function (function or string) in the decoder.
+        hidden_act_param (`float`, *optional*, defaults to 0.):
+            The bias for shiftrelu or threshold for fatrelu.
         max_position_embeddings (`int`, *optional*, defaults to 2048):
             The maximum sequence length that this model might ever be used with. MiniCPM 1 supports up to 2048 tokens,
             MiniCPM 2 up to 4096, CodeMiniCPM up to 16384.
@@ -122,7 +124,8 @@ class MiniCPMConfig(PretrainedConfig):
         num_hidden_layers=32,
         num_attention_heads=32,
         num_key_value_heads=None,
-        hidden_act="silu",
+        hidden_act="relu",
+        hidden_act_param=0.,
         max_position_embeddings=2048,
         initializer_range=0.02,
         rms_norm_eps=1e-6,
@@ -154,6 +157,7 @@ class MiniCPMConfig(PretrainedConfig):
 
         self.num_key_value_heads = num_key_value_heads
         self.hidden_act = hidden_act
+        self.hidden_act_param = hidden_act_param
         self.initializer_range = initializer_range
         self.rms_norm_eps = rms_norm_eps
         self.pretraining_tp = pretraining_tp