x54-729 commited on
Commit
2815e26
1 Parent(s): 0ed6e4a

Support dynamic ntk rope

Browse files
Files changed (1) hide show
  1. configuration_internlm.py +8 -7
configuration_internlm.py CHANGED
@@ -19,9 +19,8 @@
19
  # limitations under the License.
20
  """ InternLM model configuration"""
21
 
22
- from transformers.utils import logging
23
  from transformers.configuration_utils import PretrainedConfig
24
-
25
 
26
  logger = logging.get_logger(__name__)
27
 
@@ -30,9 +29,9 @@ INTERNLM_PRETRAINED_CONFIG_ARCHIVE_MAP = {}
30
 
31
  class InternLMConfig(PretrainedConfig):
32
  r"""
33
- This is the configuration class to store the configuration of a [`InternLMModel`]. It is used to instantiate an InternLM
34
- model according to the specified arguments, defining the model architecture. Instantiating a configuration with the
35
- defaults will yield a similar configuration to that of the InternLM-7B.
36
 
37
  Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
38
  documentation from [`PretrainedConfig`] for more information.
@@ -81,7 +80,7 @@ class InternLMConfig(PretrainedConfig):
81
  model_type = "internlm"
82
  _auto_class = "AutoConfig"
83
 
84
- def __init__(
85
  self,
86
  vocab_size=103168,
87
  hidden_size=4096,
@@ -98,6 +97,7 @@ class InternLMConfig(PretrainedConfig):
98
  eos_token_id=2,
99
  tie_word_embeddings=False,
100
  bias=True,
 
101
  **kwargs,
102
  ):
103
  self.vocab_size = vocab_size
@@ -111,10 +111,11 @@ class InternLMConfig(PretrainedConfig):
111
  self.rms_norm_eps = rms_norm_eps
112
  self.use_cache = use_cache
113
  self.bias = bias
 
114
  super().__init__(
115
  pad_token_id=pad_token_id,
116
  bos_token_id=bos_token_id,
117
  eos_token_id=eos_token_id,
118
  tie_word_embeddings=tie_word_embeddings,
119
  **kwargs,
120
- )
 
19
  # limitations under the License.
20
  """ InternLM model configuration"""
21
 
 
22
  from transformers.configuration_utils import PretrainedConfig
23
+ from transformers.utils import logging
24
 
25
  logger = logging.get_logger(__name__)
26
 
 
29
 
30
  class InternLMConfig(PretrainedConfig):
31
  r"""
32
+ This is the configuration class to store the configuration of a [`InternLMModel`]. It is used to instantiate
33
+ an InternLM model according to the specified arguments, defining the model architecture. Instantiating a
34
+ configuration with the defaults will yield a similar configuration to that of the InternLM-7B.
35
 
36
  Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
37
  documentation from [`PretrainedConfig`] for more information.
 
80
  model_type = "internlm"
81
  _auto_class = "AutoConfig"
82
 
83
+ def __init__( # pylint: disable=W0102
84
  self,
85
  vocab_size=103168,
86
  hidden_size=4096,
 
97
  eos_token_id=2,
98
  tie_word_embeddings=False,
99
  bias=True,
100
+ rotary={"base": 10000, "type": "dynamic"}, # pylint: disable=W0102
101
  **kwargs,
102
  ):
103
  self.vocab_size = vocab_size
 
111
  self.rms_norm_eps = rms_norm_eps
112
  self.use_cache = use_cache
113
  self.bias = bias
114
+ self.rotary = rotary
115
  super().__init__(
116
  pad_token_id=pad_token_id,
117
  bos_token_id=bos_token_id,
118
  eos_token_id=eos_token_id,
119
  tie_word_embeddings=tie_word_embeddings,
120
  **kwargs,
121
+ )