|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
""" StableLM β model configuration""" |
|
|
|
from transformers import PretrainedConfig |
|
from transformers.utils import logging |
|
|
|
|
|
logger = logging.get_logger(__name__) |
|
|
|
STABLE_LM_PRETRAINED_CONFIG_ARCHIVE_MAP = {} |
|
|
|
|
|
class StableLMAlphaConfig(PretrainedConfig): |
|
r""" |
|
Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the |
|
documentation from [`PretrainedConfig`] for more information. |
|
|
|
Args: |
|
vocab_size (`int`, *optional*, defaults to 50432): |
|
Vocabulary size of the StableLM model. Defines the number of different tokens that |
|
can be represented by the `inputs_ids` passed when calling [`StableLMAlphaModel`]. |
|
hidden_size (`int`, *optional*, defaults to 6144): |
|
Dimension of the decoder layers and the pooler layer. |
|
num_hidden_layers (`int`, *optional*, defaults to 44): |
|
Number of hidden layers in the Transformer decoder. |
|
num_heads (`int`, *optional*, defaults to 64): |
|
Number of attention heads for each attention layer in the Transformer decoder. |
|
hidden_act (`str` or `function`, *optional*, defaults to `"silu"`): |
|
The non-linear activation function (function or string). |
|
rotary_pct (`float`, *optional*, defaults to 0.25): |
|
Percentage of hidden dimensions to allocate to rotary embeddings. |
|
rotary_emb_base (`int`, *optional*, defaults to 10000) |
|
Base for computing rotary embeddings frequency. |
|
max_position_embeddings (`int`, *optional*, defaults to 2048): |
|
The maximum sequence length that this model might ever be used with. |
|
Typically set this to something large just in case (e.g., 512 or 1024 or 2048). |
|
initializer_range (`float`, *optional*, defaults to 1e-5): |
|
The standard deviation of the truncated_normal_initializer for initializing |
|
all weight matrices. |
|
norm_eps (`float`, *optional*, defaults to 1e-5): |
|
The epsilon used by the normalization layers. |
|
use_cache (`bool`, *optional*, defaults to `True`): |
|
Whether or not the model should return the last key/values attentions |
|
(not used by all models). Only relevant if `config.is_decoder=True`. |
|
tie_word_embeddings(`bool`, *optional*, defaults to `False`): |
|
Whether to tie weight embeddings |
|
|
|
Example: |
|
|
|
```python |
|
>>> from transformers import StableLMAlphaConfig, StableLMAlphaModel |
|
|
|
>>> # Initializing a StableLMAlphaConfig style configuration |
|
>>> configuration = StableLMAlphaConfig() |
|
|
|
>>> # Initializing a model (with random weights) from the style configuration |
|
>>> model = StableLMAlphaModel(configuration) # doctest: +SKIP |
|
|
|
>>> # Accessing the model configuration |
|
>>> configuration = model.config # doctest: +SKIP |
|
```""" |
|
model_type = "stablelm_alpha" |
|
keys_to_ignore_at_inference = ["past_key_values"] |
|
|
|
def __init__( |
|
self, |
|
vocab_size=50_432, |
|
hidden_size=2_560, |
|
num_hidden_layers=32, |
|
num_heads=32, |
|
hidden_act="silu", |
|
rotary_pct=0.25, |
|
rotary_emb_base=10_000, |
|
max_position_embeddings=2_048, |
|
initializer_range=0.02, |
|
norm_eps=1e-5, |
|
use_cache=True, |
|
bos_token_id=0, |
|
eos_token_id=2, |
|
tie_word_embeddings=False, |
|
**kwargs, |
|
): |
|
self.vocab_size = vocab_size |
|
self.max_position_embeddings = max_position_embeddings |
|
self.hidden_size = hidden_size |
|
self.num_hidden_layers = num_hidden_layers |
|
self.num_heads = num_heads |
|
self.hidden_act = hidden_act |
|
self.rotary_pct = rotary_pct |
|
self.rotary_emb_base = rotary_emb_base |
|
self.initializer_range = initializer_range |
|
self.norm_eps = norm_eps |
|
self.use_cache = use_cache |
|
self.tie_word_embeddings = tie_word_embeddings |
|
super().__init__( |
|
bos_token_id=bos_token_id, |
|
eos_token_id=eos_token_id, |
|
tie_word_embeddings=tie_word_embeddings, |
|
**kwargs, |
|
) |
|
|
