head divisor fix

configuration_rwkv5.py

@@ -21,44 +21,46 @@ from transformers.utils import logging
 
 logger = logging.get_logger(__name__)
 
-RWKV5_PRETRAINED_CONFIG_ARCHIVE_MAP = {}
+RWKV_PRETRAINED_CONFIG_ARCHIVE_MAP = {
+    
+}
 
 
 class Rwkv5Config(PretrainedConfig):
     """
-    This is the configuration class to store the configuration of a [`Rwkv5Model`]. It is used to instantiate a RWKV5
+    This is the configuration class to store the configuration of a [`RwkvModel`]. It is used to instantiate a RWKV
     model according to the specified arguments, defining the model architecture. Instantiating a configuration with the
     defaults will yield a similar configuration to that of the RWVK-4
-    [RWKV/rwkv-5-world-1b5](https://huggingface.co/RWKV/rwkv-5-world-1b5) architecture.
+    [RWKV/rwkv-4-169m-pile](https://huggingface.co/RWKV/rwkv-4-169m-pile) architecture.
 
     Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
     documentation from [`PretrainedConfig`] for more information.
 
 
     Args:
-        vocab_size (`int`, *optional*, defaults to 65536):
-            Vocabulary size of the RWKV5 model. Defines the number of different tokens that can be represented by the
-            `inputs_ids` passed when calling [`Rwkv5Model`].
-        hidden_size (`int`, *optional*, defaults to 768):
+        vocab_size (`int`, *optional*, defaults to 50277):
+            Vocabulary size of the RWKV model. Defines the number of different tokens that can be represented by the
+            `inputs_ids` passed when calling [`RwkvModel`].
+        context_length (`int`, *optional*, defaults to 1024):
+            The maximum sequence length that this model can be be used with in a single forward (using it in RNN mode
+            lets use any sequence length).
+        hidden_size (`int`, *optional*, defaults to 4096):
             Dimensionality of the embeddings and hidden states.
-        num_hidden_layers (`int`, *optional*, defaults to 24):
+        num_hidden_layers (`int`, *optional*, defaults to 32):
             Number of hidden layers in the model.
         attention_hidden_size (`int`, *optional*):
             Dimensionality of the attention hidden states. Will default to `hidden_size` if unset.
-        num_attention_heads (`int`, *optional*, defaults to 64):
-            The attention heads to use in rwkv5 self_attention module.
-        head_size (`int`, *optional*, defaults to 64): head_size of rwkv5 self_attention module.
         intermediate_size (`int`, *optional*):
             Dimensionality of the inner feed-forward layers. Will default to 4 times `hidden_size` if unset.
-        layer_norm_epsilon (`float`, *optional*, defaults to 1e-05):
+        layer_norm_eps (`float`, *optional*, defaults to 1e-5):
             The epsilon to use in the layer normalization layers.
         bos_token_id (`int`, *optional*, defaults to 0):
-            The id of the beginning of sentence token in the vocabulary. Defaults to 0 as RWKV5 uses the same tokenizer
+            The id of the beginning of sentence token in the vocabulary. Defaults to 0 as RWKV uses the same tokenizer
             as GPTNeoX.
         eos_token_id (`int`, *optional*, defaults to 0):
-            The id of the end of sentence token in the vocabulary. Defaults to 0 as RWKV5 uses the same tokenizer as
+            The id of the end of sentence token in the vocabulary. Defaults to 0 as RWKV uses the same tokenizer as
             GPTNeoX.
-        rescale_every (`int`, *optional*, defaults to 6):
+        rescale_every (`int`, *optional*, default to 6):
             At inference, the hidden states (and weights of the correponding output layers) are divided by 2 every
             `rescale_every` layer. If set to 0 or a negative number, no rescale is done.
         tie_word_embeddings (`bool`, *optional*, defaults to `False`):
@@ -70,28 +72,30 @@ class Rwkv5Config(PretrainedConfig):
     Example:
 
     ```python
-    >>> from transformers import Rwkv5Config, Rwkv5Model
+    >>> from transformers import RwkvConfig, RwkvModel
 
-    >>> # Initializing a Rwkv5 configuration
-    >>> configuration = Rwkv5Config()
+    >>> # Initializing a Rwkv configuration
+    >>> configuration = RwkvConfig()
 
     >>> # Initializing a model (with random weights) from the configuration
-    >>> model = Rwkv5Model(configuration)
+    >>> model = RwkvModel(configuration)
 
     >>> # Accessing the model configuration
     >>> configuration = model.config
     ```"""
 
     model_type = "rwkv5"
+    attribute_map = {"max_position_embeddings": "context_length"}
 
-    def __init__(
+    def __init__( #1.5B World
         self,
         vocab_size=65536,
+        context_length=4096,
         hidden_size=768,
         num_hidden_layers=24,
         attention_hidden_size=None,
-        num_attention_heads=64,
         head_size=64,
+        head_size_divisor=8,
         intermediate_size=None,
         layer_norm_epsilon=1e-5,
         bos_token_id=0,
@@ -99,14 +103,16 @@ class Rwkv5Config(PretrainedConfig):
         rescale_every=6,
         tie_word_embeddings=False,
         use_cache=True,
+        model_version="5_2",
         **kwargs,
     ):
         self.vocab_size = vocab_size
+        self.context_length = context_length
         self.hidden_size = hidden_size
         self.num_hidden_layers = num_hidden_layers
         self.attention_hidden_size = attention_hidden_size if attention_hidden_size is not None else hidden_size
-        self.num_attention_heads = num_attention_heads
         self.head_size = head_size
+        self.head_size_divisor = head_size_divisor
         self.intermediate_size = None
         self.layer_norm_epsilon = layer_norm_epsilon
         self.rescale_every = rescale_every
@@ -114,7 +120,8 @@ class Rwkv5Config(PretrainedConfig):
 
         self.bos_token_id = bos_token_id
         self.eos_token_id = eos_token_id
+        self.model_version = model_version
 
         super().__init__(
             tie_word_embeddings=tie_word_embeddings, bos_token_id=bos_token_id, eos_token_id=eos_token_id, **kwargs
-        )
+        )

modeling_rwkv5.py

@@ -178,6 +178,7 @@ def rwkv_linear_attention_v5_cpu(
     gate,
     lxw,
     lxb,
+    head_size_divisor,
     ow,
     state,
 ):
@@ -199,7 +200,7 @@ def rwkv_linear_attention_v5_cpu(
             state = at + time_decay * state
 
     out = out.reshape(B * T, H * S)
-    out = F.group_norm(out, num_groups=H, weight=lxw, bias=lxb).reshape(B, T, H * S)
+    out = F.group_norm(out / head_size_divisor, num_groups=H, weight=lxw, bias=lxb).reshape(B, T, H * S)
     out = out.to(dtype=hidden.dtype) * gate
     out = out @ ow
 
@@ -221,6 +222,7 @@ def rwkv_linear_attention(
     gate,
     lxw,
     lxb,
+    head_size_divisor,
     ow,
     state,
 ):
@@ -244,13 +246,14 @@ def rwkv_linear_attention(
             gate,
             lxw,
             lxb,
+            head_size_divisor,
             ow,
             state,
         )
     else:
         out, state = WKV_5.apply(B, T, H * S, H, receptance, key, value, time_decay, time_first, state)
         out = out.reshape(B * T, H * S)
-        out = F.group_norm(out, num_groups=H, weight=lxw, bias=lxb).reshape(B, T, H * S)
+        out = F.group_norm(out / head_size_divisor, num_groups=H, weight=lxw, bias=lxb).reshape(B, T, H * S)
         out = out.to(dtype=hidden.dtype) * gate
         out = out @ ow
         return out, state
@@ -271,6 +274,7 @@ class RwkvSelfAttention(nn.Module):
         # https://github.com/BlinkDL/RWKV-LM/blob/main/RWKV-v4neo/src/model.py#L146
         num_attention_heads = hidden_size // config.head_size
         self.num_attention_heads = num_attention_heads
+        self.head_size_divisor = config.head_size_divisor
         attention_hidden_size = (
             config.attention_hidden_size if config.attention_hidden_size is not None else hidden_size
         )
@@ -343,6 +347,7 @@ class RwkvSelfAttention(nn.Module):
             gate,
             self.ln_x.weight,
             self.ln_x.bias,
+            self.head_size_divisor,
             self.output.weight.t(),
             state=layer_state,
         )
@@ -747,14 +752,6 @@ class Rwkv5Model(Rwkv5PreTrainedModel):
                         block.attention.output.weight.mul_(2 ** int(block_id // self.config.rescale_every))
                         block.feed_forward.value.weight.mul_(2 ** int(block_id // self.config.rescale_every))
                     else:
-                        # Deal with quantization statistics
-                        if hasattr(block.attention.output.weight, "SCB"):
-                            block.attention.output.weight.SCB.div_(2 ** int(block_id // self.config.rescale_every))
-                            block.feed_forward.value.weight.SCB.div_(2 ** int(block_id // self.config.rescale_every))
-                        elif hasattr(block.attention.output.weight, "quant_state"):
-                            self._bnb_4bit_dequantize_and_rescale(block.attention.output, block_id)
-                            self._bnb_4bit_dequantize_and_rescale(block.feed_forward.value, block_id)
-                        else:
                             block.attention.output.weight.div_(2 ** int(block_id // self.config.rescale_every))
                             block.feed_forward.value.weight.div_(2 ** int(block_id // self.config.rescale_every))
 
@@ -859,4 +856,4 @@ class Rwkv5ForCausalLM(Rwkv5PreTrainedModel):
             state=rwkv_outputs.state,
             hidden_states=rwkv_outputs.hidden_states,
             attentions=rwkv_outputs.attentions,
-        )
+        )