RWKV
/

rwkv-6-world-7b

Text Generation

Model card Files Files and versions Community

KaleiNeely commited on May 10

Commit

dfc9ae3

•

1 Parent(s): 29770da

Update modeling_rwkv6.py

Files changed (1) hide show

modeling_rwkv6.py +1 -1

modeling_rwkv6.py CHANGED Viewed

@@ -132,7 +132,7 @@ class Rwkv6SelfAttention(nn.Module):
         TIME_DECAY_EXTRA_DIM = 64
         if hidden_size == 4096: #7b
-            TIME_MIX_EXTRA_DIM = 128
         self.time_decay_w1 = nn.Parameter(torch.empty(hidden_size, TIME_DECAY_EXTRA_DIM))
         self.time_decay_w2 = nn.Parameter(torch.empty(TIME_DECAY_EXTRA_DIM, attention_hidden_size))

         TIME_DECAY_EXTRA_DIM = 64
         if hidden_size == 4096: #7b
+            TIME_DECAY_EXTRA_DIM = 128
         self.time_decay_w1 = nn.Parameter(torch.empty(hidden_size, TIME_DECAY_EXTRA_DIM))
         self.time_decay_w2 = nn.Parameter(torch.empty(TIME_DECAY_EXTRA_DIM, attention_hidden_size))