Commit
•
5eb3aa1
1
Parent(s):
213fe11
Upload HyenaDNAForCausalLM
Browse files- modeling_hyena.py +1 -1
modeling_hyena.py
CHANGED
|
@@ -62,7 +62,7 @@ class HyenaPositionalEmbedding(nn.Module):
|
|
| 62 |
f = torch.linspace(1e-4, bands - 1, bands)[None, None]
|
| 63 |
|
| 64 |
z = torch.cat([t, torch.cos(-f * w), torch.sin(-f * w)], dim=-1)
|
| 65 |
-
#
|
| 66 |
self.z = nn.Parameter(z, requires_grad=True)
|
| 67 |
self.register_buffer("t", t)
|
| 68 |
|
|
|
|
| 62 |
f = torch.linspace(1e-4, bands - 1, bands)[None, None]
|
| 63 |
|
| 64 |
z = torch.cat([t, torch.cos(-f * w), torch.sin(-f * w)], dim=-1)
|
| 65 |
+
# The original code sets z's LR to lr_pos_emb, which is 1e-5 by default
|
| 66 |
self.z = nn.Parameter(z, requires_grad=True)
|
| 67 |
self.register_buffer("t", t)
|
| 68 |
|