Upload MyLLaMa

llama.py

@@ -92,7 +92,7 @@ class RMSNorm(nn.Module):
         super().__init__()
 
         self.dim = dim
-        self.gamma = nn.Parameter(
+        self.trainable = nn.Parameter(
             data=torch.nn.init.normal_(torch.zeros((dim,))), requires_grad=True
         )
         self.eps = eps
@@ -111,7 +111,7 @@ class RMSNorm(nn.Module):
         return (
             x
             / torch.sqrt_(torch.mean(torch.square(x), dim=-1) + self.eps)[:, :, None]
-            * self.gamma
+            * self.trainable
         )
 
     def extra_repr(self) -> str:

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6380bd512c40cdd9705099299688b1b4965a8da9da94f8f9d4b29a5b3ac5bf06
-size 3161813608
+oid sha256:daef12499060e6075aece26c79ed43c39d7a1f9ebade23e92004922b5eba2a87
+size 3161813808