OpenLab-NLP
/

model-prototype

Model card Files Files and versions

Yuchan commited on 30 days ago

Commit

d916c7d

·

verified ·

1 Parent(s): a2ab022

Update Mo.py

Files changed (1) hide show

Mo.py +1 -1

Mo.py CHANGED Viewed

@@ -118,7 +118,7 @@ with strategy.scope():
 class SwiGLU(layers.Layer):
     def __init__(self, d_model, d_ff):
         super().__init__()
-        self.proj = layers.Dense(dff)
         self.out = layers.Dense(d_model)
     def call(self, x):
         x_proj = self.proj(x)

 class SwiGLU(layers.Layer):
     def __init__(self, d_model, d_ff):
         super().__init__()
+        self.proj = layers.Dense(d_ff)
         self.out = layers.Dense(d_model)
     def call(self, x):
         x_proj = self.proj(x)