Update Mo.py

Mo.py

@@ -99,7 +99,7 @@ def txt_stream(file_path):
             )
 
 
-LIMIT =   # 원하는 만큼
+LIMIT = 2000000  # 원하는 만큼
 
 dataset = tf.data.Dataset.from_generator(
     lambda: txt_stream(DATA_PATH),
@@ -141,7 +141,7 @@ class LoU(layers.Layer):
         self.norm1 = layers.LayerNormalization(epsilon=1e-5, dtype='float32')
 
         # 비선형 표현력
-        self.glu = SwiGLU(d_model, 3500)
+        self.glu = SwiGLU(d_model, 320)
 
         # 학습 가능한 과거 토큰 가중치
         self.alpha = self.add_weight(shape=(d_model,), initializer='ones', trainable=True)
@@ -241,7 +241,7 @@ def masked_perplexity(y_true, y_pred, eps=0.1):
 # 모델 생성 & 컴파일
 # =======================
 with strategy.scope():
-    model = CumaLM(vocab_size=vocab_size, max_seq_len=max_len, d_ff=768, n_layers=12)
+    model = CumaLM(vocab_size=vocab_size, max_seq_len=max_len, d_ff=256, n_layers=1)
     dummy_input = tf.zeros((batch_size, max_len), dtype=tf.int32)
     _ = model(dummy_input, training=False)
     model.summary()