Upload 2 files

LilLM_300M_9_5B_Tokens.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d1fc8ffbf766c30da74939cc1629aaf12f424cbe905a5f4e44224b283f2eae96
+size 3498930790

config.py

@@ -0,0 +1,16 @@
+from dataclasses import dataclass 
+
+@dataclass
+class Config:
+    vocab_size: int = 2**13
+    d_model: int = 1024
+    n_layers: int = 24
+    max_seq_len: int = 512
+    q_heads: int = 16
+    kv_heads: int = 8
+    dropout: float = 0.0
+    max_batch_size: int = 32
+    hidden_dim: int = None
+    multiple_of: int = 128
+    eps: float = 1e-6
+    flash: bool = True