Upload config.json

config.json

@@ -0,0 +1,13 @@
+{
+ "vocab_size": 259,
+ "num_layers": 30,
+ "num_heads": 9,
+ "num_kv_heads": 3,
+ "embed_dim": 576,
+ "max_seq_len": 384,
+ "intermediate_dim": 1536,
+ "attn_dropout": 0.0,
+ "norm_eps": 1e-5,
+ "rope_base": 10000,
+ "scale_factor": 32
+}