config.json

{
  "n_embed": 3584,
  "n_heads": 28,
  "n_kv_heads": 4,
  "n_layer": 28,
  "n_mlp": 18944,
  "rms_norm_eps": 1e-06,
  "rope_theta": 1000000.0,
  "tie_word_embeddings": false,
  "vocab_size": 152064
}