open_lm/model_configs/l7b_neox.json

{
    "hidden_dim": 4096,
    "n_layers": 32,
    "n_heads": 32,
    "seq_len": 2048,
    "vocab_size": 50432,
    "post_embed_norm": false,
    "weight_tying": false
}