File size: 853 Bytes
b46c239 2bd49dc b46c239 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 |
{
"architectures": [
"OpenLMModel"
],
"model_type": "openlm",
"params": {
"apply_qk_norm": true,
"attn_activation": null,
"attn_name": "auto",
"attn_seq_scalar": null,
"attn_seq_scalar_alpha": null,
"dim": 4096,
"ffn_type": "swiglu",
"model": "open_lm_7b",
"model_norm": "gain_only_lp_layer_norm",
"moe_capacity_factor": 1.25,
"moe_expert_model_parallelism": false,
"moe_freq": 0,
"moe_loss_weight": 0.1,
"moe_num_experts": null,
"moe_top_k": 2,
"moe_weight_parallelism": false,
"n_heads": 32,
"n_layers": 32,
"norm_eps": 1e-05,
"positional_embedding_type": "rotary",
"post_embed_norm": false,
"qk_norm": true,
"seq_len": 2048,
"vocab_size": 50432,
"weight_tying": false
},
"torch_dtype": "float32",
"transformers_version": "4.41.2"
}
|