Llama3.2-Mamba-3B-distill / mamba_config.json
Jingyu6
add models
2b07191
raw
history blame contribute delete
423 Bytes
{
"d_model": 3072,
"ssm_cfg": {
"expand": 1
},
"rms_norm_eps": 1e-05,
"vocab_size": null,
"d_inner": null,
"d_xb": 1024,
"intermediate_size": 8192,
"hidden_act": "silu",
"n_layer": 28,
"attn_layers": [
1,
3,
5,
7,
9,
11,
13,
15,
17,
19,
21,
23,
25,
27
]
}