File size: 1,014 Bytes
3838886 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 |
{
"_name_or_path": "./checkpoints/LoveLace-M-vocab_X9Ge",
"architectures": [
"LSWTForCausalLM"
],
"bos_token_id": 2,
"d_ffn": 4096,
"d_model": 1536,
"d_vocab": 768,
"dropout_att_mat": 0.0,
"dropout_att_out": 0.0,
"dropout_ffn_int": 0.0,
"dropout_ffn_out": 0.0,
"dropout_layers": 0.0,
"enable_bias": true,
"eos_token_id": 2,
"gated_att": false,
"gated_ffn": true,
"init_std": 0.02,
"model_type": "lsw_transformer",
"n_heads": 24,
"n_layers": 18,
"n_registers": 0,
"pad_token_id": 1,
"parent_embeddings": "facebook/opt-125m",
"qk_norm": false,
"recompute_kv": false,
"reward_dropout": 0.0,
"reward_heads": null,
"reward_pooler": "identity",
"rope_base_freq": 500000,
"rope_dynamic": false,
"rope_ntk_scale": 1.0,
"rope_positions": 4096,
"rope_reversed": true,
"rope_yarn_a": 0.07,
"rope_yarn_b": 1.0,
"torch_dtype": "float16",
"trainable_embeddings": true,
"transformers_version": "4.37.2",
"use_cache": true,
"vocab_size": 50272
}
|