|
{ |
|
"model": "glm2", |
|
"template": "glm4", |
|
"hidden_act": "swiglu", |
|
"hidden_size": 4096, |
|
"intermediate_size": 13696, |
|
"layer_norm_eps": 1.5625e-07, |
|
"max_sequence_length": 131072, |
|
"num_attention_heads": 32, |
|
"num_hidden_layers": 40, |
|
"vocab_size": 151552, |
|
"segment_vocab_size": 0, |
|
"num_key_value_heads": 2, |
|
"skip_init": true, |
|
"tie_emb_prj_weight": false, |
|
"rmsnorm": true, |
|
"rope_rank": "adjacent", |
|
"position_encoding_2d": true, |
|
"flash_attention": true, |
|
"eos_token_id": [151329, 151336, 151338], |
|
"pad_token_id": 151329, |
|
"generation_config": {"tokenizer_config": {"skip_special_tokens": true}, |
|
"eos_token_id": [151329, 151336, 151338], "max_length": 131072} |
|
} |