{ "attn_pdrop": 0.0, "bos_token_id": 1, "eos_token_id": 2, "pad_token_id": null, "hidden_act": "silu", "hidden_size": 2304, "initializer_range": 0.1, "intermediate_size": 5760, "max_position_embeddings": 2048, "num_attention_heads": 36, "num_hidden_layers": 40, "num_key_value_heads": 36, "pretraining_tp": 1, "rms_norm_eps": 1e-05, "rope_theta": 10000.0, "tie_word_embeddings": true, "use_cache": true, "vocab_size": 122753, "scale_emb": 12, "dim_model_base": 256, "scale_depth": 1.4 }