{ | |
"apply_residual_connection_post_layernorm": false, | |
"attention_dropout": 0.0, | |
"attention_softmax_in_fp32": true, | |
"bias_dropout_fusion": true, | |
"bos_token_id": 1, | |
"eos_token_id": 2, | |
"pad_token_id": 3, | |
"unk_token_id": 0, | |
"hidden_dropout": 0.0, | |
"hidden_act": "gelu_fast", | |
"initializer_range": 0.02, | |
"layer_norm_eps": 1e-05, | |
"hidden_size": 1024, | |
"intermediate_size": 4096, | |
"num_hidden_layers": 24, | |
"num_attention_heads": 16, | |
"offset_alibi": 100, | |
"pretraining_tp": 1, | |
"skip_bias_add": true, | |
"skip_bias_add_qkv": false, | |
"vocab_size": 250880, | |
"segment_vocab_size": 0, | |
"pre_layernorm": true, | |
"tie_word_embeddings": true, | |
"model": "bloom" | |
} |