{ | |
"_name_or_path": "efederici/ipt-350m", | |
"architectures": [ | |
"MPTForCausalLM" | |
], | |
"attn_config": { | |
"model_type": "" | |
}, | |
"auto_map": { | |
"AutoConfig": "efederici/ipt-350m--configuration_mpt.MPTConfig", | |
"AutoModelForCausalLM": "efederici/ipt-350m--modeling_mpt.MPTForCausalLM" | |
}, | |
"d_model": 1024, | |
"emb_pdrop": 0.0, | |
"embedding_fraction": 1.0, | |
"expansion_ratio": 4, | |
"init_config": { | |
"emb_init_std": null, | |
"emb_init_uniform_lim": null, | |
"fan_mode": "fan_in", | |
"init_div_is_residual": true, | |
"init_gain": 0.0, | |
"init_nonlinearity": "relu", | |
"init_std": null, | |
"name": "kaiming_normal_", | |
"verbose": 0 | |
}, | |
"init_device": "cpu", | |
"initializer_range": 0.02, | |
"layer_norm_epsilon": 1e-05, | |
"learned_pos_emb": true, | |
"logit_scale": null, | |
"max_seq_len": 2048, | |
"model_type": "mpt", | |
"n_heads": 16, | |
"n_layers": 24, | |
"no_bias": true, | |
"norm_type": "low_precision_layernorm", | |
"resid_pdrop": 0.0, | |
"transformers_version": "4.32.1", | |
"use_cache": false, | |
"verbose": 0, | |
"vocab_size": 50432 | |
} | |