|
{ |
|
"_name_or_path": "/zhouxibin/models/huggingface/Evolla-10B-hf", |
|
"aligner_attention_probs_dropout_prob": 0.1, |
|
"aligner_enable_bias": true, |
|
"aligner_ffn_mult": 4, |
|
"aligner_num_add_layers": 8, |
|
"architectures": [ |
|
"EvollaForProteinText2Text" |
|
], |
|
"attention_bias": false, |
|
"attention_dropout": 0.0, |
|
"bos_token_id": 128000, |
|
"eos_token_id": 128009, |
|
"hidden_act": "silu", |
|
"hidden_size": 4096, |
|
"initializer_range": 0.02, |
|
"intermediate_size": 14336, |
|
"llm_config": { |
|
"_attn_implementation_autoset": true, |
|
"gradient_checkpointing": false, |
|
"model_type": "evolla_llm", |
|
"sequence_aligner_config": { |
|
"protein_encoder_dim": 4096 |
|
} |
|
}, |
|
"max_position_embeddings": 8192, |
|
"mlp_bias": false, |
|
"model_type": "EvollaModel", |
|
"num_attention_heads": 32, |
|
"num_hidden_layers": 32, |
|
"num_key_value_heads": 8, |
|
"pretraining_tp": 1, |
|
"protein_attention_probs_dropout_prob": 0.1, |
|
"protein_config": { |
|
"model_type": "evolla_protein", |
|
"resampler_config": { |
|
"output_repr_dim": 4096, |
|
"protein_repr_dim": 1280 |
|
} |
|
}, |
|
"protein_emb_layer_norm_before": false, |
|
"protein_hidden_dropout_prob": 0.1, |
|
"protein_hidden_size": 1280, |
|
"protein_intermediate_size": 5120, |
|
"protein_layer_norm_eps": 1e-05, |
|
"protein_mask_token_id": 4, |
|
"protein_max_position_embeddings": 1026, |
|
"protein_num_attention_heads": 20, |
|
"protein_num_hidden_layers": 33, |
|
"protein_pad_token_id": 1, |
|
"protein_position_embedding_type": "rotary", |
|
"protein_token_dropout": true, |
|
"protein_vocab_size": 446, |
|
"resampler_depth": 6, |
|
"resampler_dim_head": 64, |
|
"resampler_ff_mult": 4, |
|
"resampler_heads": 8, |
|
"resampler_num_latents": 64, |
|
"rms_norm_eps": 1e-05, |
|
"rope_scaling": null, |
|
"rope_theta": 500000.0, |
|
"tie_word_embeddings": false, |
|
"torch_dtype": "float32", |
|
"transformers_version": "4.49.0.dev0", |
|
"use_cache": false, |
|
"vocab_size": 128256 |
|
} |
|
|