|
{ |
|
"_name_or_path": "artifacts/model-llama3__gs_ai__cont_lora:v0", |
|
"architectures": [ |
|
"PartiallyFrozenGazelleForConditionalGeneration" |
|
], |
|
"audio_config": { |
|
"_name_or_path": "facebook/wav2vec2-base-960h", |
|
"architectures": [ |
|
"Wav2Vec2ForCTC" |
|
], |
|
"feat_extract_dropout": 0.0, |
|
"feat_proj_dropout": 0.1, |
|
"gradient_checkpointing": false, |
|
"hidden_dropout_prob": 0.1, |
|
"model_type": "wav2vec2" |
|
}, |
|
"audio_model_id": "facebook/wav2vec2-base-960h", |
|
"audio_model_lora_config": { |
|
"lora_alpha": 8, |
|
"r": 0, |
|
"target_modules": [ |
|
"k_proj", |
|
"q_proj", |
|
"linear_k", |
|
"linear_q" |
|
] |
|
}, |
|
"audio_token_index": 128257, |
|
"hidden_size": 4096, |
|
"ignore_index": -100, |
|
"model_type": "gazelle", |
|
"stack_factor": 8, |
|
"text_config": { |
|
"_name_or_path": "meta-llama/Meta-Llama-3-8B-Instruct", |
|
"architectures": [ |
|
"LlamaForCausalLM" |
|
], |
|
"bos_token_id": 128000, |
|
"eos_token_id": 128001, |
|
"intermediate_size": 14336, |
|
"max_position_embeddings": 8192, |
|
"model_type": "llama", |
|
"num_key_value_heads": 8, |
|
"rms_norm_eps": 1e-05, |
|
"rope_theta": 500000.0, |
|
"torch_dtype": "bfloat16", |
|
"vocab_size": 128258 |
|
}, |
|
"text_model_id": "meta-llama/Meta-Llama-3-8B-Instruct", |
|
"text_model_lora_config": { |
|
"lora_alpha": 8, |
|
"r": 64, |
|
"target_modules": [ |
|
"mlp.gate_proj", |
|
"mlp.up_proj", |
|
"mlp.down_proj", |
|
"v_proj", |
|
"o_proj", |
|
"k_proj", |
|
"q_proj" |
|
] |
|
}, |
|
"torch_dtype": "float32", |
|
"transformers_version": "4.40.0", |
|
"vocab_size": 32 |
|
} |
|
|