Update config.json
Browse files- config.json +2 -19
config.json
CHANGED
|
@@ -29,28 +29,11 @@
|
|
| 29 |
"mm_vision_select_feature": "patch",
|
| 30 |
"mm_vision_select_layer": -2,
|
| 31 |
"mm_vision_tower": "/path/to/clip-vit-large-patch14-336",
|
| 32 |
-
"model_type": "
|
| 33 |
"moe": true,
|
| 34 |
"num_attention_heads": 32,
|
| 35 |
-
"num_experts":
|
| 36 |
"num_experts_per_tok": 2,
|
| 37 |
-
"moe_type": "dense",
|
| 38 |
-
"ep_size": 8,
|
| 39 |
-
"capacity_factor": 1.5,
|
| 40 |
-
"lora_r": 8,
|
| 41 |
-
"lora_alpha": 16,
|
| 42 |
-
"image_expert_indices": [2, 3, 4, 6],
|
| 43 |
-
"video_expert_indices": [2, 3, 4, 6],
|
| 44 |
-
"audio_expert_indices": [0, 4, 5, 7],
|
| 45 |
-
"use_fm_block": false,
|
| 46 |
-
"use_flash_attn": false,
|
| 47 |
-
"aux_balance_loss": true,
|
| 48 |
-
"aux_balance_loss_coef": 0.01,
|
| 49 |
-
"aux_multimodal_loss": false,
|
| 50 |
-
"aux_multimodal_loss_coef": 0.01,
|
| 51 |
-
"aux_kl_loss": false,
|
| 52 |
-
"aux_kl_loss_coef": 0.01,
|
| 53 |
-
"lora_dropout": 0.05,
|
| 54 |
"num_hidden_layers": 32,
|
| 55 |
"num_key_value_heads": 32,
|
| 56 |
"pad_token_id": 0,
|
|
|
|
| 29 |
"mm_vision_select_feature": "patch",
|
| 30 |
"mm_vision_select_layer": -2,
|
| 31 |
"mm_vision_tower": "/path/to/clip-vit-large-patch14-336",
|
| 32 |
+
"model_type": "llava",
|
| 33 |
"moe": true,
|
| 34 |
"num_attention_heads": 32,
|
| 35 |
+
"num_experts": 4,
|
| 36 |
"num_experts_per_tok": 2,
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 37 |
"num_hidden_layers": 32,
|
| 38 |
"num_key_value_heads": 32,
|
| 39 |
"pad_token_id": 0,
|