Update config.json
Browse files- config.json +2 -19
config.json
CHANGED
@@ -29,28 +29,11 @@
|
|
29 |
"mm_vision_select_feature": "patch",
|
30 |
"mm_vision_select_layer": -2,
|
31 |
"mm_vision_tower": "/path/to/clip-vit-large-patch14-336",
|
32 |
-
"model_type": "
|
33 |
"moe": true,
|
34 |
"num_attention_heads": 32,
|
35 |
-
"num_experts":
|
36 |
"num_experts_per_tok": 2,
|
37 |
-
"moe_type": "dense",
|
38 |
-
"ep_size": 8,
|
39 |
-
"capacity_factor": 1.5,
|
40 |
-
"lora_r": 8,
|
41 |
-
"lora_alpha": 16,
|
42 |
-
"image_expert_indices": [2, 3, 4, 6],
|
43 |
-
"video_expert_indices": [2, 3, 4, 6],
|
44 |
-
"audio_expert_indices": [0, 4, 5, 7],
|
45 |
-
"use_fm_block": false,
|
46 |
-
"use_flash_attn": false,
|
47 |
-
"aux_balance_loss": true,
|
48 |
-
"aux_balance_loss_coef": 0.01,
|
49 |
-
"aux_multimodal_loss": false,
|
50 |
-
"aux_multimodal_loss_coef": 0.01,
|
51 |
-
"aux_kl_loss": false,
|
52 |
-
"aux_kl_loss_coef": 0.01,
|
53 |
-
"lora_dropout": 0.05,
|
54 |
"num_hidden_layers": 32,
|
55 |
"num_key_value_heads": 32,
|
56 |
"pad_token_id": 0,
|
|
|
29 |
"mm_vision_select_feature": "patch",
|
30 |
"mm_vision_select_layer": -2,
|
31 |
"mm_vision_tower": "/path/to/clip-vit-large-patch14-336",
|
32 |
+
"model_type": "llava",
|
33 |
"moe": true,
|
34 |
"num_attention_heads": 32,
|
35 |
+
"num_experts": 4,
|
36 |
"num_experts_per_tok": 2,
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
37 |
"num_hidden_layers": 32,
|
38 |
"num_key_value_heads": 32,
|
39 |
"pad_token_id": 0,
|