Upload config.json
Browse files- config.json +7 -1
config.json
CHANGED
|
@@ -1,4 +1,5 @@
|
|
| 1 |
{
|
|
|
|
| 2 |
"add_bias_linear": false,
|
| 3 |
"architectures": [
|
| 4 |
"Zamba2ForCausalLM"
|
|
@@ -69,8 +70,12 @@
|
|
| 69 |
"m",
|
| 70 |
"m"
|
| 71 |
],
|
|
|
|
|
|
|
| 72 |
"lora_rank": 128,
|
| 73 |
"mamba_headdim": 64,
|
|
|
|
|
|
|
| 74 |
"model_type": "zamba2",
|
| 75 |
"num_attention_heads": 32,
|
| 76 |
"num_hidden_layers": 54,
|
|
@@ -86,5 +91,6 @@
|
|
| 86 |
"use_cache": true,
|
| 87 |
"use_mamba_kernels": true,
|
| 88 |
"use_shared_block_lora": true,
|
| 89 |
-
"
|
|
|
|
| 90 |
}
|
|
|
|
| 1 |
{
|
| 2 |
+
"_name_or_path": "Zyphra/Zamba2-2.7B",
|
| 3 |
"add_bias_linear": false,
|
| 4 |
"architectures": [
|
| 5 |
"Zamba2ForCausalLM"
|
|
|
|
| 70 |
"m",
|
| 71 |
"m"
|
| 72 |
],
|
| 73 |
+
"num_mem_blocks": 2,
|
| 74 |
+
"max_position_embeddings": 4096,
|
| 75 |
"lora_rank": 128,
|
| 76 |
"mamba_headdim": 64,
|
| 77 |
+
"use_mem_rope": false,
|
| 78 |
+
"rope_theta": 10000,
|
| 79 |
"model_type": "zamba2",
|
| 80 |
"num_attention_heads": 32,
|
| 81 |
"num_hidden_layers": 54,
|
|
|
|
| 91 |
"use_cache": true,
|
| 92 |
"use_mamba_kernels": true,
|
| 93 |
"use_shared_block_lora": true,
|
| 94 |
+
"se_shared_attention_lora": false,
|
| 95 |
+
"vocab_size": 32001
|
| 96 |
}
|