Commit
•
15022e2
1
Parent(s):
ca597f0
Upload 2 files
Browse files- 2.8b.pt +3 -0
- 2.8b_config.json +1 -0
2.8b.pt
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:350ca5cc49fe08200c1fd21148fee071bd6e4ee760047290eac99c70ccb7cf7f
|
3 |
+
size 5566617017
|
2.8b_config.json
ADDED
@@ -0,0 +1 @@
|
|
|
|
|
1 |
+
{"num_layers": 36, "hidden_size": 1152, "state_size": 16, "conv_dimension": 4, "vocab_size": 50304, "expansion_factor": 2, "mamba_moe_layers": ["r", "8", "r", "8", "r", "8", "r", "8", "r", "8", "r", "8", "r", "8", "r", "8", "r", "8", "r", "8", "r", "8", "r", "8", "r", "8", "r", "8", "r", "8", "r", "8", "r", "8", "r", "8"], "ffn_hidden_size": 3872, "bias": false, "add_bias_linear": false, "swiglu": true, "max_sequence_length": 2048}
|