Upload 2 files

2.8b.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:350ca5cc49fe08200c1fd21148fee071bd6e4ee760047290eac99c70ccb7cf7f
+size 5566617017

2.8b_config.json

@@ -0,0 +1 @@
+{"num_layers": 36, "hidden_size": 1152, "state_size": 16, "conv_dimension": 4, "vocab_size": 50304, "expansion_factor": 2, "mamba_moe_layers": ["r", "8", "r", "8", "r", "8", "r", "8", "r", "8", "r", "8", "r", "8", "r", "8", "r", "8", "r", "8", "r", "8", "r", "8", "r", "8", "r", "8", "r", "8", "r", "8", "r", "8", "r", "8"], "ffn_hidden_size": 3872, "bias": false, "add_bias_linear": false, "swiglu": true, "max_sequence_length": 2048}