Upload JambaForCausalLM

config.json

@@ -1,5 +1,5 @@
 {
-  "_name_or_path": "ai21labs/Jamba-v0.1",
+  "_name_or_path": "isemmanuelolowe/Jamba-4xMoE_slerp",
   "architectures": [
     "JambaForCausalLM"
   ],
@@ -7,10 +7,10 @@
   "attn_layer_offset": 4,
   "attn_layer_period": 8,
   "auto_map": {
-    "AutoConfig": "ai21labs/Jamba-v0.1--configuration_jamba.JambaConfig",
-    "AutoModel": "ai21labs/Jamba-v0.1--modeling_jamba.JambaModel",
-    "AutoModelForCausalLM": "ai21labs/Jamba-v0.1--modeling_jamba.JambaForCausalLM",
-    "AutoModelForSequenceClassification": "ai21labs/Jamba-v0.1--model.JambaForSequenceClassification"
+    "AutoConfig": "isemmanuelolowe/Jamba-4xMoE_slerp--configuration_jamba.JambaConfig",
+    "AutoModel": "isemmanuelolowe/Jamba-4xMoE_slerp--modeling_jamba.JambaModel",
+    "AutoModelForCausalLM": "isemmanuelolowe/Jamba-4xMoE_slerp--modeling_jamba.JambaForCausalLM",
+    "AutoModelForSequenceClassification": "isemmanuelolowe/Jamba-4xMoE_slerp--model.JambaForSequenceClassification"
   },
   "bos_token_id": 1,
   "calc_logits_for_entire_prompt": false,
@@ -31,29 +31,12 @@
   "model_type": "jamba",
   "n_ctx": 262144,
   "num_attention_heads": 32,
-  "num_experts": 16,
+  "num_experts": 4,
   "num_experts_per_tok": 2,
   "num_hidden_layers": 32,
   "num_key_value_heads": 8,
   "output_router_logits": false,
   "pad_token_id": 0,
-  "quantization_config": {
-    "_load_in_4bit": true,
-    "_load_in_8bit": false,
-    "bnb_4bit_compute_dtype": "float32",
-    "bnb_4bit_quant_storage": "uint8",
-    "bnb_4bit_quant_type": "fp4",
-    "bnb_4bit_use_double_quant": false,
-    "llm_int8_enable_fp32_cpu_offload": false,
-    "llm_int8_has_fp16_weight": false,
-    "llm_int8_skip_modules": [
-      "mamba"
-    ],
-    "llm_int8_threshold": 6.0,
-    "load_in_4bit": true,
-    "load_in_8bit": false,
-    "quant_method": "bitsandbytes"
-  },
   "rms_norm_eps": 1e-06,
   "router_aux_loss_coef": 0.001,
   "sliding_window": null,

model-00001-of-00008.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9222ed281e8f2cbf3b531e22ca1514267559e5c408668f3985e944850a8a61ba
+size 4987630232

model-00002-of-00008.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9074b89993cb1c8a4df25b413c15ac811367ab34263e2cfc26aeb3f4b544ce15
+size 4930124392

model-00003-of-00008.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1b72f78ad801bde740c0860faf2d24e2423e0771faae1b791c5e023f9e798999
+size 4944522688

model-00004-of-00008.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a49c767416c0f468462f6f25fcb49ac927647ac8117a248c5d862deff77a9171
+size 4954075928

model-00005-of-00008.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:66d1521e9dfbf5479c2bc589ea1a202a5f5beae8a5c088cb47f2bf92fd3a8ef7
+size 4921178040

model-00006-of-00008.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:87864b5e4d96a9a76d71bc798b67854ba567fbfdf77b9580a9adb58ca9efc19b
+size 4929484872

model-00007-of-00008.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0b73c561783801dc085949ac87eae658d122dd3c7826dc660e10d4b530be547e
+size 4944522776

model-00008-of-00008.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f88e8de536daee5b7da10980e98b4d735efa5a5ab16bd671a4332143aac9a042
+size 889217840