Update config.json

config.json

@@ -1,7 +1,7 @@
 {
-  "_name_or_path": "nisten/lobotollama3",
+  "_name_or_path": "/content/llama-3-8b-instruct",
   "architectures": [
-    "LlamaForCausalLM"
+    "MixtralForCausalLM"
   ],
   "attention_bias": false,
   "attention_dropout": 0.0,
@@ -11,64 +11,24 @@
   "hidden_size": 4096,
   "initializer_range": 0.02,
   "intermediate_size": 14336,
-  "max_position_embeddings": 8192,
-  "model_type": "llama",
+  "max_position_embeddings": 65536,
+  "model_type": "mixtral",
   "num_attention_heads": 32,
-  "num_experts": 2,
   "num_experts_per_tok": 2,
   "num_hidden_layers": 32,
   "num_key_value_heads": 8,
+  "num_local_experts": 2,
+  "output_router_logits": false,
   "pretraining_tp": 1,
   "rms_norm_eps": 1e-05,
-    "rope_scaling": {
-    "factor": 4.0,
-    "original_max_position_embeddings": 8192,
-    "type": "yarn",
-    "finetuned": "true"
-  },
-  "rope_theta": 8000000.0,
-  "router_layers": [
-    "gate_proj",
-    "up_proj",
-    "down_proj"
-  ],
-  "router_layers_index": [
-    0,
-    1,
-    2,
-    3,
-    4,
-    5,
-    6,
-    7,
-    8,
-    9,
-    10,
-    11,
-    12,
-    13,
-    14,
-    15,
-    16,
-    17,
-    18,
-    19,
-    20,
-    21,
-    22,
-    23,
-    24,
-    25,
-    26,
-    27,
-    28,
-    29,
-    30,
-    31
-  ],
+  "rope_scaling": null,
+  "rope_theta": 500000.0,
+  "router_aux_loss_coef": 0.001,
+  "router_jitter_noise": 0.0,
+  "sliding_window": null,
   "tie_word_embeddings": false,
   "torch_dtype": "bfloat16",
-  "transformers_version": "4.40.0.dev0",
+  "transformers_version": "4.41.0.dev0",
   "use_cache": true,
   "vocab_size": 128256
-}
+}