Upload config.json with huggingface_hub

config.json

@@ -4,13 +4,13 @@
     ],
     "vocab_size": 20064,
     "hidden_size": 1024,
-    "num_hidden_layers": 2,
-    "num_attention_heads": 8,
+    "num_hidden_layers": 16,
+    "num_attention_heads": 64,
     "embd_pdrop": 0.1,
     "use_moe": true,
-    "num_experts": 4,
-    "expert_hidden_size": 4096,
+    "num_experts": 24,
+    "expert_hidden_size": 496,
     "dropout": 0.1,
     "model_type": "lake",
-    "torch_dtype": "float64"
+    "torch_dtype": "float16"
 }