Push model using huggingface_hub.

README.md

@@ -0,0 +1,9 @@
+---
+tags:
+- pytorch_model_hub_mixin
+- model_hub_mixin
+---
+
+This model has been pushed to the Hub using ****:
+- Repo: [More Information Needed]
+- Docs: [More Information Needed]

config.json

@@ -0,0 +1,24 @@
+{
+  "bias": false,
+  "capacity_factor": 0.12,
+  "d_model": 1024,
+  "dropout": 0.2,
+  "ffn": "swiglu",
+  "hidden_dim": 4096,
+  "mixture_of_depth": true,
+  "mixture_of_expert": false,
+  "model_type": {
+    "mixture_of_depth": true,
+    "name": "mixture of depth"
+  },
+  "moe_num_experts": 4,
+  "moe_num_experts_per_tok": 2,
+  "multiple_of": 4,
+  "num_heads": 16,
+  "num_kv_heads": 0,
+  "num_layers": 16,
+  "seq_len": 512,
+  "vocab_size": 50257,
+  "weight_tying": true,
+  "window_size": 128
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a31bfebb9bca10b5ee49c499aa9f8e74bb81f7df94345a7043a72cdad4a90386
+size 1281064680