Upload MoAMetricLM

config.json

@@ -3,7 +3,7 @@
   "architectures": [
     "MoAMetricLM"
   ],
-  "attn_drop": 0.1,
+  "attn_drop": 0.0,
   "attn_heads": 16,
   "bos_token_id": 0,
   "chunk_size": 256,
@@ -15,8 +15,8 @@
   "dtype": "float32",
   "enable_feature_gates": true,
   "enable_router_gates": true,
-  "enable_v_energy": false,
-  "energy_amplification": 3.1415,
+  "enable_v_energy": true,
+  "energy_amplification": 9.869604401089358,
   "eos_token_id": 0,
   "ff_mult": 3,
   "ffn_hidden": 1536,
@@ -53,10 +53,10 @@
   "pad_token_id": 0,
   "proj_drop": 0.1,
   "r_basis": 16,
-  "radius_init": 3.5,
+  "radius_init": 4.0,
   "router_dropout": 0.1,
   "router_hidden": 128,
-  "router_temperature": 1.25,
+  "router_temperature": 1.0,
   "router_topk": 2,
   "theta_base": 10000.0,
   "ti_reg_samples": 64,

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b62fe65bba633f30f053e73782caedd97f2f856d0dd987c01bc6bb7fb23cf9da
+oid sha256:47e76b52d39b9b2f99a15083e2d9a96941634d9ba21affd52bfc4f1055c0d0cb
 size 218651604