amd
/

X-EcoMLA-1B8B-fixed-kv64-DPO

alignment-handbook

Generated from Trainer

Model card Files Files and versions

Mingyuyang-1 commited on Aug 11

Commit

66bb400

·

1 Parent(s): 264bdbb

Update mla_layer_config.json

Files changed (1) hide show

mla_layer_config.json +1 -4

mla_layer_config.json CHANGED Viewed

@@ -99,10 +99,7 @@
   "rms_norm_eps": 1e-06,
   "rope_scaling": {
     "factor": 32.0,
-    "beta_fast": 32.0,
-    "beta_slow": 1.0,
-    "mscale": 1.0,
-    "mscale_all_dim": 0.0,
     "type": "yarn"
   },
   "rope_theta": 500000.0,

   "rms_norm_eps": 1e-06,
   "rope_scaling": {
     "factor": 32.0,
+    "original_max_position_embeddings": 8192,
     "type": "yarn"
   },
   "rope_theta": 500000.0,