Training in progress, step 50, checkpoint

last-checkpoint/adapter_config.json

@@ -20,13 +20,13 @@
   "rank_pattern": {},
   "revision": null,
   "target_modules": [
-    "down_proj",
     "up_proj",
-    "v_proj",
-    "o_proj",
     "q_proj",
+    "o_proj",
     "k_proj",
-    "gate_proj"
+    "v_proj",
+    "gate_proj",
+    "down_proj"
   ],
   "task_type": "CAUSAL_LM",
   "use_dora": false,

last-checkpoint/adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:934c417838770417091496d57694c16214b63ebf07326638c467b9ddd2104c94
+oid sha256:98272708b6bbcbd3897853bb6762aa8f8e39c8e5beb8ad70dd3a41f8cd0fa629
 size 90207248

last-checkpoint/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c8ed8f403df2815c17ad953379ae7a900c120097bc47240818f1463362ef225e
+oid sha256:10459f2fddba06dab03bf745d96ea2cee6eabbef45638797dc35edec0e636aa5
 size 180543866

last-checkpoint/trainer_state.json

@@ -1,5 +1,5 @@
 {
-  "best_metric": 0.4893987774848938,
+  "best_metric": 0.4894934296607971,
   "best_model_checkpoint": "miner_id_24/checkpoint-50",
   "epoch": 1.146131805157593,
   "eval_steps": 25,
@@ -10,7 +10,7 @@
   "log_history": [
     {
       "epoch": 0.022922636103151862,
-      "grad_norm": 31.560155868530273,
+      "grad_norm": 31.059669494628906,
       "learning_rate": 5e-05,
       "loss": 5.3734,
       "step": 1
@@ -18,368 +18,368 @@
     {
       "epoch": 0.022922636103151862,
       "eval_loss": 5.7447590827941895,
-      "eval_runtime": 2.6408,
-      "eval_samples_per_second": 111.328,
-      "eval_steps_per_second": 14.011,
+      "eval_runtime": 2.5969,
+      "eval_samples_per_second": 113.21,
+      "eval_steps_per_second": 14.248,
       "step": 1
     },
     {
       "epoch": 0.045845272206303724,
-      "grad_norm": 32.654319763183594,
+      "grad_norm": 32.138160705566406,
       "learning_rate": 0.0001,
       "loss": 4.9718,
       "step": 2
     },
     {
       "epoch": 0.06876790830945559,
-      "grad_norm": 28.378223419189453,
+      "grad_norm": 28.099864959716797,
       "learning_rate": 9.989294616193017e-05,
-      "loss": 4.2416,
+      "loss": 4.2536,
       "step": 3
     },
     {
       "epoch": 0.09169054441260745,
-      "grad_norm": 13.83468246459961,
+      "grad_norm": 13.594313621520996,
       "learning_rate": 9.957224306869053e-05,
-      "loss": 1.8319,
+      "loss": 1.83,
       "step": 4
     },
     {
       "epoch": 0.11461318051575932,
-      "grad_norm": 4.2792744636535645,
+      "grad_norm": 4.183432102203369,
       "learning_rate": 9.903926402016153e-05,
-      "loss": 0.9838,
+      "loss": 0.9864,
       "step": 5
     },
     {
       "epoch": 0.13753581661891118,
-      "grad_norm": 2.4947264194488525,
+      "grad_norm": 2.494304656982422,
       "learning_rate": 9.829629131445342e-05,
-      "loss": 0.8656,
+      "loss": 0.8677,
       "step": 6
     },
     {
       "epoch": 0.16045845272206305,
-      "grad_norm": 3.506563901901245,
+      "grad_norm": 3.5136210918426514,
       "learning_rate": 9.73465064747553e-05,
-      "loss": 0.9589,
+      "loss": 0.9556,
       "step": 7
     },
     {
       "epoch": 0.1833810888252149,
-      "grad_norm": 6.0768961906433105,
+      "grad_norm": 5.965522766113281,
       "learning_rate": 9.619397662556435e-05,
-      "loss": 1.0542,
+      "loss": 1.0476,
       "step": 8
     },
     {
       "epoch": 0.20630372492836677,
-      "grad_norm": 4.98295259475708,
+      "grad_norm": 5.070796012878418,
       "learning_rate": 9.484363707663442e-05,
-      "loss": 0.7218,
+      "loss": 0.7328,
       "step": 9
     },
     {
       "epoch": 0.22922636103151864,
-      "grad_norm": 4.773955821990967,
+      "grad_norm": 5.053555965423584,
       "learning_rate": 9.330127018922194e-05,
-      "loss": 0.5742,
+      "loss": 0.5888,
       "step": 10
     },
     {
       "epoch": 0.2521489971346705,
-      "grad_norm": 4.034650802612305,
+      "grad_norm": 4.020860195159912,
       "learning_rate": 9.157348061512727e-05,
-      "loss": 0.7807,
+      "loss": 0.7801,
       "step": 11
     },
     {
       "epoch": 0.27507163323782235,
-      "grad_norm": 6.305081367492676,
+      "grad_norm": 6.159996032714844,
       "learning_rate": 8.966766701456177e-05,
-      "loss": 0.8395,
+      "loss": 0.8327,
       "step": 12
     },
     {
       "epoch": 0.2979942693409742,
-      "grad_norm": 5.141690731048584,
+      "grad_norm": 5.404626369476318,
       "learning_rate": 8.759199037394887e-05,
-      "loss": 0.6147,
+      "loss": 0.6177,
       "step": 13
     },
     {
       "epoch": 0.3209169054441261,
-      "grad_norm": 5.286665916442871,
+      "grad_norm": 5.267985820770264,
       "learning_rate": 8.535533905932738e-05,
-      "loss": 0.5611,
+      "loss": 0.5642,
       "step": 14
     },
     {
       "epoch": 0.3438395415472779,
-      "grad_norm": 3.641829013824463,
+      "grad_norm": 3.348625898361206,
       "learning_rate": 8.296729075500344e-05,
-      "loss": 0.5436,
+      "loss": 0.535,
       "step": 15
     },
     {
       "epoch": 0.3667621776504298,
-      "grad_norm": 3.472712516784668,
+      "grad_norm": 3.2940914630889893,
       "learning_rate": 8.043807145043604e-05,
-      "loss": 0.5924,
+      "loss": 0.5919,
       "step": 16
     },
     {
       "epoch": 0.38968481375358166,
-      "grad_norm": 2.7295916080474854,
+      "grad_norm": 2.4778571128845215,
       "learning_rate": 7.777851165098012e-05,
-      "loss": 0.5254,
+      "loss": 0.5233,
       "step": 17
     },
     {
       "epoch": 0.41260744985673353,
-      "grad_norm": 3.2518417835235596,
+      "grad_norm": 3.1166768074035645,
       "learning_rate": 7.500000000000001e-05,
-      "loss": 0.557,
+      "loss": 0.5563,
       "step": 18
     },
     {
       "epoch": 0.4355300859598854,
-      "grad_norm": 3.27976393699646,
+      "grad_norm": 3.1437597274780273,
       "learning_rate": 7.211443451095007e-05,
-      "loss": 0.5605,
+      "loss": 0.559,
       "step": 19
     },
     {
       "epoch": 0.4584527220630373,
-      "grad_norm": 2.3857486248016357,
+      "grad_norm": 2.291743278503418,
       "learning_rate": 6.91341716182545e-05,
-      "loss": 0.5687,
+      "loss": 0.5693,
       "step": 20
     },
     {
       "epoch": 0.4813753581661891,
-      "grad_norm": 2.0792624950408936,
+      "grad_norm": 2.1247966289520264,
       "learning_rate": 6.607197326515808e-05,
-      "loss": 0.4818,
+      "loss": 0.4814,
       "step": 21
     },
     {
       "epoch": 0.504297994269341,
-      "grad_norm": 4.030040740966797,
+      "grad_norm": 3.7222495079040527,
       "learning_rate": 6.294095225512603e-05,
-      "loss": 0.8559,
+      "loss": 0.8573,
       "step": 22
     },
     {
       "epoch": 0.5272206303724928,
-      "grad_norm": 2.7893178462982178,
+      "grad_norm": 3.1192123889923096,
       "learning_rate": 5.9754516100806423e-05,
-      "loss": 0.6796,
+      "loss": 0.6928,
       "step": 23
     },
     {
       "epoch": 0.5501432664756447,
-      "grad_norm": 5.384159564971924,
+      "grad_norm": 5.665029525756836,
       "learning_rate": 5.6526309611002594e-05,
-      "loss": 0.5617,
+      "loss": 0.5776,
       "step": 24
     },
     {
       "epoch": 0.5730659025787965,
-      "grad_norm": 2.352954626083374,
+      "grad_norm": 2.4942095279693604,
       "learning_rate": 5.327015646150716e-05,
-      "loss": 0.4691,
+      "loss": 0.4731,
       "step": 25
     },
     {
       "epoch": 0.5730659025787965,
-      "eval_loss": 0.5326946973800659,
-      "eval_runtime": 2.6433,
-      "eval_samples_per_second": 111.225,
-      "eval_steps_per_second": 13.998,
+      "eval_loss": 0.5357323288917542,
+      "eval_runtime": 2.601,
+      "eval_samples_per_second": 113.035,
+      "eval_steps_per_second": 14.226,
       "step": 25
     },
     {
       "epoch": 0.5959885386819485,
-      "grad_norm": 2.685312032699585,
+      "grad_norm": 2.6250343322753906,
       "learning_rate": 5e-05,
-      "loss": 0.5259,
+      "loss": 0.5303,
       "step": 26
     },
     {
       "epoch": 0.6189111747851003,
-      "grad_norm": 2.0061182975769043,
+      "grad_norm": 1.9742813110351562,
       "learning_rate": 4.6729843538492847e-05,
-      "loss": 0.4453,
+      "loss": 0.4454,
       "step": 27
     },
     {
       "epoch": 0.6418338108882522,
-      "grad_norm": 2.389681816101074,
+      "grad_norm": 2.37349534034729,
       "learning_rate": 4.347369038899744e-05,
-      "loss": 0.5067,
+      "loss": 0.5133,
       "step": 28
     },
     {
       "epoch": 0.664756446991404,
-      "grad_norm": 1.3938570022583008,
+      "grad_norm": 1.4418854713439941,
       "learning_rate": 4.0245483899193595e-05,
-      "loss": 0.4948,
+      "loss": 0.4993,
       "step": 29
     },
     {
       "epoch": 0.6876790830945558,
-      "grad_norm": 2.048872709274292,
+      "grad_norm": 1.9568227529525757,
       "learning_rate": 3.705904774487396e-05,
-      "loss": 0.521,
+      "loss": 0.5203,
       "step": 30
     },
     {
       "epoch": 0.7106017191977078,
-      "grad_norm": 2.7085561752319336,
+      "grad_norm": 2.596329689025879,
       "learning_rate": 3.392802673484193e-05,
-      "loss": 0.5026,
+      "loss": 0.4965,
       "step": 31
     },
     {
       "epoch": 0.7335243553008596,
-      "grad_norm": 2.864086866378784,
+      "grad_norm": 3.1707613468170166,
       "learning_rate": 3.086582838174551e-05,
-      "loss": 0.5095,
+      "loss": 0.5212,
       "step": 32
     },
     {
       "epoch": 0.7564469914040115,
-      "grad_norm": 3.286463975906372,
+      "grad_norm": 3.171983003616333,
       "learning_rate": 2.7885565489049946e-05,
-      "loss": 0.9472,
+      "loss": 0.9544,
       "step": 33
     },
     {
       "epoch": 0.7793696275071633,
-      "grad_norm": 2.851412057876587,
+      "grad_norm": 2.745166301727295,
       "learning_rate": 2.500000000000001e-05,
       "loss": 0.6693,
       "step": 34
     },
     {
       "epoch": 0.8022922636103151,
-      "grad_norm": 3.426495313644409,
+      "grad_norm": 3.527172803878784,
       "learning_rate": 2.2221488349019903e-05,
-      "loss": 0.6036,
+      "loss": 0.6108,
       "step": 35
     },
     {
       "epoch": 0.8252148997134671,
-      "grad_norm": 1.8203542232513428,
+      "grad_norm": 1.7733834981918335,
       "learning_rate": 1.9561928549563968e-05,
-      "loss": 0.4371,
+      "loss": 0.4456,
       "step": 36
     },
     {
       "epoch": 0.8481375358166189,
-      "grad_norm": 1.9482104778289795,
+      "grad_norm": 1.8841828107833862,
       "learning_rate": 1.703270924499656e-05,
-      "loss": 0.4431,
+      "loss": 0.4501,
       "step": 37
     },
     {
       "epoch": 0.8710601719197708,
-      "grad_norm": 1.7652301788330078,
+      "grad_norm": 1.7205829620361328,
       "learning_rate": 1.4644660940672627e-05,
-      "loss": 0.495,
+      "loss": 0.4989,
       "step": 38
     },
     {
       "epoch": 0.8939828080229226,
-      "grad_norm": 1.864548683166504,
+      "grad_norm": 1.8143179416656494,
       "learning_rate": 1.2408009626051137e-05,
-      "loss": 0.4491,
+      "loss": 0.4512,
       "step": 39
     },
     {
       "epoch": 0.9169054441260746,
-      "grad_norm": 1.3835070133209229,
+      "grad_norm": 1.4070065021514893,
       "learning_rate": 1.0332332985438248e-05,
-      "loss": 0.5288,
+      "loss": 0.534,
       "step": 40
     },
     {
       "epoch": 0.9398280802292264,
-      "grad_norm": 1.6401492357254028,
+      "grad_norm": 1.5566071271896362,
       "learning_rate": 8.426519384872733e-06,
-      "loss": 0.5039,
+      "loss": 0.5068,
       "step": 41
     },
     {
       "epoch": 0.9627507163323782,
-      "grad_norm": 1.3801206350326538,
+      "grad_norm": 1.2321510314941406,
       "learning_rate": 6.698729810778065e-06,
-      "loss": 0.5268,
+      "loss": 0.5257,
       "step": 42
     },
     {
       "epoch": 0.9856733524355301,
-      "grad_norm": 1.2480432987213135,
+      "grad_norm": 1.0602449178695679,
       "learning_rate": 5.156362923365588e-06,
-      "loss": 0.4542,
+      "loss": 0.453,
       "step": 43
     },
     {
       "epoch": 1.008595988538682,
-      "grad_norm": 4.597485542297363,
+      "grad_norm": 4.39303731918335,
       "learning_rate": 3.8060233744356633e-06,
-      "loss": 1.1249,
+      "loss": 1.1265,
       "step": 44
     },
     {
       "epoch": 1.0315186246418337,
-      "grad_norm": 1.7384655475616455,
+      "grad_norm": 1.6075620651245117,
       "learning_rate": 2.653493525244721e-06,
-      "loss": 0.6526,
+      "loss": 0.655,
       "step": 45
     },
     {
       "epoch": 1.0544412607449858,
-      "grad_norm": 0.9391457438468933,
+      "grad_norm": 0.8850023150444031,
       "learning_rate": 1.70370868554659e-06,
-      "loss": 0.4905,
+      "loss": 0.4917,
       "step": 46
     },
     {
       "epoch": 1.0773638968481376,
-      "grad_norm": 1.4423028230667114,
+      "grad_norm": 1.4221233129501343,
       "learning_rate": 9.607359798384785e-07,
-      "loss": 0.4868,
+      "loss": 0.4885,
       "step": 47
     },
     {
       "epoch": 1.1002865329512894,
-      "grad_norm": 1.5640639066696167,
+      "grad_norm": 1.5302201509475708,
       "learning_rate": 4.277569313094809e-07,
-      "loss": 0.4497,
+      "loss": 0.4518,
       "step": 48
     },
     {
       "epoch": 1.1232091690544412,
-      "grad_norm": 1.555716633796692,
+      "grad_norm": 1.4984784126281738,
       "learning_rate": 1.0705383806982606e-07,
-      "loss": 0.4147,
+      "loss": 0.4188,
       "step": 49
     },
     {
       "epoch": 1.146131805157593,
-      "grad_norm": 2.531493663787842,
+      "grad_norm": 2.7869534492492676,
       "learning_rate": 0.0,
-      "loss": 0.379,
+      "loss": 0.3817,
       "step": 50
     },
     {
       "epoch": 1.146131805157593,
-      "eval_loss": 0.4893987774848938,
-      "eval_runtime": 2.6473,
-      "eval_samples_per_second": 111.056,
-      "eval_steps_per_second": 13.976,
+      "eval_loss": 0.4894934296607971,
+      "eval_runtime": 2.6019,
+      "eval_samples_per_second": 112.993,
+      "eval_steps_per_second": 14.22,
       "step": 50
     }
   ],

last-checkpoint/training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:595f13f9b541b1821e9d98d762bfaf4708c03c003f626eee9ea3aebae6357ba6
+oid sha256:1d76f26d77bea81bf37439ed33d8fcf7a5fb5304d81c23c42d88d1d67e1cebef
 size 6776