Training in progress, step 300, checkpoint

checkpoint-300/adapter_config.json

@@ -24,12 +24,12 @@
   "revision": null,
   "target_modules": [
     "gate_proj",
-    "q_proj",
+    "v_proj",
     "k_proj",
-    "up_proj",
+    "q_proj",
     "o_proj",
-    "down_proj",
-    "v_proj"
+    "up_proj",
+    "down_proj"
   ],
   "task_type": "CAUSAL_LM",
   "use_dora": false,

checkpoint-300/adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:4e620d5341c5f33452674a2cf5db54e7621d0d9ac6df59d54fb764032cc74308
+oid sha256:181e4c0b26d165813ae306c3fc797adfd7f010b04e42f39f677759505aa04a82
 size 216151256

checkpoint-300/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5c66713f966db3ce2979b04d4a23e75efe4df94fee22faf6069dec595e9d5716
+oid sha256:aa4039faaa76388412a2ebaf73d7acf3a15d82fd851184f4bf6aedfd251b0c75
 size 432640054

checkpoint-300/rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:cbda1e5a654d2530c53207896f8d4286870323a78a5385a32747e04d91df3aa2
+oid sha256:ca45f95bcdfa3dc52d93139cc196214d00e007284ef333414453bd2343c3d8ea
 size 14244

checkpoint-300/trainer_state.json

@@ -2,7 +2,7 @@
   "best_metric": null,
   "best_model_checkpoint": null,
   "epoch": 0.379746835443038,
-  "eval_steps": 50,
+  "eval_steps": 10,
   "global_step": 300,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
@@ -10,260 +10,452 @@
   "log_history": [
     {
       "epoch": 0.012658227848101266,
-      "grad_norm": 16.027029037475586,
+      "grad_norm": 12.934767723083496,
       "learning_rate": 4.9789029535864986e-05,
-      "loss": 2.6925,
+      "loss": 2.6869,
+      "step": 10
+    },
+    {
+      "epoch": 0.012658227848101266,
+      "eval_loss": 1.651185393333435,
+      "eval_runtime": 11.831,
+      "eval_samples_per_second": 40.571,
+      "eval_steps_per_second": 2.536,
       "step": 10
     },
     {
       "epoch": 0.02531645569620253,
-      "grad_norm": 1.0948777198791504,
+      "grad_norm": 1.104798674583435,
       "learning_rate": 4.957805907172996e-05,
-      "loss": 1.386,
+      "loss": 1.3694,
+      "step": 20
+    },
+    {
+      "epoch": 0.02531645569620253,
+      "eval_loss": 1.2200205326080322,
+      "eval_runtime": 11.8928,
+      "eval_samples_per_second": 40.361,
+      "eval_steps_per_second": 2.523,
       "step": 20
     },
     {
       "epoch": 0.0379746835443038,
-      "grad_norm": 1.1341983079910278,
+      "grad_norm": 1.1069142818450928,
       "learning_rate": 4.936708860759494e-05,
-      "loss": 1.108,
+      "loss": 1.1029,
+      "step": 30
+    },
+    {
+      "epoch": 0.0379746835443038,
+      "eval_loss": 1.0691736936569214,
+      "eval_runtime": 11.9127,
+      "eval_samples_per_second": 40.293,
+      "eval_steps_per_second": 2.518,
       "step": 30
     },
     {
       "epoch": 0.05063291139240506,
-      "grad_norm": 1.152009129524231,
+      "grad_norm": 1.1594161987304688,
       "learning_rate": 4.9156118143459915e-05,
-      "loss": 0.9436,
+      "loss": 0.9395,
+      "step": 40
+    },
+    {
+      "epoch": 0.05063291139240506,
+      "eval_loss": 0.9162012934684753,
+      "eval_runtime": 11.9373,
+      "eval_samples_per_second": 40.21,
+      "eval_steps_per_second": 2.513,
       "step": 40
     },
     {
       "epoch": 0.06329113924050633,
-      "grad_norm": 1.0990614891052246,
+      "grad_norm": 1.1133538484573364,
       "learning_rate": 4.89451476793249e-05,
-      "loss": 0.7499,
+      "loss": 0.7489,
       "step": 50
     },
     {
       "epoch": 0.06329113924050633,
-      "eval_loss": 0.7157873511314392,
-      "eval_runtime": 12.1594,
-      "eval_samples_per_second": 39.476,
-      "eval_steps_per_second": 2.467,
+      "eval_loss": 0.713701605796814,
+      "eval_runtime": 11.9661,
+      "eval_samples_per_second": 40.113,
+      "eval_steps_per_second": 2.507,
       "step": 50
     },
     {
       "epoch": 0.0759493670886076,
-      "grad_norm": 1.0284477472305298,
+      "grad_norm": 1.0406183004379272,
       "learning_rate": 4.8734177215189874e-05,
-      "loss": 0.6095,
+      "loss": 0.6096,
+      "step": 60
+    },
+    {
+      "epoch": 0.0759493670886076,
+      "eval_loss": 0.6309535503387451,
+      "eval_runtime": 11.9895,
+      "eval_samples_per_second": 40.035,
+      "eval_steps_per_second": 2.502,
       "step": 60
     },
     {
       "epoch": 0.08860759493670886,
-      "grad_norm": 0.8609589338302612,
+      "grad_norm": 0.8599340915679932,
       "learning_rate": 4.852320675105486e-05,
-      "loss": 0.5355,
+      "loss": 0.5357,
+      "step": 70
+    },
+    {
+      "epoch": 0.08860759493670886,
+      "eval_loss": 0.6159886717796326,
+      "eval_runtime": 12.0107,
+      "eval_samples_per_second": 39.965,
+      "eval_steps_per_second": 2.498,
       "step": 70
     },
     {
       "epoch": 0.10126582278481013,
-      "grad_norm": 0.9162376523017883,
+      "grad_norm": 0.9128267168998718,
       "learning_rate": 4.8312236286919834e-05,
-      "loss": 0.5705,
+      "loss": 0.5703,
+      "step": 80
+    },
+    {
+      "epoch": 0.10126582278481013,
+      "eval_loss": 0.5933937430381775,
+      "eval_runtime": 11.9716,
+      "eval_samples_per_second": 40.095,
+      "eval_steps_per_second": 2.506,
       "step": 80
     },
     {
       "epoch": 0.11392405063291139,
-      "grad_norm": 0.9415847659111023,
+      "grad_norm": 0.9396541118621826,
       "learning_rate": 4.810126582278481e-05,
-      "loss": 0.5449,
+      "loss": 0.5445,
+      "step": 90
+    },
+    {
+      "epoch": 0.11392405063291139,
+      "eval_loss": 0.5727818608283997,
+      "eval_runtime": 11.9685,
+      "eval_samples_per_second": 40.105,
+      "eval_steps_per_second": 2.507,
       "step": 90
     },
     {
       "epoch": 0.12658227848101267,
-      "grad_norm": 0.8756884336471558,
+      "grad_norm": 0.8805290460586548,
       "learning_rate": 4.789029535864979e-05,
-      "loss": 0.5157,
+      "loss": 0.5151,
       "step": 100
     },
     {
       "epoch": 0.12658227848101267,
-      "eval_loss": 0.563517153263092,
-      "eval_runtime": 12.1236,
-      "eval_samples_per_second": 39.592,
-      "eval_steps_per_second": 2.475,
+      "eval_loss": 0.5640087127685547,
+      "eval_runtime": 11.9824,
+      "eval_samples_per_second": 40.059,
+      "eval_steps_per_second": 2.504,
       "step": 100
     },
     {
       "epoch": 0.13924050632911392,
-      "grad_norm": 0.8702118396759033,
+      "grad_norm": 0.8829126954078674,
       "learning_rate": 4.767932489451477e-05,
-      "loss": 0.53,
+      "loss": 0.5301,
+      "step": 110
+    },
+    {
+      "epoch": 0.13924050632911392,
+      "eval_loss": 0.5558986067771912,
+      "eval_runtime": 11.9942,
+      "eval_samples_per_second": 40.019,
+      "eval_steps_per_second": 2.501,
       "step": 110
     },
     {
       "epoch": 0.1518987341772152,
-      "grad_norm": 0.8843992352485657,
+      "grad_norm": 0.8889341950416565,
       "learning_rate": 4.7468354430379746e-05,
-      "loss": 0.4923,
+      "loss": 0.4929,
+      "step": 120
+    },
+    {
+      "epoch": 0.1518987341772152,
+      "eval_loss": 0.5422877073287964,
+      "eval_runtime": 11.9624,
+      "eval_samples_per_second": 40.126,
+      "eval_steps_per_second": 2.508,
       "step": 120
     },
     {
       "epoch": 0.16455696202531644,
-      "grad_norm": 0.8294386863708496,
+      "grad_norm": 0.8428446054458618,
       "learning_rate": 4.725738396624473e-05,
-      "loss": 0.4936,
+      "loss": 0.4932,
+      "step": 130
+    },
+    {
+      "epoch": 0.16455696202531644,
+      "eval_loss": 0.5370256900787354,
+      "eval_runtime": 11.9786,
+      "eval_samples_per_second": 40.071,
+      "eval_steps_per_second": 2.504,
       "step": 130
     },
     {
       "epoch": 0.17721518987341772,
-      "grad_norm": 0.8978216648101807,
+      "grad_norm": 0.8985374569892883,
       "learning_rate": 4.704641350210971e-05,
-      "loss": 0.4581,
+      "loss": 0.4589,
+      "step": 140
+    },
+    {
+      "epoch": 0.17721518987341772,
+      "eval_loss": 0.5208094716072083,
+      "eval_runtime": 11.9711,
+      "eval_samples_per_second": 40.097,
+      "eval_steps_per_second": 2.506,
       "step": 140
     },
     {
       "epoch": 0.189873417721519,
-      "grad_norm": 0.8757727742195129,
+      "grad_norm": 0.8704663515090942,
       "learning_rate": 4.683544303797468e-05,
-      "loss": 0.4571,
+      "loss": 0.4585,
       "step": 150
     },
     {
       "epoch": 0.189873417721519,
-      "eval_loss": 0.5096740126609802,
-      "eval_runtime": 12.0892,
-      "eval_samples_per_second": 39.705,
-      "eval_steps_per_second": 2.482,
+      "eval_loss": 0.5105039477348328,
+      "eval_runtime": 11.9751,
+      "eval_samples_per_second": 40.083,
+      "eval_steps_per_second": 2.505,
       "step": 150
     },
     {
       "epoch": 0.20253164556962025,
-      "grad_norm": 0.8959233164787292,
+      "grad_norm": 0.8930565714836121,
       "learning_rate": 4.6624472573839666e-05,
-      "loss": 0.4429,
+      "loss": 0.4438,
+      "step": 160
+    },
+    {
+      "epoch": 0.20253164556962025,
+      "eval_loss": 0.4988265931606293,
+      "eval_runtime": 11.9819,
+      "eval_samples_per_second": 40.06,
+      "eval_steps_per_second": 2.504,
       "step": 160
     },
     {
       "epoch": 0.21518987341772153,
-      "grad_norm": 0.9160757660865784,
+      "grad_norm": 0.9236075282096863,
       "learning_rate": 4.641350210970464e-05,
-      "loss": 0.4167,
+      "loss": 0.4171,
+      "step": 170
+    },
+    {
+      "epoch": 0.21518987341772153,
+      "eval_loss": 0.4941176176071167,
+      "eval_runtime": 11.9797,
+      "eval_samples_per_second": 40.068,
+      "eval_steps_per_second": 2.504,
       "step": 170
     },
     {
       "epoch": 0.22784810126582278,
-      "grad_norm": 0.8140855431556702,
+      "grad_norm": 0.8057528138160706,
       "learning_rate": 4.6202531645569625e-05,
-      "loss": 0.4249,
+      "loss": 0.4244,
+      "step": 180
+    },
+    {
+      "epoch": 0.22784810126582278,
+      "eval_loss": 0.489634245634079,
+      "eval_runtime": 11.9664,
+      "eval_samples_per_second": 40.112,
+      "eval_steps_per_second": 2.507,
       "step": 180
     },
     {
       "epoch": 0.24050632911392406,
-      "grad_norm": 0.8790073990821838,
+      "grad_norm": 0.8802728056907654,
       "learning_rate": 4.59915611814346e-05,
-      "loss": 0.4198,
+      "loss": 0.4196,
+      "step": 190
+    },
+    {
+      "epoch": 0.24050632911392406,
+      "eval_loss": 0.48033541440963745,
+      "eval_runtime": 11.9754,
+      "eval_samples_per_second": 40.082,
+      "eval_steps_per_second": 2.505,
       "step": 190
     },
     {
       "epoch": 0.25316455696202533,
-      "grad_norm": 0.8366842269897461,
+      "grad_norm": 0.8316253423690796,
       "learning_rate": 4.5780590717299585e-05,
-      "loss": 0.4148,
+      "loss": 0.4144,
       "step": 200
     },
     {
       "epoch": 0.25316455696202533,
-      "eval_loss": 0.47525277733802795,
-      "eval_runtime": 12.0592,
-      "eval_samples_per_second": 39.803,
-      "eval_steps_per_second": 2.488,
+      "eval_loss": 0.4757111966609955,
+      "eval_runtime": 11.9744,
+      "eval_samples_per_second": 40.086,
+      "eval_steps_per_second": 2.505,
       "step": 200
     },
     {
       "epoch": 0.26582278481012656,
-      "grad_norm": 0.8733569383621216,
+      "grad_norm": 0.8675290942192078,
       "learning_rate": 4.556962025316456e-05,
-      "loss": 0.3865,
+      "loss": 0.3866,
+      "step": 210
+    },
+    {
+      "epoch": 0.26582278481012656,
+      "eval_loss": 0.4751383662223816,
+      "eval_runtime": 11.9976,
+      "eval_samples_per_second": 40.008,
+      "eval_steps_per_second": 2.501,
       "step": 210
     },
     {
       "epoch": 0.27848101265822783,
-      "grad_norm": 0.8695210218429565,
+      "grad_norm": 0.8654202818870544,
       "learning_rate": 4.535864978902954e-05,
-      "loss": 0.3681,
+      "loss": 0.3685,
+      "step": 220
+    },
+    {
+      "epoch": 0.27848101265822783,
+      "eval_loss": 0.47636380791664124,
+      "eval_runtime": 11.988,
+      "eval_samples_per_second": 40.04,
+      "eval_steps_per_second": 2.502,
       "step": 220
     },
     {
       "epoch": 0.2911392405063291,
-      "grad_norm": 0.8353093862533569,
+      "grad_norm": 0.835969090461731,
       "learning_rate": 4.5147679324894514e-05,
-      "loss": 0.4121,
+      "loss": 0.4117,
+      "step": 230
+    },
+    {
+      "epoch": 0.2911392405063291,
+      "eval_loss": 0.4723130762577057,
+      "eval_runtime": 11.9995,
+      "eval_samples_per_second": 40.002,
+      "eval_steps_per_second": 2.5,
       "step": 230
     },
     {
       "epoch": 0.3037974683544304,
-      "grad_norm": 0.9650343656539917,
+      "grad_norm": 0.9585816860198975,
       "learning_rate": 4.49367088607595e-05,
-      "loss": 0.3987,
+      "loss": 0.398,
+      "step": 240
+    },
+    {
+      "epoch": 0.3037974683544304,
+      "eval_loss": 0.472170889377594,
+      "eval_runtime": 12.0019,
+      "eval_samples_per_second": 39.994,
+      "eval_steps_per_second": 2.5,
       "step": 240
     },
     {
       "epoch": 0.31645569620253167,
-      "grad_norm": 0.9851623773574829,
+      "grad_norm": 1.0022097826004028,
       "learning_rate": 4.4725738396624474e-05,
-      "loss": 0.4028,
+      "loss": 0.403,
       "step": 250
     },
     {
       "epoch": 0.31645569620253167,
-      "eval_loss": 0.4751642048358917,
-      "eval_runtime": 12.0237,
-      "eval_samples_per_second": 39.921,
-      "eval_steps_per_second": 2.495,
+      "eval_loss": 0.47611597180366516,
+      "eval_runtime": 12.0087,
+      "eval_samples_per_second": 39.971,
+      "eval_steps_per_second": 2.498,
       "step": 250
     },
     {
       "epoch": 0.3291139240506329,
-      "grad_norm": 0.8186360597610474,
+      "grad_norm": 0.8225414156913757,
       "learning_rate": 4.451476793248946e-05,
-      "loss": 0.405,
+      "loss": 0.4045,
+      "step": 260
+    },
+    {
+      "epoch": 0.3291139240506329,
+      "eval_loss": 0.47163695096969604,
+      "eval_runtime": 11.9976,
+      "eval_samples_per_second": 40.008,
+      "eval_steps_per_second": 2.501,
       "step": 260
     },
     {
       "epoch": 0.34177215189873417,
-      "grad_norm": 0.76863032579422,
+      "grad_norm": 0.765426754951477,
       "learning_rate": 4.430379746835443e-05,
-      "loss": 0.4134,
+      "loss": 0.4147,
+      "step": 270
+    },
+    {
+      "epoch": 0.34177215189873417,
+      "eval_loss": 0.4663979113101959,
+      "eval_runtime": 12.0321,
+      "eval_samples_per_second": 39.893,
+      "eval_steps_per_second": 2.493,
       "step": 270
     },
     {
       "epoch": 0.35443037974683544,
-      "grad_norm": 0.785301685333252,
+      "grad_norm": 0.783598780632019,
       "learning_rate": 4.409282700421941e-05,
-      "loss": 0.3745,
+      "loss": 0.3747,
+      "step": 280
+    },
+    {
+      "epoch": 0.35443037974683544,
+      "eval_loss": 0.4673294723033905,
+      "eval_runtime": 12.0086,
+      "eval_samples_per_second": 39.971,
+      "eval_steps_per_second": 2.498,
       "step": 280
     },
     {
       "epoch": 0.3670886075949367,
-      "grad_norm": 0.8613535165786743,
+      "grad_norm": 0.830932080745697,
       "learning_rate": 4.388185654008439e-05,
-      "loss": 0.3804,
+      "loss": 0.3803,
+      "step": 290
+    },
+    {
+      "epoch": 0.3670886075949367,
+      "eval_loss": 0.4643842875957489,
+      "eval_runtime": 12.0174,
+      "eval_samples_per_second": 39.942,
+      "eval_steps_per_second": 2.496,
       "step": 290
     },
     {
       "epoch": 0.379746835443038,
-      "grad_norm": 0.8152708411216736,
+      "grad_norm": 0.8144668936729431,
       "learning_rate": 4.367088607594937e-05,
-      "loss": 0.3757,
+      "loss": 0.3756,
       "step": 300
     },
     {
       "epoch": 0.379746835443038,
-      "eval_loss": 0.4623233675956726,
-      "eval_runtime": 12.0099,
-      "eval_samples_per_second": 39.967,
-      "eval_steps_per_second": 2.498,
+      "eval_loss": 0.462455689907074,
+      "eval_runtime": 12.0337,
+      "eval_samples_per_second": 39.888,
+      "eval_steps_per_second": 2.493,
       "step": 300
     }
   ],
@@ -271,7 +463,7 @@
   "max_steps": 2370,
   "num_input_tokens_seen": 0,
   "num_train_epochs": 3,
-  "save_steps": 100,
+  "save_steps": 10,
   "stateful_callbacks": {
     "TrainerControl": {
       "args": {

checkpoint-300/training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b8ef5fa4aad3a350c14df025074931ad8a003d4b851f4886f3b2f66ae6653e4b
+oid sha256:f1e7aae8b855413d55586dd498c7d7d805796f0c02067ce9d8ccb1ef37f72d29
 size 5880