Model save

all_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 1.0,
-    "train_loss": 0.34950463063008075,
-    "train_runtime": 5351.3328,
-    "train_samples": 47302,
-    "train_samples_per_second": 8.839,
-    "train_steps_per_second": 0.035
+    "train_loss": 0.6121151776493344,
+    "train_runtime": 6995.9878,
+    "train_samples": 61134,
+    "train_samples_per_second": 8.738,
+    "train_steps_per_second": 0.034
 }

model-00001-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:56367e7dbb6f5270c0dcc95ad6939910e996a1c1fb6c297aaeb617baffa003d6
+oid sha256:f4cfba4b2bccd53078b6689bd3a353c0c6b8f8f63f7dd05990329ce2a4cd3dc2
 size 4943162336

model-00002-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:4e774046ad280291cfd6df4053eff4b493adb31d069fda0fe2e8d1ae0c5daaad
+oid sha256:6caee221211932c0d895c01ccf79dca18c9e4d8f596c8c6e5c470beb9f1daa58
 size 4999819336

model-00003-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:bab3b2f66a403503efbeb6de7df5e24fff2ee65a874830b7a9e4bd93f0bc9efa
+oid sha256:1d173bc5999339b83eaf9e08e87fad34084acdec72038e115d114762c5c0c29b
 size 4540516344

runs/Jun21_02-39-47_n136-129-074/events.out.tfevents.1718908857.n136-129-074.3997378.0

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:fc1e79d2e7f05dfba079d98f950087e64c5efc6bb21c0175df947f3d116fddd2
-size 19189
+oid sha256:4de53dba9eadb4306d14a928a1deee7e5bc77f0981b2d86995978293214f379f
+size 21607

train_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 1.0,
-    "train_loss": 0.34950463063008075,
-    "train_runtime": 5351.3328,
-    "train_samples": 47302,
-    "train_samples_per_second": 8.839,
-    "train_steps_per_second": 0.035
+    "train_loss": 0.6121151776493344,
+    "train_runtime": 6995.9878,
+    "train_samples": 61134,
+    "train_samples_per_second": 8.738,
+    "train_steps_per_second": 0.034
 }

trainer_state.json

@@ -3,20 +3,20 @@
   "best_model_checkpoint": null,
   "epoch": 1.0,
   "eval_steps": 500,
-  "global_step": 185,
+  "global_step": 239,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
-      "epoch": 0.01,
-      "grad_norm": 346.69678421347976,
-      "learning_rate": 2.6315789473684208e-08,
-      "logits/chosen": -0.1266070306301117,
-      "logits/rejected": 0.7204304933547974,
-      "logps/chosen": -319.01666259765625,
-      "logps/rejected": -252.47039794921875,
-      "loss": 0.6944,
+      "epoch": 0.0,
+      "grad_norm": 1228.6970114702256,
+      "learning_rate": 2.083333333333333e-08,
+      "logits/chosen": -0.5983926653862,
+      "logits/rejected": -0.3142164349555969,
+      "logps/chosen": -366.9183349609375,
+      "logps/rejected": -437.1544189453125,
+      "loss": 0.6931,
       "rewards/accuracies": 0.0,
       "rewards/chosen": 0.0,
       "rewards/margins": 0.0,
@@ -24,287 +24,362 @@
       "step": 1
     },
     {
-      "epoch": 0.05,
-      "grad_norm": 303.99650058962374,
-      "learning_rate": 2.631578947368421e-07,
-      "logits/chosen": -0.3861178159713745,
-      "logits/rejected": 0.3360586166381836,
-      "logps/chosen": -266.5268249511719,
-      "logps/rejected": -224.07962036132812,
-      "loss": 0.6646,
-      "rewards/accuracies": 0.5590277910232544,
-      "rewards/chosen": -0.06958804279565811,
-      "rewards/margins": 0.056223273277282715,
-      "rewards/rejected": -0.12581130862236023,
+      "epoch": 0.04,
+      "grad_norm": 550.4800163195696,
+      "learning_rate": 2.0833333333333333e-07,
+      "logits/chosen": -0.3194640576839447,
+      "logits/rejected": 0.544389009475708,
+      "logps/chosen": -343.9505920410156,
+      "logps/rejected": -398.5333557128906,
+      "loss": 0.6237,
+      "rewards/accuracies": 0.5520833134651184,
+      "rewards/chosen": -0.3209317922592163,
+      "rewards/margins": 0.3546760380268097,
+      "rewards/rejected": -0.6756078600883484,
       "step": 10
     },
     {
-      "epoch": 0.11,
-      "grad_norm": 178.58708149032248,
-      "learning_rate": 4.999552306674344e-07,
-      "logits/chosen": -0.3403215706348419,
-      "logits/rejected": 0.6743763089179993,
-      "logps/chosen": -285.2884826660156,
-      "logps/rejected": -244.68014526367188,
-      "loss": 0.4493,
-      "rewards/accuracies": 0.8187500238418579,
-      "rewards/chosen": -0.12177922576665878,
-      "rewards/margins": 2.0803089141845703,
-      "rewards/rejected": -2.202087879180908,
+      "epoch": 0.08,
+      "grad_norm": 703.9089332225451,
+      "learning_rate": 4.1666666666666667e-07,
+      "logits/chosen": -0.16047191619873047,
+      "logits/rejected": 0.8960522413253784,
+      "logps/chosen": -363.82220458984375,
+      "logps/rejected": -391.59588623046875,
+      "loss": 0.7373,
+      "rewards/accuracies": 0.784375011920929,
+      "rewards/chosen": -4.046866416931152,
+      "rewards/margins": 3.9459426403045654,
+      "rewards/rejected": -7.992809295654297,
       "step": 20
     },
     {
-      "epoch": 0.16,
-      "grad_norm": 189.32496944078557,
-      "learning_rate": 4.946022852363932e-07,
-      "logits/chosen": -0.46592849493026733,
-      "logits/rejected": 0.38956326246261597,
-      "logps/chosen": -267.5489501953125,
-      "logps/rejected": -250.01095581054688,
-      "loss": 0.4321,
+      "epoch": 0.13,
+      "grad_norm": 552.799882218425,
+      "learning_rate": 4.990398100856366e-07,
+      "logits/chosen": -0.1726907640695572,
+      "logits/rejected": 0.5421887636184692,
+      "logps/chosen": -354.1240234375,
+      "logps/rejected": -437.666259765625,
+      "loss": 0.7316,
       "rewards/accuracies": 0.8062499761581421,
-      "rewards/chosen": -0.5830897092819214,
-      "rewards/margins": 4.184449672698975,
-      "rewards/rejected": -4.7675395011901855,
+      "rewards/chosen": -2.8292102813720703,
+      "rewards/margins": 4.830966949462891,
+      "rewards/rejected": -7.660176753997803,
       "step": 30
     },
     {
-      "epoch": 0.22,
-      "grad_norm": 202.83189119672568,
-      "learning_rate": 4.805146507594034e-07,
-      "logits/chosen": -0.5013719797134399,
-      "logits/rejected": 0.4470803141593933,
-      "logps/chosen": -267.41217041015625,
-      "logps/rejected": -244.5727081298828,
-      "loss": 0.398,
-      "rewards/accuracies": 0.8531249761581421,
-      "rewards/chosen": -1.7180732488632202,
-      "rewards/margins": 5.005571365356445,
-      "rewards/rejected": -6.723645210266113,
+      "epoch": 0.17,
+      "grad_norm": 533.313986686025,
+      "learning_rate": 4.931986719649298e-07,
+      "logits/chosen": -0.4836675524711609,
+      "logits/rejected": 0.30907225608825684,
+      "logps/chosen": -369.8108215332031,
+      "logps/rejected": -389.89508056640625,
+      "loss": 0.6541,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -1.1337063312530518,
+      "rewards/margins": 3.0231781005859375,
+      "rewards/rejected": -4.15688419342041,
       "step": 40
     },
     {
-      "epoch": 0.27,
-      "grad_norm": 206.38765246393052,
-      "learning_rate": 4.581953932909403e-07,
-      "logits/chosen": -0.67442786693573,
-      "logits/rejected": 0.16651669144630432,
-      "logps/chosen": -286.7380676269531,
-      "logps/rejected": -263.77264404296875,
-      "loss": 0.3431,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": -0.6169568300247192,
-      "rewards/margins": 4.902881622314453,
-      "rewards/rejected": -5.519838333129883,
+      "epoch": 0.21,
+      "grad_norm": 556.3373380132485,
+      "learning_rate": 4.821741763807186e-07,
+      "logits/chosen": -0.36255133152008057,
+      "logits/rejected": 0.5106185674667358,
+      "logps/chosen": -335.77191162109375,
+      "logps/rejected": -392.73406982421875,
+      "loss": 0.593,
+      "rewards/accuracies": 0.8218749761581421,
+      "rewards/chosen": -0.6605129837989807,
+      "rewards/margins": 3.997671604156494,
+      "rewards/rejected": -4.658184051513672,
       "step": 50
     },
     {
-      "epoch": 0.32,
-      "grad_norm": 167.85166107715966,
-      "learning_rate": 4.284415281717847e-07,
-      "logits/chosen": -0.5345869064331055,
-      "logits/rejected": 0.39840853214263916,
-      "logps/chosen": -283.1356201171875,
-      "logps/rejected": -266.4700927734375,
-      "loss": 0.3141,
-      "rewards/accuracies": 0.878125011920929,
-      "rewards/chosen": -0.9925642013549805,
-      "rewards/margins": 4.798120975494385,
-      "rewards/rejected": -5.790685176849365,
+      "epoch": 0.25,
+      "grad_norm": 587.3266847652458,
+      "learning_rate": 4.662012913161997e-07,
+      "logits/chosen": -0.48477110266685486,
+      "logits/rejected": 0.4101050794124603,
+      "logps/chosen": -372.4800109863281,
+      "logps/rejected": -392.51702880859375,
+      "loss": 0.665,
+      "rewards/accuracies": 0.7593749761581421,
+      "rewards/chosen": -1.5650008916854858,
+      "rewards/margins": 3.357562303543091,
+      "rewards/rejected": -4.922562599182129,
       "step": 60
     },
     {
-      "epoch": 0.38,
-      "grad_norm": 207.4923528290103,
-      "learning_rate": 3.923155588020165e-07,
-      "logits/chosen": -0.4916340410709381,
-      "logits/rejected": 0.499605655670166,
-      "logps/chosen": -254.4178009033203,
-      "logps/rejected": -236.2290802001953,
-      "loss": 0.3244,
-      "rewards/accuracies": 0.859375,
-      "rewards/chosen": -0.9756989479064941,
-      "rewards/margins": 4.529528617858887,
-      "rewards/rejected": -5.505227565765381,
+      "epoch": 0.29,
+      "grad_norm": 466.5579056975509,
+      "learning_rate": 4.456204510851956e-07,
+      "logits/chosen": -0.5015081167221069,
+      "logits/rejected": 0.16009679436683655,
+      "logps/chosen": -373.043212890625,
+      "logps/rejected": -423.36041259765625,
+      "loss": 0.5988,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -2.6556434631347656,
+      "rewards/margins": 3.99314546585083,
+      "rewards/rejected": -6.648789405822754,
       "step": 70
     },
     {
-      "epoch": 0.43,
-      "grad_norm": 206.7575642359706,
-      "learning_rate": 3.511075348989692e-07,
-      "logits/chosen": -0.29824167490005493,
-      "logits/rejected": 0.5488064885139465,
-      "logps/chosen": -277.8660888671875,
-      "logps/rejected": -255.5980224609375,
-      "loss": 0.3538,
-      "rewards/accuracies": 0.8656250238418579,
-      "rewards/chosen": -1.1310797929763794,
-      "rewards/margins": 4.543191432952881,
-      "rewards/rejected": -5.674271583557129,
+      "epoch": 0.33,
+      "grad_norm": 569.6628049875762,
+      "learning_rate": 4.2087030056579986e-07,
+      "logits/chosen": -0.6794461607933044,
+      "logits/rejected": 0.288917601108551,
+      "logps/chosen": -352.2886657714844,
+      "logps/rejected": -395.7070617675781,
+      "loss": 0.6693,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -2.3514175415039062,
+      "rewards/margins": 3.268317699432373,
+      "rewards/rejected": -5.619735240936279,
       "step": 80
     },
     {
-      "epoch": 0.49,
-      "grad_norm": 181.07597775080902,
-      "learning_rate": 3.062889851306735e-07,
-      "logits/chosen": -0.18872031569480896,
-      "logits/rejected": 0.6850183606147766,
-      "logps/chosen": -269.26458740234375,
-      "logps/rejected": -247.515380859375,
-      "loss": 0.3436,
-      "rewards/accuracies": 0.8687499761581421,
-      "rewards/chosen": -1.4394252300262451,
-      "rewards/margins": 4.60965633392334,
-      "rewards/rejected": -6.049081802368164,
+      "epoch": 0.38,
+      "grad_norm": 514.8591353074631,
+      "learning_rate": 3.9247834624635404e-07,
+      "logits/chosen": -0.302684485912323,
+      "logits/rejected": 0.4978007674217224,
+      "logps/chosen": -331.9069519042969,
+      "logps/rejected": -362.5081787109375,
+      "loss": 0.6574,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -2.7024354934692383,
+      "rewards/margins": 3.1714892387390137,
+      "rewards/rejected": -5.873924732208252,
       "step": 90
     },
     {
-      "epoch": 0.54,
-      "grad_norm": 149.75808830683948,
-      "learning_rate": 2.594603691794176e-07,
-      "logits/chosen": -0.3397860825061798,
-      "logits/rejected": 0.5904129147529602,
-      "logps/chosen": -273.0763244628906,
-      "logps/rejected": -248.20419311523438,
-      "loss": 0.3162,
-      "rewards/accuracies": 0.84375,
-      "rewards/chosen": -1.644334077835083,
-      "rewards/margins": 4.102373123168945,
-      "rewards/rejected": -5.746707439422607,
+      "epoch": 0.42,
+      "grad_norm": 552.3368165213882,
+      "learning_rate": 3.610497133404795e-07,
+      "logits/chosen": -0.3550896942615509,
+      "logits/rejected": 0.3170868754386902,
+      "logps/chosen": -339.351318359375,
+      "logps/rejected": -385.2740173339844,
+      "loss": 0.6296,
+      "rewards/accuracies": 0.765625,
+      "rewards/chosen": -1.6958658695220947,
+      "rewards/margins": 3.125446319580078,
+      "rewards/rejected": -4.821312427520752,
       "step": 100
     },
     {
-      "epoch": 0.59,
-      "grad_norm": 203.30065061176032,
-      "learning_rate": 2.1229392570965654e-07,
-      "logits/chosen": 0.09322866052389145,
-      "logits/rejected": 0.8340438604354858,
-      "logps/chosen": -268.5257873535156,
-      "logps/rejected": -256.6501159667969,
-      "loss": 0.2936,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": -1.9208399057388306,
-      "rewards/margins": 4.171625137329102,
-      "rewards/rejected": -6.092464447021484,
+      "epoch": 0.46,
+      "grad_norm": 538.8081508757055,
+      "learning_rate": 3.272542485937368e-07,
+      "logits/chosen": -0.3151847720146179,
+      "logits/rejected": 0.5818986296653748,
+      "logps/chosen": -359.85504150390625,
+      "logps/rejected": -380.2372741699219,
+      "loss": 0.6323,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -2.4269652366638184,
+      "rewards/margins": 3.6542327404022217,
+      "rewards/rejected": -6.081197738647461,
       "step": 110
     },
     {
-      "epoch": 0.65,
-      "grad_norm": 166.429390404518,
-      "learning_rate": 1.6647395712565254e-07,
-      "logits/chosen": -0.3318867087364197,
-      "logits/rejected": 0.688382089138031,
-      "logps/chosen": -282.2430419921875,
-      "logps/rejected": -257.60906982421875,
-      "loss": 0.3294,
-      "rewards/accuracies": 0.878125011920929,
-      "rewards/chosen": -1.4651422500610352,
-      "rewards/margins": 4.4070515632629395,
-      "rewards/rejected": -5.872194290161133,
+      "epoch": 0.5,
+      "grad_norm": 480.1882844894413,
+      "learning_rate": 2.9181224366319943e-07,
+      "logits/chosen": -0.3534928262233734,
+      "logits/rejected": 0.37930893898010254,
+      "logps/chosen": -339.82684326171875,
+      "logps/rejected": -385.6717834472656,
+      "loss": 0.6157,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -3.0394375324249268,
+      "rewards/margins": 3.763584852218628,
+      "rewards/rejected": -6.803022861480713,
       "step": 120
     },
     {
-      "epoch": 0.7,
-      "grad_norm": 148.92458308520173,
-      "learning_rate": 1.2363668353585485e-07,
-      "logits/chosen": -0.3975695073604584,
-      "logits/rejected": 0.5577799081802368,
-      "logps/chosen": -269.56903076171875,
-      "logps/rejected": -246.27853393554688,
-      "loss": 0.2833,
-      "rewards/accuracies": 0.887499988079071,
-      "rewards/chosen": -1.3065744638442993,
-      "rewards/margins": 4.66982364654541,
-      "rewards/rejected": -5.97639799118042,
+      "epoch": 0.54,
+      "grad_norm": 440.0988758060072,
+      "learning_rate": 2.55479083351317e-07,
+      "logits/chosen": -0.4240172505378723,
+      "logits/rejected": 0.5124756097793579,
+      "logps/chosen": -373.0182189941406,
+      "logps/rejected": -388.32049560546875,
+      "loss": 0.6406,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -1.8788728713989258,
+      "rewards/margins": 3.37638521194458,
+      "rewards/rejected": -5.255258560180664,
       "step": 130
     },
     {
-      "epoch": 0.76,
-      "grad_norm": 163.61735672919622,
-      "learning_rate": 8.53118137245516e-08,
-      "logits/chosen": -0.16422179341316223,
-      "logits/rejected": 0.7476053237915039,
-      "logps/chosen": -278.07012939453125,
-      "logps/rejected": -263.6396179199219,
-      "loss": 0.2785,
-      "rewards/accuracies": 0.8687499761581421,
-      "rewards/chosen": -1.7827621698379517,
-      "rewards/margins": 4.88272762298584,
-      "rewards/rejected": -6.66549015045166,
+      "epoch": 0.59,
+      "grad_norm": 567.7119079444957,
+      "learning_rate": 2.19029145890313e-07,
+      "logits/chosen": -0.40219712257385254,
+      "logits/rejected": 0.43135371804237366,
+      "logps/chosen": -348.668701171875,
+      "logps/rejected": -383.73431396484375,
+      "loss": 0.6295,
+      "rewards/accuracies": 0.784375011920929,
+      "rewards/chosen": -2.540496826171875,
+      "rewards/margins": 3.5945935249328613,
+      "rewards/rejected": -6.135090351104736,
       "step": 140
     },
     {
-      "epoch": 0.81,
-      "grad_norm": 194.29564748495451,
-      "learning_rate": 5.2867919617408553e-08,
-      "logits/chosen": -0.2681286334991455,
-      "logits/rejected": 0.6748679876327515,
-      "logps/chosen": -273.7714538574219,
-      "logps/rejected": -249.87802124023438,
-      "loss": 0.2965,
-      "rewards/accuracies": 0.887499988079071,
-      "rewards/chosen": -1.0340642929077148,
-      "rewards/margins": 4.415706157684326,
-      "rewards/rejected": -5.449770927429199,
+      "epoch": 0.63,
+      "grad_norm": 580.9997311687505,
+      "learning_rate": 1.8323929841460178e-07,
+      "logits/chosen": -0.45832833647727966,
+      "logits/rejected": 0.4165743887424469,
+      "logps/chosen": -356.80035400390625,
+      "logps/rejected": -386.0637512207031,
+      "loss": 0.6079,
+      "rewards/accuracies": 0.765625,
+      "rewards/chosen": -2.680931568145752,
+      "rewards/margins": 3.39042592048645,
+      "rewards/rejected": -6.071357250213623,
       "step": 150
     },
     {
-      "epoch": 0.86,
-      "grad_norm": 161.72266919142106,
-      "learning_rate": 2.7463564905650853e-08,
-      "logits/chosen": -0.38469237089157104,
-      "logits/rejected": 0.4570757746696472,
-      "logps/chosen": -274.9485168457031,
-      "logps/rejected": -255.884033203125,
-      "loss": 0.2761,
-      "rewards/accuracies": 0.903124988079071,
-      "rewards/chosen": -0.9241056442260742,
-      "rewards/margins": 4.453596591949463,
-      "rewards/rejected": -5.377701759338379,
+      "epoch": 0.67,
+      "grad_norm": 571.50548834413,
+      "learning_rate": 1.488723393865766e-07,
+      "logits/chosen": -0.2687947750091553,
+      "logits/rejected": 0.5257094502449036,
+      "logps/chosen": -372.47467041015625,
+      "logps/rejected": -392.5708923339844,
+      "loss": 0.5822,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -2.8401567935943604,
+      "rewards/margins": 3.3291401863098145,
+      "rewards/rejected": -6.169297218322754,
       "step": 160
     },
     {
-      "epoch": 0.92,
-      "grad_norm": 170.5375651202773,
-      "learning_rate": 1.0005933014019307e-08,
-      "logits/chosen": -0.3120626211166382,
-      "logits/rejected": 0.671655535697937,
-      "logps/chosen": -275.0873718261719,
-      "logps/rejected": -257.51458740234375,
-      "loss": 0.2835,
-      "rewards/accuracies": 0.890625,
-      "rewards/chosen": -1.5441855192184448,
-      "rewards/margins": 4.443791389465332,
-      "rewards/rejected": -5.987977027893066,
+      "epoch": 0.71,
+      "grad_norm": 635.1524720165668,
+      "learning_rate": 1.1666074087171627e-07,
+      "logits/chosen": -0.4828396439552307,
+      "logits/rejected": 0.25855886936187744,
+      "logps/chosen": -358.3913879394531,
+      "logps/rejected": -405.04833984375,
+      "loss": 0.5831,
+      "rewards/accuracies": 0.765625,
+      "rewards/chosen": -2.5792622566223145,
+      "rewards/margins": 3.4110665321350098,
+      "rewards/rejected": -5.990328788757324,
       "step": 170
     },
     {
-      "epoch": 0.97,
-      "grad_norm": 164.92791969056674,
-      "learning_rate": 1.1184317978602808e-09,
-      "logits/chosen": -0.4767976701259613,
-      "logits/rejected": 0.42347556352615356,
-      "logps/chosen": -266.172607421875,
-      "logps/rejected": -251.0802001953125,
-      "loss": 0.3487,
-      "rewards/accuracies": 0.925000011920929,
-      "rewards/chosen": -1.1791002750396729,
-      "rewards/margins": 4.955374717712402,
-      "rewards/rejected": -6.134474277496338,
+      "epoch": 0.75,
+      "grad_norm": 475.58804797511704,
+      "learning_rate": 8.729103716819111e-08,
+      "logits/chosen": -0.4832540452480316,
+      "logits/rejected": 0.45813870429992676,
+      "logps/chosen": -360.1710510253906,
+      "logps/rejected": -385.29180908203125,
+      "loss": 0.5327,
+      "rewards/accuracies": 0.796875,
+      "rewards/chosen": -3.303698778152466,
+      "rewards/margins": 3.9217076301574707,
+      "rewards/rejected": -7.225406646728516,
       "step": 180
     },
+    {
+      "epoch": 0.79,
+      "grad_norm": 505.1938261302253,
+      "learning_rate": 6.138919252022435e-08,
+      "logits/chosen": -0.2969876229763031,
+      "logits/rejected": 0.3868524134159088,
+      "logps/chosen": -341.55084228515625,
+      "logps/rejected": -388.48858642578125,
+      "loss": 0.6088,
+      "rewards/accuracies": 0.815625011920929,
+      "rewards/chosen": -3.5507800579071045,
+      "rewards/margins": 3.956249713897705,
+      "rewards/rejected": -7.507031440734863,
+      "step": 190
+    },
+    {
+      "epoch": 0.84,
+      "grad_norm": 538.707594486278,
+      "learning_rate": 3.9507259776993954e-08,
+      "logits/chosen": -0.4874555170536041,
+      "logits/rejected": 0.24319347739219666,
+      "logps/chosen": -368.9916687011719,
+      "logps/rejected": -413.5546875,
+      "loss": 0.5485,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -2.9248712062835693,
+      "rewards/margins": 3.447073459625244,
+      "rewards/rejected": -6.371943950653076,
+      "step": 200
+    },
+    {
+      "epoch": 0.88,
+      "grad_norm": 653.4659715305337,
+      "learning_rate": 2.2111614344599684e-08,
+      "logits/chosen": -0.5115900635719299,
+      "logits/rejected": 0.3286168575286865,
+      "logps/chosen": -377.30523681640625,
+      "logps/rejected": -402.25811767578125,
+      "loss": 0.5517,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -2.5977110862731934,
+      "rewards/margins": 3.4149317741394043,
+      "rewards/rejected": -6.012642860412598,
+      "step": 210
+    },
+    {
+      "epoch": 0.92,
+      "grad_norm": 509.6499303160426,
+      "learning_rate": 9.57301420397924e-09,
+      "logits/chosen": -0.5053508877754211,
+      "logits/rejected": 0.2792305648326874,
+      "logps/chosen": -361.4806213378906,
+      "logps/rejected": -397.5937805175781,
+      "loss": 0.5519,
+      "rewards/accuracies": 0.8031250238418579,
+      "rewards/chosen": -2.464998960494995,
+      "rewards/margins": 3.379429578781128,
+      "rewards/rejected": -5.844428062438965,
+      "step": 220
+    },
+    {
+      "epoch": 0.96,
+      "grad_norm": 515.045513417983,
+      "learning_rate": 2.158697848236607e-09,
+      "logits/chosen": -0.4747482240200043,
+      "logits/rejected": 0.19420669972896576,
+      "logps/chosen": -361.44647216796875,
+      "logps/rejected": -409.26702880859375,
+      "loss": 0.5529,
+      "rewards/accuracies": 0.746874988079071,
+      "rewards/chosen": -2.540475368499756,
+      "rewards/margins": 3.0807271003723145,
+      "rewards/rejected": -5.62120246887207,
+      "step": 230
+    },
     {
       "epoch": 1.0,
-      "step": 185,
+      "step": 239,
       "total_flos": 0.0,
-      "train_loss": 0.34950463063008075,
-      "train_runtime": 5351.3328,
-      "train_samples_per_second": 8.839,
-      "train_steps_per_second": 0.035
+      "train_loss": 0.6121151776493344,
+      "train_runtime": 6995.9878,
+      "train_samples_per_second": 8.738,
+      "train_steps_per_second": 0.034
     }
   ],
   "logging_steps": 10,
-  "max_steps": 185,
+  "max_steps": 239,
   "num_input_tokens_seen": 0,
   "num_train_epochs": 1,
   "save_steps": 100,

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b6f555e8b191e5dee3bafecffb3a88f44ea1db78545e88e62c9e7b6e0969daad
+oid sha256:4fcf5c85f80c92a9eacc9ea3ac3046fce1e674dd5e2c6a60e50c6fe6b0e4a3a2
 size 6328