Model save

README.md

@@ -35,7 +35,7 @@ The following hyperparameters were used during training:
 - learning_rate: 3e-06
 - train_batch_size: 2
 - eval_batch_size: 8
-- seed: 5
+- seed: 1
 - distributed_type: multi-GPU
 - num_devices: 8
 - gradient_accumulation_steps: 8

all_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 1.0,
-    "train_loss": 0.36347593488827556,
-    "train_runtime": 5281.0991,
-    "train_samples": 45548,
-    "train_samples_per_second": 8.625,
-    "train_steps_per_second": 0.067
+    "train_loss": 0.06463673111027891,
+    "train_runtime": 6408.5161,
+    "train_samples": 61134,
+    "train_samples_per_second": 9.539,
+    "train_steps_per_second": 0.074
 }

model-00001-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2f6a53f0de4447eb60a67b94195b21f61ff96d26641e186ce68d47553b85e759
+oid sha256:27af38a706248a0b99ae38fc74e38845493c9f49e3ca192ac362b2cdeb19307c
 size 4976698672

model-00002-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:992cfde5d1ddcabd7207fb645d58f7600e8efcbfea60a4bca5c54264d3e3f8d6
+oid sha256:ca904bd84f3705e920e8b8be6855e11df01f0a3ed29009d2c7ddc39b5509121b
 size 4999802720

model-00003-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:658f104b0a1fd1120eaff2ca804fcd58a1623759de1eaff60f24a3db80cfd2fc
+oid sha256:a2f5a3559e562c55ca125abbea481359306cbfe893289ea7799a26f7a84812eb
 size 4915916176

model-00004-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ce53dea6eec3396aa58034e0ca8d59e1d64baa02ef8e6bb7cbd9619deeb20423
+oid sha256:23a2ae3fba6aa6db34d28591f49acae193d0078360041a1d200071a70813f087
 size 1168138808

train_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 1.0,
-    "train_loss": 0.36347593488827556,
-    "train_runtime": 5281.0991,
-    "train_samples": 45548,
-    "train_samples_per_second": 8.625,
-    "train_steps_per_second": 0.067
+    "train_loss": 0.06463673111027891,
+    "train_runtime": 6408.5161,
+    "train_samples": 61134,
+    "train_samples_per_second": 9.539,
+    "train_steps_per_second": 0.074
 }

trainer_state.json

@@ -1,515 +1,683 @@
 {
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 0.9975412715138743,
+  "epoch": 0.998691442030882,
   "eval_steps": 10000,
-  "global_step": 355,
+  "global_step": 477,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
-      "epoch": 0.03,
-      "learning_rate": 8.333333333333334e-07,
-      "logits/chosen": -0.017936866730451584,
-      "logits/rejected": 0.045307982712984085,
-      "logps/chosen": -322.30169677734375,
-      "logps/rejected": -218.5985107421875,
-      "loss": 0.5197,
-      "rewards/accuracies": 0.42500001192092896,
-      "rewards/chosen": 0.0008429696899838746,
-      "rewards/margins": 0.0020774812437593937,
-      "rewards/rejected": -0.0012345117283985019,
+      "epoch": 0.02,
+      "learning_rate": 6.25e-07,
+      "logits/chosen": 0.17706245183944702,
+      "logits/rejected": 0.2540971636772156,
+      "logps/chosen": -354.3509826660156,
+      "logps/rejected": -305.29473876953125,
+      "loss": 0.1819,
+      "rewards/accuracies": 0.4124999940395355,
+      "rewards/chosen": 0.0004928814596496522,
+      "rewards/margins": 0.001260685734450817,
+      "rewards/rejected": -0.0007678042748011649,
       "step": 10
     },
     {
-      "epoch": 0.06,
-      "learning_rate": 1.6666666666666669e-06,
-      "logits/chosen": -0.030718382447957993,
-      "logits/rejected": -0.028245821595191956,
-      "logps/chosen": -333.917236328125,
-      "logps/rejected": -203.45745849609375,
-      "loss": 0.5119,
-      "rewards/accuracies": 0.668749988079071,
-      "rewards/chosen": 0.004156398121267557,
-      "rewards/margins": 0.04196573421359062,
-      "rewards/rejected": -0.0378093346953392,
+      "epoch": 0.04,
+      "learning_rate": 1.25e-06,
+      "logits/chosen": 0.08185596764087677,
+      "logits/rejected": 0.20913369953632355,
+      "logps/chosen": -316.39178466796875,
+      "logps/rejected": -277.11273193359375,
+      "loss": 0.1822,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": 0.0034345737658441067,
+      "rewards/margins": 0.014207230880856514,
+      "rewards/rejected": -0.010772655718028545,
       "step": 20
     },
     {
-      "epoch": 0.08,
-      "learning_rate": 2.5e-06,
-      "logits/chosen": -0.08321847021579742,
-      "logits/rejected": -0.0037727858871221542,
-      "logps/chosen": -432.2870178222656,
-      "logps/rejected": -275.5068664550781,
-      "loss": 0.5456,
-      "rewards/accuracies": 0.550000011920929,
-      "rewards/chosen": -0.017191946506500244,
-      "rewards/margins": 0.21662700176239014,
-      "rewards/rejected": -0.23381897807121277,
+      "epoch": 0.06,
+      "learning_rate": 1.875e-06,
+      "logits/chosen": 0.2634967267513275,
+      "logits/rejected": 0.3354651629924774,
+      "logps/chosen": -301.5518493652344,
+      "logps/rejected": -310.61309814453125,
+      "loss": 0.1772,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.06584969907999039,
+      "rewards/margins": 0.057301245629787445,
+      "rewards/rejected": -0.12315094470977783,
       "step": 30
     },
     {
-      "epoch": 0.11,
-      "learning_rate": 2.9988362934929793e-06,
-      "logits/chosen": -0.037398938089609146,
-      "logits/rejected": -0.007799749728292227,
-      "logps/chosen": -359.85650634765625,
-      "logps/rejected": -278.62567138671875,
-      "loss": 0.5288,
-      "rewards/accuracies": 0.65625,
-      "rewards/chosen": 0.04756924882531166,
-      "rewards/margins": 0.2058154046535492,
-      "rewards/rejected": -0.15824612975120544,
+      "epoch": 0.08,
+      "learning_rate": 2.5e-06,
+      "logits/chosen": 0.27594679594039917,
+      "logits/rejected": 0.38354000449180603,
+      "logps/chosen": -374.3425598144531,
+      "logps/rejected": -361.6021423339844,
+      "loss": 0.1397,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.2677033543586731,
+      "rewards/margins": 0.14364728331565857,
+      "rewards/rejected": -0.41135063767433167,
       "step": 40
     },
     {
-      "epoch": 0.14,
-      "learning_rate": 2.985765322825759e-06,
-      "logits/chosen": -0.0844019427895546,
-      "logits/rejected": -0.06939023733139038,
-      "logps/chosen": -276.42376708984375,
-      "logps/rejected": -213.7895965576172,
-      "loss": 0.4848,
-      "rewards/accuracies": 0.59375,
-      "rewards/chosen": -0.014759841375052929,
-      "rewards/margins": 0.1379159539937973,
-      "rewards/rejected": -0.1526757776737213,
+      "epoch": 0.1,
+      "learning_rate": 2.999839121261416e-06,
+      "logits/chosen": 0.3216066062450409,
+      "logits/rejected": 0.4195839762687683,
+      "logps/chosen": -371.8332824707031,
+      "logps/rejected": -384.00860595703125,
+      "loss": 0.0862,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.6128066778182983,
+      "rewards/margins": 0.39045700430870056,
+      "rewards/rejected": -1.0032637119293213,
       "step": 50
     },
     {
-      "epoch": 0.17,
-      "learning_rate": 2.9582958419982717e-06,
-      "logits/chosen": 0.016000710427761078,
-      "logits/rejected": 0.07897808402776718,
-      "logps/chosen": -367.83868408203125,
-      "logps/rejected": -227.6174774169922,
-      "loss": 0.4654,
-      "rewards/accuracies": 0.675000011920929,
-      "rewards/chosen": -0.05412111431360245,
-      "rewards/margins": 0.4200070798397064,
-      "rewards/rejected": -0.4741281569004059,
+      "epoch": 0.13,
+      "learning_rate": 2.994211988057582e-06,
+      "logits/chosen": 0.16005071997642517,
+      "logits/rejected": 0.29673656821250916,
+      "logps/chosen": -350.0096740722656,
+      "logps/rejected": -372.6922912597656,
+      "loss": 0.0896,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.5713089108467102,
+      "rewards/margins": 0.4011602997779846,
+      "rewards/rejected": -0.9724692106246948,
       "step": 60
     },
     {
-      "epoch": 0.2,
-      "learning_rate": 2.916694056980408e-06,
-      "logits/chosen": -0.0025311470963060856,
-      "logits/rejected": 0.02233794890344143,
-      "logps/chosen": -312.28948974609375,
-      "logps/rejected": -265.64617919921875,
-      "loss": 0.464,
-      "rewards/accuracies": 0.6312500238418579,
-      "rewards/chosen": -0.3601204752922058,
-      "rewards/margins": 0.24295297265052795,
-      "rewards/rejected": -0.6030734181404114,
+      "epoch": 0.15,
+      "learning_rate": 2.9805753939568693e-06,
+      "logits/chosen": 0.0862194150686264,
+      "logits/rejected": 0.18484732508659363,
+      "logps/chosen": -361.5823974609375,
+      "logps/rejected": -393.074462890625,
+      "loss": 0.1054,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.3185870051383972,
+      "rewards/margins": 0.42267927527427673,
+      "rewards/rejected": -0.7412663102149963,
       "step": 70
     },
     {
-      "epoch": 0.22,
-      "learning_rate": 2.8613631295064357e-06,
-      "logits/chosen": -0.21361954510211945,
-      "logits/rejected": -0.14258694648742676,
-      "logps/chosen": -390.8519287109375,
-      "logps/rejected": -232.2322540283203,
-      "loss": 0.4208,
-      "rewards/accuracies": 0.6875,
-      "rewards/chosen": -0.1345187872648239,
-      "rewards/margins": 0.3031948208808899,
-      "rewards/rejected": -0.4377136826515198,
+      "epoch": 0.17,
+      "learning_rate": 2.959002435526626e-06,
+      "logits/chosen": 0.14816270768642426,
+      "logits/rejected": 0.2659767270088196,
+      "logps/chosen": -378.1669616699219,
+      "logps/rejected": -401.71990966796875,
+      "loss": 0.0828,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.5950332880020142,
+      "rewards/margins": 0.5514911413192749,
+      "rewards/rejected": -1.1465245485305786,
       "step": 80
     },
     {
-      "epoch": 0.25,
-      "learning_rate": 2.792839270045916e-06,
-      "logits/chosen": -0.07295586168766022,
-      "logits/rejected": -0.0871509537100792,
-      "logps/chosen": -291.86248779296875,
-      "logps/rejected": -255.58837890625,
-      "loss": 0.4122,
-      "rewards/accuracies": 0.5562499761581421,
-      "rewards/chosen": -0.2528306543827057,
-      "rewards/margins": 0.14771175384521484,
-      "rewards/rejected": -0.4005424380302429,
+      "epoch": 0.19,
+      "learning_rate": 2.929608750821129e-06,
+      "logits/chosen": 0.025451337918639183,
+      "logits/rejected": 0.19968536496162415,
+      "logps/chosen": -441.95806884765625,
+      "logps/rejected": -415.0679626464844,
+      "loss": 0.0542,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -1.0501652956008911,
+      "rewards/margins": 0.4297688901424408,
+      "rewards/rejected": -1.4799340963363647,
       "step": 90
     },
     {
-      "epoch": 0.28,
-      "learning_rate": 2.711786541403051e-06,
-      "logits/chosen": -0.09233228862285614,
-      "logits/rejected": -0.02968590334057808,
-      "logps/chosen": -370.9435729980469,
-      "logps/rejected": -273.88336181640625,
-      "loss": 0.4075,
-      "rewards/accuracies": 0.6187499761581421,
-      "rewards/chosen": -0.41826897859573364,
-      "rewards/margins": 0.3016797602176666,
-      "rewards/rejected": -0.7199487686157227,
+      "epoch": 0.21,
+      "learning_rate": 2.892551899524109e-06,
+      "logits/chosen": 0.12514810264110565,
+      "logits/rejected": 0.3461776673793793,
+      "logps/chosen": -407.69647216796875,
+      "logps/rejected": -429.8515625,
+      "loss": 0.0753,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -0.7313500046730042,
+      "rewards/margins": 0.6433447599411011,
+      "rewards/rejected": -1.37469482421875,
       "step": 100
     },
     {
-      "epoch": 0.31,
-      "learning_rate": 2.6189904233026363e-06,
-      "logits/chosen": -0.154756560921669,
-      "logits/rejected": -0.07636446505784988,
-      "logps/chosen": -376.0904235839844,
-      "logps/rejected": -294.49786376953125,
-      "loss": 0.3787,
-      "rewards/accuracies": 0.612500011920929,
-      "rewards/chosen": -0.4825851023197174,
-      "rewards/margins": 0.3285244107246399,
-      "rewards/rejected": -0.8111095428466797,
+      "epoch": 0.23,
+      "learning_rate": 2.848030518377739e-06,
+      "logits/chosen": 0.14924690127372742,
+      "logits/rejected": 0.21244895458221436,
+      "logps/chosen": -398.8333740234375,
+      "logps/rejected": -449.89080810546875,
+      "loss": 0.0684,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -0.8621395230293274,
+      "rewards/margins": 0.6422672271728516,
+      "rewards/rejected": -1.5044066905975342,
       "step": 110
     },
     {
-      "epoch": 0.34,
-      "learning_rate": 2.515350200328027e-06,
-      "logits/chosen": -0.1974565088748932,
-      "logits/rejected": -0.15066322684288025,
-      "logps/chosen": -353.86907958984375,
-      "logps/rejected": -284.64935302734375,
-      "loss": 0.3675,
-      "rewards/accuracies": 0.59375,
-      "rewards/chosen": -0.5684340000152588,
-      "rewards/margins": 0.19724634289741516,
-      "rewards/rejected": -0.7656803131103516,
+      "epoch": 0.25,
+      "learning_rate": 2.7962832564252724e-06,
+      "logits/chosen": 0.06671784818172455,
+      "logits/rejected": 0.14493227005004883,
+      "logps/chosen": -412.3326110839844,
+      "logps/rejected": -474.692138671875,
+      "loss": 0.0523,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -0.9177900552749634,
+      "rewards/margins": 0.6171673536300659,
+      "rewards/rejected": -1.5349572896957397,
       "step": 120
     },
     {
-      "epoch": 0.37,
-      "learning_rate": 2.401870246979413e-06,
-      "logits/chosen": -0.19759312272071838,
-      "logits/rejected": -0.14790299534797668,
-      "logps/chosen": -408.52532958984375,
-      "logps/rejected": -301.2213439941406,
-      "loss": 0.3345,
+      "epoch": 0.27,
+      "learning_rate": 2.7375874957747644e-06,
+      "logits/chosen": 0.037053029984235764,
+      "logits/rejected": 0.2220267802476883,
+      "logps/chosen": -437.1244201660156,
+      "logps/rejected": -436.5159606933594,
+      "loss": 0.0564,
       "rewards/accuracies": 0.6625000238418579,
-      "rewards/chosen": -0.6431132555007935,
-      "rewards/margins": 0.26965436339378357,
-      "rewards/rejected": -0.9127677083015442,
+      "rewards/chosen": -0.9862833023071289,
+      "rewards/margins": 0.5303726196289062,
+      "rewards/rejected": -1.5166559219360352,
       "step": 130
     },
     {
-      "epoch": 0.39,
-      "learning_rate": 2.279650294308645e-06,
-      "logits/chosen": -0.275656521320343,
-      "logits/rejected": -0.1713864505290985,
-      "logps/chosen": -383.6747131347656,
-      "logps/rejected": -270.3328857421875,
-      "loss": 0.3348,
+      "epoch": 0.29,
+      "learning_rate": 2.672257864741005e-06,
+      "logits/chosen": 0.05896978825330734,
+      "logits/rejected": 0.23563237488269806,
+      "logps/chosen": -409.24481201171875,
+      "logps/rejected": -379.90924072265625,
+      "loss": 0.0706,
       "rewards/accuracies": 0.637499988079071,
-      "rewards/chosen": -0.6792303323745728,
-      "rewards/margins": 0.3043554425239563,
-      "rewards/rejected": -0.9835857152938843,
+      "rewards/chosen": -0.7720142602920532,
+      "rewards/margins": 0.32800909876823425,
+      "rewards/rejected": -1.1000233888626099,
       "step": 140
     },
     {
-      "epoch": 0.42,
-      "learning_rate": 2.1498747724563957e-06,
-      "logits/chosen": -0.27238455414772034,
-      "logits/rejected": -0.22029218077659607,
-      "logps/chosen": -391.3775329589844,
-      "logps/rejected": -335.1509704589844,
-      "loss": 0.3153,
-      "rewards/accuracies": 0.5625,
-      "rewards/chosen": -0.8400734663009644,
-      "rewards/margins": 0.28538864850997925,
-      "rewards/rejected": -1.1254620552062988,
+      "epoch": 0.31,
+      "learning_rate": 2.600644551335706e-06,
+      "logits/chosen": 0.10780592262744904,
+      "logits/rejected": 0.15396630764007568,
+      "logps/chosen": -420.455078125,
+      "logps/rejected": -461.3589782714844,
+      "loss": 0.0715,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.8515451550483704,
+      "rewards/margins": 0.6125485301017761,
+      "rewards/rejected": -1.464093804359436,
       "step": 150
     },
     {
-      "epoch": 0.45,
-      "learning_rate": 2.0138013323728074e-06,
-      "logits/chosen": -0.315167099237442,
-      "logits/rejected": -0.23690147697925568,
-      "logps/chosen": -406.4299011230469,
-      "logps/rejected": -320.24237060546875,
-      "loss": 0.2884,
-      "rewards/accuracies": 0.6312500238418579,
-      "rewards/chosen": -0.9414618611335754,
-      "rewards/margins": 0.28279590606689453,
-      "rewards/rejected": -1.2242577075958252,
+      "epoch": 0.33,
+      "learning_rate": 2.5231314261461732e-06,
+      "logits/chosen": 0.001874491572380066,
+      "logits/rejected": 0.0530397966504097,
+      "logps/chosen": -404.02886962890625,
+      "logps/rejected": -476.5221252441406,
+      "loss": 0.0509,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -1.0196731090545654,
+      "rewards/margins": 0.9385590553283691,
+      "rewards/rejected": -1.9582321643829346,
       "step": 160
     },
     {
-      "epoch": 0.48,
-      "learning_rate": 1.8727486579573409e-06,
-      "logits/chosen": -0.31794100999832153,
-      "logits/rejected": -0.24908527731895447,
-      "logps/chosen": -438.0152893066406,
-      "logps/rejected": -302.74273681640625,
-      "loss": 0.3332,
-      "rewards/accuracies": 0.65625,
-      "rewards/chosen": -0.6466701626777649,
-      "rewards/margins": 0.3362303376197815,
-      "rewards/rejected": -0.9829004406929016,
+      "epoch": 0.36,
+      "learning_rate": 2.440133984664454e-06,
+      "logits/chosen": 0.03371699899435043,
+      "logits/rejected": 0.11997250467538834,
+      "logps/chosen": -449.0216369628906,
+      "logps/rejected": -491.6813049316406,
+      "loss": 0.0569,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -1.126532793045044,
+      "rewards/margins": 0.7117874026298523,
+      "rewards/rejected": -1.8383201360702515,
       "step": 170
     },
     {
-      "epoch": 0.51,
-      "learning_rate": 1.7280836867300083e-06,
-      "logits/chosen": -0.3656935691833496,
-      "logits/rejected": -0.32054948806762695,
-      "logps/chosen": -409.75927734375,
-      "logps/rejected": -302.7572021484375,
-      "loss": 0.3209,
-      "rewards/accuracies": 0.6499999761581421,
-      "rewards/chosen": -0.7197253108024597,
-      "rewards/margins": 0.2899569571018219,
-      "rewards/rejected": -1.009682297706604,
+      "epoch": 0.38,
+      "learning_rate": 2.3520971200967337e-06,
+      "logits/chosen": -0.039311788976192474,
+      "logits/rejected": 0.11384377628564835,
+      "logps/chosen": -429.73907470703125,
+      "logps/rejected": -488.38665771484375,
+      "loss": 0.0497,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -1.037070870399475,
+      "rewards/margins": 0.8480435609817505,
+      "rewards/rejected": -1.8851144313812256,
       "step": 180
     },
     {
-      "epoch": 0.53,
-      "learning_rate": 1.5812083628781265e-06,
-      "logits/chosen": -0.3520641624927521,
-      "logits/rejected": -0.31913143396377563,
-      "logps/chosen": -355.8263244628906,
-      "logps/rejected": -325.4466247558594,
-      "loss": 0.3024,
-      "rewards/accuracies": 0.6000000238418579,
-      "rewards/chosen": -1.009711503982544,
-      "rewards/margins": 0.15852129459381104,
-      "rewards/rejected": -1.1682326793670654,
+      "epoch": 0.4,
+      "learning_rate": 2.2594927385914546e-06,
+      "logits/chosen": -0.043736983090639114,
+      "logits/rejected": 0.0784933939576149,
+      "logps/chosen": -478.27044677734375,
+      "logps/rejected": -528.6622314453125,
+      "loss": 0.0427,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -1.4361096620559692,
+      "rewards/margins": 0.7232626676559448,
+      "rewards/rejected": -2.159372329711914,
       "step": 190
     },
     {
-      "epoch": 0.56,
-      "learning_rate": 1.433546051054432e-06,
-      "logits/chosen": -0.334557443857193,
-      "logits/rejected": -0.31981879472732544,
-      "logps/chosen": -380.0057067871094,
-      "logps/rejected": -353.52392578125,
-      "loss": 0.304,
-      "rewards/accuracies": 0.612500011920929,
-      "rewards/chosen": -1.09440016746521,
-      "rewards/margins": 0.2975843846797943,
-      "rewards/rejected": -1.3919847011566162,
+      "epoch": 0.42,
+      "learning_rate": 2.1628172296692954e-06,
+      "logits/chosen": -0.07505004107952118,
+      "logits/rejected": 0.08389478921890259,
+      "logps/chosen": -441.46142578125,
+      "logps/rejected": -471.33380126953125,
+      "loss": 0.0507,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -1.009716510772705,
+      "rewards/margins": 0.762354850769043,
+      "rewards/rejected": -1.7720712423324585,
       "step": 200
     },
     {
-      "epoch": 0.59,
-      "learning_rate": 1.2865277425900725e-06,
-      "logits/chosen": -0.37745895981788635,
-      "logits/rejected": -0.3244924545288086,
-      "logps/chosen": -395.09844970703125,
-      "logps/rejected": -331.40655517578125,
-      "loss": 0.2921,
-      "rewards/accuracies": 0.6187499761581421,
-      "rewards/chosen": -1.0221917629241943,
-      "rewards/margins": 0.2329825460910797,
-      "rewards/rejected": -1.2551742792129517,
+      "epoch": 0.44,
+      "learning_rate": 2.062588805414343e-06,
+      "logits/chosen": 0.04285336285829544,
+      "logits/rejected": 0.2198754996061325,
+      "logps/chosen": -396.66552734375,
+      "logps/rejected": -416.57763671875,
+      "loss": 0.0602,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -0.8250829577445984,
+      "rewards/margins": 0.6419562101364136,
+      "rewards/rejected": -1.4670391082763672,
       "step": 210
     },
     {
-      "epoch": 0.62,
-      "learning_rate": 1.141578187797663e-06,
-      "logits/chosen": -0.2749294638633728,
-      "logits/rejected": -0.2453482449054718,
-      "logps/chosen": -422.58477783203125,
-      "logps/rejected": -323.53900146484375,
-      "loss": 0.3127,
-      "rewards/accuracies": 0.65625,
-      "rewards/chosen": -0.8388057947158813,
-      "rewards/margins": 0.24924850463867188,
-      "rewards/rejected": -1.0880544185638428,
+      "epoch": 0.46,
+      "learning_rate": 1.9593447226892386e-06,
+      "logits/chosen": -0.06010212376713753,
+      "logits/rejected": 0.09760335832834244,
+      "logps/chosen": -397.46624755859375,
+      "logps/rejected": -395.8749084472656,
+      "loss": 0.0661,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.8078718185424805,
+      "rewards/margins": 0.5141724944114685,
+      "rewards/rejected": -1.3220441341400146,
       "step": 220
     },
     {
-      "epoch": 0.65,
-      "learning_rate": 1.0001020887558839e-06,
-      "logits/chosen": -0.2927996516227722,
-      "logits/rejected": -0.2675902247428894,
-      "logps/chosen": -373.8236083984375,
-      "logps/rejected": -345.1734924316406,
-      "loss": 0.3375,
-      "rewards/accuracies": 0.606249988079071,
-      "rewards/chosen": -0.8683871030807495,
-      "rewards/margins": 0.34467414021492004,
-      "rewards/rejected": -1.2130613327026367,
+      "epoch": 0.48,
+      "learning_rate": 1.853638403264141e-06,
+      "logits/chosen": -0.03770698606967926,
+      "logits/rejected": 0.013910258188843727,
+      "logps/chosen": -444.28985595703125,
+      "logps/rejected": -494.4593811035156,
+      "loss": 0.0507,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -1.182284951210022,
+      "rewards/margins": 0.9311016798019409,
+      "rewards/rejected": -2.113386631011963,
       "step": 230
     },
     {
-      "epoch": 0.67,
-      "learning_rate": 8.634704863809502e-07,
-      "logits/chosen": -0.26392003893852234,
-      "logits/rejected": -0.26393693685531616,
-      "logps/chosen": -417.762451171875,
-      "logps/rejected": -343.877197265625,
-      "loss": 0.3195,
-      "rewards/accuracies": 0.6000000238418579,
-      "rewards/chosen": -0.9277390241622925,
-      "rewards/margins": 0.2722298502922058,
-      "rewards/rejected": -1.199968934059143,
+      "epoch": 0.5,
+      "learning_rate": 1.7460364672965328e-06,
+      "logits/chosen": -0.16847534477710724,
+      "logits/rejected": -0.11212627589702606,
+      "logps/chosen": -446.85369873046875,
+      "logps/rejected": -486.97698974609375,
+      "loss": 0.051,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -1.3490196466445923,
+      "rewards/margins": 0.5016757845878601,
+      "rewards/rejected": -1.8506953716278076,
       "step": 240
     },
     {
-      "epoch": 0.7,
-      "learning_rate": 7.330074737074666e-07,
-      "logits/chosen": -0.32263797521591187,
-      "logits/rejected": -0.26007968187332153,
-      "logps/chosen": -449.156005859375,
-      "logps/rejected": -341.98846435546875,
-      "loss": 0.3217,
-      "rewards/accuracies": 0.581250011920929,
-      "rewards/chosen": -0.9365653991699219,
-      "rewards/margins": 0.19516155123710632,
-      "rewards/rejected": -1.1317269802093506,
+      "epoch": 0.52,
+      "learning_rate": 1.637115696063402e-06,
+      "logits/chosen": -0.20698556303977966,
+      "logits/rejected": -0.1366804540157318,
+      "logps/chosen": -411.64892578125,
+      "logps/rejected": -442.436279296875,
+      "loss": 0.0518,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -0.9960080981254578,
+      "rewards/margins": 0.6079148650169373,
+      "rewards/rejected": -1.6039228439331055,
       "step": 250
     },
     {
-      "epoch": 0.73,
-      "learning_rate": 6.099773641398835e-07,
-      "logits/chosen": -0.26862549781799316,
-      "logits/rejected": -0.21074727177619934,
-      "logps/chosen": -437.16864013671875,
-      "logps/rejected": -361.11383056640625,
-      "loss": 0.3233,
-      "rewards/accuracies": 0.637499988079071,
-      "rewards/chosen": -0.8692190051078796,
-      "rewards/margins": 0.34968703985214233,
-      "rewards/rejected": -1.218906044960022,
+      "epoch": 0.54,
+      "learning_rate": 1.5274599402265162e-06,
+      "logits/chosen": -0.30626335740089417,
+      "logits/rejected": -0.21004387736320496,
+      "logps/chosen": -493.10693359375,
+      "logps/rejected": -501.22113037109375,
+      "loss": 0.0491,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -1.3846315145492554,
+      "rewards/margins": 0.6269583106040955,
+      "rewards/rejected": -2.011589765548706,
       "step": 260
     },
     {
-      "epoch": 0.76,
-      "learning_rate": 4.955724390266841e-07,
-      "logits/chosen": -0.25061318278312683,
-      "logits/rejected": -0.21778492629528046,
-      "logps/chosen": -413.89892578125,
-      "logps/rejected": -349.9348449707031,
-      "loss": 0.3177,
-      "rewards/accuracies": 0.612500011920929,
-      "rewards/chosen": -0.9383090138435364,
-      "rewards/margins": 0.2968607246875763,
-      "rewards/rejected": -1.235169768333435,
+      "epoch": 0.57,
+      "learning_rate": 1.4176569902035088e-06,
+      "logits/chosen": -0.21837463974952698,
+      "logits/rejected": -0.18761083483695984,
+      "logps/chosen": -464.00579833984375,
+      "logps/rejected": -516.2754516601562,
+      "loss": 0.0453,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -1.348719596862793,
+      "rewards/margins": 0.7095474600791931,
+      "rewards/rejected": -2.058267116546631,
       "step": 270
     },
     {
-      "epoch": 0.79,
-      "learning_rate": 3.9090139329520333e-07,
-      "logits/chosen": -0.2774963974952698,
-      "logits/rejected": -0.24690791964530945,
-      "logps/chosen": -425.43524169921875,
-      "logps/rejected": -339.97259521484375,
-      "loss": 0.316,
-      "rewards/accuracies": 0.675000011920929,
-      "rewards/chosen": -0.9426881670951843,
-      "rewards/margins": 0.4042733609676361,
-      "rewards/rejected": -1.346961259841919,
+      "epoch": 0.59,
+      "learning_rate": 1.308295425420593e-06,
+      "logits/chosen": -0.20622439682483673,
+      "logits/rejected": -0.22035178542137146,
+      "logps/chosen": -418.81097412109375,
+      "logps/rejected": -477.7355041503906,
+      "loss": 0.0522,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -1.2232805490493774,
+      "rewards/margins": 0.6434040665626526,
+      "rewards/rejected": -1.8666845560073853,
       "step": 280
     },
     {
-      "epoch": 0.81,
-      "learning_rate": 2.9697859112011724e-07,
-      "logits/chosen": -0.2760419547557831,
-      "logits/rejected": -0.24498526751995087,
-      "logps/chosen": -412.57318115234375,
-      "logps/rejected": -335.27398681640625,
-      "loss": 0.3082,
-      "rewards/accuracies": 0.71875,
-      "rewards/chosen": -0.7900550961494446,
-      "rewards/margins": 0.4711545407772064,
-      "rewards/rejected": -1.261209487915039,
+      "epoch": 0.61,
+      "learning_rate": 1.1999614593359337e-06,
+      "logits/chosen": -0.18864139914512634,
+      "logits/rejected": -0.12041006982326508,
+      "logps/chosen": -460.80059814453125,
+      "logps/rejected": -523.751953125,
+      "loss": 0.0502,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -1.1436948776245117,
+      "rewards/margins": 0.8658909797668457,
+      "rewards/rejected": -2.0095858573913574,
       "step": 290
     },
     {
-      "epoch": 0.84,
-      "learning_rate": 2.1471423574861643e-07,
-      "logits/chosen": -0.3094063997268677,
-      "logits/rejected": -0.2514321208000183,
-      "logps/chosen": -432.04644775390625,
-      "logps/rejected": -345.2815856933594,
-      "loss": 0.3048,
-      "rewards/accuracies": 0.65625,
-      "rewards/chosen": -0.8672820329666138,
-      "rewards/margins": 0.400570809841156,
-      "rewards/rejected": -1.2678529024124146,
+      "epoch": 0.63,
+      "learning_rate": 1.0932357971453745e-06,
+      "logits/chosen": -0.142107754945755,
+      "logits/rejected": -0.11606737226247787,
+      "logps/chosen": -407.21630859375,
+      "logps/rejected": -519.4342041015625,
+      "loss": 0.0422,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -1.1546132564544678,
+      "rewards/margins": 0.8063042759895325,
+      "rewards/rejected": -1.9609174728393555,
       "step": 300
     },
     {
-      "epoch": 0.87,
-      "learning_rate": 1.449055487462102e-07,
-      "logits/chosen": -0.3206644654273987,
-      "logits/rejected": -0.27432483434677124,
-      "logps/chosen": -436.33056640625,
-      "logps/rejected": -320.2474670410156,
-      "loss": 0.3097,
-      "rewards/accuracies": 0.637499988079071,
-      "rewards/chosen": -0.9078308343887329,
-      "rewards/margins": 0.35346266627311707,
-      "rewards/rejected": -1.2612934112548828,
+      "epoch": 0.65,
+      "learning_rate": 9.886905230142433e-07,
+      "logits/chosen": -0.10813410580158234,
+      "logits/rejected": -0.10192851722240448,
+      "logps/chosen": -414.4422912597656,
+      "logps/rejected": -515.26416015625,
+      "loss": 0.0439,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -1.1871048212051392,
+      "rewards/margins": 0.8484551310539246,
+      "rewards/rejected": -2.035560131072998,
       "step": 310
     },
     {
-      "epoch": 0.9,
-      "learning_rate": 8.822904414485194e-08,
-      "logits/chosen": -0.2824671268463135,
-      "logits/rejected": -0.23151321709156036,
-      "logps/chosen": -428.50921630859375,
-      "logps/rejected": -328.49884033203125,
-      "loss": 0.3103,
-      "rewards/accuracies": 0.6312500238418579,
-      "rewards/chosen": -0.8764031529426575,
-      "rewards/margins": 0.3287006914615631,
-      "rewards/rejected": -1.205103874206543,
+      "epoch": 0.67,
+      "learning_rate": 8.868860335206678e-07,
+      "logits/chosen": -0.10642366111278534,
+      "logits/rejected": 0.012383558787405491,
+      "logps/chosen": -418.55712890625,
+      "logps/rejected": -490.4601135253906,
+      "loss": 0.0478,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -1.0292608737945557,
+      "rewards/margins": 0.9733268618583679,
+      "rewards/rejected": -2.0025877952575684,
       "step": 320
     },
     {
-      "epoch": 0.93,
-      "learning_rate": 4.523397236438398e-08,
-      "logits/chosen": -0.24987097084522247,
-      "logits/rejected": -0.22064971923828125,
-      "logps/chosen": -393.6728515625,
-      "logps/rejected": -334.59051513671875,
-      "loss": 0.3136,
-      "rewards/accuracies": 0.625,
-      "rewards/chosen": -0.8506187200546265,
-      "rewards/margins": 0.35463377833366394,
-      "rewards/rejected": -1.2052525281906128,
+      "epoch": 0.69,
+      "learning_rate": 7.883680337481599e-07,
+      "logits/chosen": -0.05966154858469963,
+      "logits/rejected": 0.014715162105858326,
+      "logps/chosen": -396.83026123046875,
+      "logps/rejected": -494.3729553222656,
+      "loss": 0.047,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -0.9806185960769653,
+      "rewards/margins": 0.8556007146835327,
+      "rewards/rejected": -1.8362191915512085,
       "step": 330
     },
     {
-      "epoch": 0.96,
-      "learning_rate": 1.6336997442095825e-08,
-      "logits/chosen": -0.2847253680229187,
-      "logits/rejected": -0.21411709487438202,
-      "logps/chosen": -375.3291015625,
-      "logps/rejected": -291.4599914550781,
-      "loss": 0.3118,
-      "rewards/accuracies": 0.550000011920929,
-      "rewards/chosen": -0.9054251909255981,
-      "rewards/margins": 0.24709534645080566,
-      "rewards/rejected": -1.1525206565856934,
+      "epoch": 0.71,
+      "learning_rate": 6.936646121293654e-07,
+      "logits/chosen": -0.10212980210781097,
+      "logits/rejected": -0.02083268202841282,
+      "logps/chosen": -382.59844970703125,
+      "logps/rejected": -440.49908447265625,
+      "loss": 0.0489,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -0.959019660949707,
+      "rewards/margins": 0.8376408815383911,
+      "rewards/rejected": -1.7966604232788086,
       "step": 340
     },
     {
-      "epoch": 0.98,
-      "learning_rate": 1.8181591531977737e-09,
-      "logits/chosen": -0.28506776690483093,
-      "logits/rejected": -0.27561822533607483,
-      "logps/chosen": -402.9325866699219,
-      "logps/rejected": -352.8019104003906,
-      "loss": 0.3253,
-      "rewards/accuracies": 0.6187499761581421,
-      "rewards/chosen": -0.9050580263137817,
-      "rewards/margins": 0.32159894704818726,
-      "rewards/rejected": -1.2266569137573242,
+      "epoch": 0.73,
+      "learning_rate": 6.032834097207889e-07,
+      "logits/chosen": -0.14827466011047363,
+      "logits/rejected": -0.1494934856891632,
+      "logps/chosen": -398.29461669921875,
+      "logps/rejected": -487.94012451171875,
+      "loss": 0.0474,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -0.9679247736930847,
+      "rewards/margins": 0.805729866027832,
+      "rewards/rejected": -1.7736546993255615,
       "step": 350
     },
+    {
+      "epoch": 0.75,
+      "learning_rate": 5.177088990820725e-07,
+      "logits/chosen": -0.19919797778129578,
+      "logits/rejected": -0.08585543930530548,
+      "logps/chosen": -423.6690979003906,
+      "logps/rejected": -448.3824768066406,
+      "loss": 0.0483,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -0.9551182985305786,
+      "rewards/margins": 0.7068794965744019,
+      "rewards/rejected": -1.6619977951049805,
+      "step": 360
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 4.3739978734594494e-07,
+      "logits/chosen": -0.19064149260520935,
+      "logits/rejected": -0.08522866666316986,
+      "logps/chosen": -448.2352600097656,
+      "logps/rejected": -489.485107421875,
+      "loss": 0.0495,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -0.9980375170707703,
+      "rewards/margins": 0.7906314134597778,
+      "rewards/rejected": -1.7886688709259033,
+      "step": 370
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 3.627865573992087e-07,
+      "logits/chosen": -0.13051238656044006,
+      "logits/rejected": -0.06193440407514572,
+      "logps/chosen": -398.67425537109375,
+      "logps/rejected": -444.31951904296875,
+      "loss": 0.0518,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -1.0560102462768555,
+      "rewards/margins": 0.6653419137001038,
+      "rewards/rejected": -1.721352219581604,
+      "step": 380
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 2.9426916035484166e-07,
+      "logits/chosen": -0.24809321761131287,
+      "logits/rejected": -0.13946378231048584,
+      "logps/chosen": -439.02069091796875,
+      "logps/rejected": -481.7464904785156,
+      "loss": 0.0461,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.9732930064201355,
+      "rewards/margins": 0.8343151807785034,
+      "rewards/rejected": -1.8076083660125732,
+      "step": 390
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 2.322148716843081e-07,
+      "logits/chosen": -0.17726832628250122,
+      "logits/rejected": -0.10749037563800812,
+      "logps/chosen": -414.79779052734375,
+      "logps/rejected": -467.64593505859375,
+      "loss": 0.0514,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.9779269099235535,
+      "rewards/margins": 0.7506555318832397,
+      "rewards/rejected": -1.7285826206207275,
+      "step": 400
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 1.7695632250191002e-07,
+      "logits/chosen": -0.2721072733402252,
+      "logits/rejected": -0.2248070240020752,
+      "logps/chosen": -403.7864685058594,
+      "logps/rejected": -474.9762268066406,
+      "loss": 0.0427,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -1.1458890438079834,
+      "rewards/margins": 0.6978408098220825,
+      "rewards/rejected": -1.8437299728393555,
+      "step": 410
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 1.2878971655412515e-07,
+      "logits/chosen": -0.15144512057304382,
+      "logits/rejected": -0.11361583322286606,
+      "logps/chosen": -396.65521240234375,
+      "logps/rejected": -473.33447265625,
+      "loss": 0.0429,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -1.0332143306732178,
+      "rewards/margins": 0.8435841798782349,
+      "rewards/rejected": -1.876798391342163,
+      "step": 420
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 8.797324247145411e-08,
+      "logits/chosen": -0.18158239126205444,
+      "logits/rejected": -0.16928087174892426,
+      "logps/chosen": -459.45928955078125,
+      "logps/rejected": -527.222412109375,
+      "loss": 0.0479,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -1.2538243532180786,
+      "rewards/margins": 0.7805837392807007,
+      "rewards/rejected": -2.0344080924987793,
+      "step": 430
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 5.472568979361853e-08,
+      "logits/chosen": -0.1764029711484909,
+      "logits/rejected": -0.1563466489315033,
+      "logps/chosen": -399.78094482421875,
+      "logps/rejected": -463.10284423828125,
+      "loss": 0.0473,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -1.1522372961044312,
+      "rewards/margins": 0.7010194063186646,
+      "rewards/rejected": -1.8532568216323853,
+      "step": 440
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 2.922527618666465e-08,
+      "logits/chosen": -0.23132136464118958,
+      "logits/rejected": -0.0833607167005539,
+      "logps/chosen": -476.85986328125,
+      "logps/rejected": -498.6736755371094,
+      "loss": 0.0442,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -1.269084095954895,
+      "rewards/margins": 0.8781474232673645,
+      "rewards/rejected": -2.1472315788269043,
+      "step": 450
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 1.1608692138469379e-08,
+      "logits/chosen": -0.17594662308692932,
+      "logits/rejected": 0.0026182211004197598,
+      "logps/chosen": -404.1844177246094,
+      "logps/rejected": -453.0740661621094,
+      "loss": 0.0409,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -1.0624682903289795,
+      "rewards/margins": 0.810712456703186,
+      "rewards/rejected": -1.8731807470321655,
+      "step": 460
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 1.970368253390198e-09,
+      "logits/chosen": -0.20819933712482452,
+      "logits/rejected": -0.09198556840419769,
+      "logps/chosen": -453.80291748046875,
+      "logps/rejected": -505.82733154296875,
+      "loss": 0.0462,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -1.1465378999710083,
+      "rewards/margins": 0.9369093179702759,
+      "rewards/rejected": -2.083447217941284,
+      "step": 470
+    },
     {
       "epoch": 1.0,
-      "step": 355,
+      "step": 477,
       "total_flos": 0.0,
-      "train_loss": 0.36347593488827556,
-      "train_runtime": 5281.0991,
-      "train_samples_per_second": 8.625,
-      "train_steps_per_second": 0.067
+      "train_loss": 0.06463673111027891,
+      "train_runtime": 6408.5161,
+      "train_samples_per_second": 9.539,
+      "train_steps_per_second": 0.074
     }
   ],
   "logging_steps": 10,
-  "max_steps": 355,
+  "max_steps": 477,
   "num_train_epochs": 1,
   "save_steps": 10000,
   "total_flos": 0.0,

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:75be8a76f25532ae2230808d2c77d6bec171d0619c7c964b4a7bc7c485ab42d1
+oid sha256:3bbe91eae1fd7d578def68ada6516b84b7a8f45cd2735b45f8b5198ffb913cb1
 size 6648