zephyr-7b-sft-full-SPIN-iter1 / trainer_state.json
ydeng9's picture
First model version
5b69325
raw
history blame
167 kB
{
"best_metric": null,
"best_model_checkpoint": null,
"epoch": 2.0,
"eval_steps": 100,
"global_step": 3112,
"is_hyper_param_search": false,
"is_local_process_zero": true,
"is_world_process_zero": true,
"log_history": [
{
"epoch": 0.0,
"learning_rate": 5.353319057815846e-10,
"logits/chosen": -2.322030782699585,
"logits/rejected": -2.360077381134033,
"logps/chosen": -413.0701599121094,
"logps/rejected": -503.9693603515625,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": 0.0,
"rewards/margins": 0.0,
"rewards/rejected": 0.0,
"step": 1
},
{
"epoch": 0.01,
"learning_rate": 5.353319057815845e-09,
"logits/chosen": -2.3369531631469727,
"logits/rejected": -2.352255344390869,
"logps/chosen": -334.3316650390625,
"logps/rejected": -329.3804016113281,
"loss": 0.6949,
"rewards/accuracies": 0.5416666865348816,
"rewards/chosen": 0.0067011509090662,
"rewards/margins": 0.011725478805601597,
"rewards/rejected": -0.005024327430874109,
"step": 10
},
{
"epoch": 0.01,
"learning_rate": 1.070663811563169e-08,
"logits/chosen": -2.307915210723877,
"logits/rejected": -2.3184759616851807,
"logps/chosen": -383.9498291015625,
"logps/rejected": -349.3071594238281,
"loss": 0.6939,
"rewards/accuracies": 0.4000000059604645,
"rewards/chosen": -0.020408455282449722,
"rewards/margins": -0.008849766105413437,
"rewards/rejected": -0.011558687314391136,
"step": 20
},
{
"epoch": 0.02,
"learning_rate": 1.6059957173447538e-08,
"logits/chosen": -2.3540706634521484,
"logits/rejected": -2.3323521614074707,
"logps/chosen": -382.1279602050781,
"logps/rejected": -429.32147216796875,
"loss": 0.6794,
"rewards/accuracies": 0.5625,
"rewards/chosen": -0.009235721081495285,
"rewards/margins": 0.022110218182206154,
"rewards/rejected": -0.03134594112634659,
"step": 30
},
{
"epoch": 0.03,
"learning_rate": 2.141327623126338e-08,
"logits/chosen": -2.281588315963745,
"logits/rejected": -2.2917075157165527,
"logps/chosen": -329.18243408203125,
"logps/rejected": -268.74761962890625,
"loss": 0.6573,
"rewards/accuracies": 0.762499988079071,
"rewards/chosen": 0.008301397785544395,
"rewards/margins": 0.07960359007120132,
"rewards/rejected": -0.07130218297243118,
"step": 40
},
{
"epoch": 0.03,
"learning_rate": 2.676659528907923e-08,
"logits/chosen": -2.2929608821868896,
"logits/rejected": -2.3293919563293457,
"logps/chosen": -348.1233215332031,
"logps/rejected": -352.0605163574219,
"loss": 0.6243,
"rewards/accuracies": 0.8374999761581421,
"rewards/chosen": 0.02957112155854702,
"rewards/margins": 0.16750425100326538,
"rewards/rejected": -0.1379331350326538,
"step": 50
},
{
"epoch": 0.04,
"learning_rate": 3.2119914346895076e-08,
"logits/chosen": -2.2743587493896484,
"logits/rejected": -2.271085262298584,
"logps/chosen": -369.2187805175781,
"logps/rejected": -364.81781005859375,
"loss": 0.5746,
"rewards/accuracies": 0.8500000238418579,
"rewards/chosen": 0.016968611627817154,
"rewards/margins": 0.25147581100463867,
"rewards/rejected": -0.23450717329978943,
"step": 60
},
{
"epoch": 0.04,
"learning_rate": 3.747323340471092e-08,
"logits/chosen": -2.3003923892974854,
"logits/rejected": -2.3059000968933105,
"logps/chosen": -358.39813232421875,
"logps/rejected": -377.95391845703125,
"loss": 0.5103,
"rewards/accuracies": 0.8500000238418579,
"rewards/chosen": 0.06770019978284836,
"rewards/margins": 0.510492205619812,
"rewards/rejected": -0.44279199838638306,
"step": 70
},
{
"epoch": 0.05,
"learning_rate": 4.282655246252676e-08,
"logits/chosen": -2.2736544609069824,
"logits/rejected": -2.2347025871276855,
"logps/chosen": -350.6732177734375,
"logps/rejected": -310.04315185546875,
"loss": 0.4697,
"rewards/accuracies": 0.862500011920929,
"rewards/chosen": 0.11150838434696198,
"rewards/margins": 0.5623964071273804,
"rewards/rejected": -0.45088809728622437,
"step": 80
},
{
"epoch": 0.06,
"learning_rate": 4.817987152034261e-08,
"logits/chosen": -2.2885279655456543,
"logits/rejected": -2.2864298820495605,
"logps/chosen": -376.4400939941406,
"logps/rejected": -356.20245361328125,
"loss": 0.4095,
"rewards/accuracies": 0.875,
"rewards/chosen": 0.13773031532764435,
"rewards/margins": 0.7758801579475403,
"rewards/rejected": -0.6381498575210571,
"step": 90
},
{
"epoch": 0.06,
"learning_rate": 5.353319057815846e-08,
"logits/chosen": -2.2083301544189453,
"logits/rejected": -2.191849946975708,
"logps/chosen": -388.324462890625,
"logps/rejected": -376.165771484375,
"loss": 0.3876,
"rewards/accuracies": 0.9125000238418579,
"rewards/chosen": 0.19459189474582672,
"rewards/margins": 0.9206008911132812,
"rewards/rejected": -0.7260090112686157,
"step": 100
},
{
"epoch": 0.06,
"eval_logits/chosen": -2.2498624324798584,
"eval_logits/rejected": -2.221994400024414,
"eval_logps/chosen": -328.4210510253906,
"eval_logps/rejected": -324.17828369140625,
"eval_loss": 0.3689849376678467,
"eval_rewards/accuracies": 0.8984375,
"eval_rewards/chosen": 0.0942004844546318,
"eval_rewards/margins": 0.8769543170928955,
"eval_rewards/rejected": -0.7827538251876831,
"eval_runtime": 76.5553,
"eval_samples_per_second": 13.062,
"eval_steps_per_second": 0.418,
"step": 100
},
{
"epoch": 0.07,
"learning_rate": 5.88865096359743e-08,
"logits/chosen": -2.2286324501037598,
"logits/rejected": -2.2091312408447266,
"logps/chosen": -372.5191650390625,
"logps/rejected": -399.0936584472656,
"loss": 0.3497,
"rewards/accuracies": 0.887499988079071,
"rewards/chosen": 0.1948019564151764,
"rewards/margins": 1.1746506690979004,
"rewards/rejected": -0.9798487424850464,
"step": 110
},
{
"epoch": 0.08,
"learning_rate": 6.423982869379015e-08,
"logits/chosen": -2.1091272830963135,
"logits/rejected": -2.118820905685425,
"logps/chosen": -361.56353759765625,
"logps/rejected": -404.4062805175781,
"loss": 0.3081,
"rewards/accuracies": 0.887499988079071,
"rewards/chosen": 0.11251312494277954,
"rewards/margins": 1.413537621498108,
"rewards/rejected": -1.3010244369506836,
"step": 120
},
{
"epoch": 0.08,
"learning_rate": 6.959314775160599e-08,
"logits/chosen": -2.1156742572784424,
"logits/rejected": -2.092258930206299,
"logps/chosen": -406.53851318359375,
"logps/rejected": -382.41717529296875,
"loss": 0.2846,
"rewards/accuracies": 0.9624999761581421,
"rewards/chosen": 0.18506963551044464,
"rewards/margins": 1.6774908304214478,
"rewards/rejected": -1.492421269416809,
"step": 130
},
{
"epoch": 0.09,
"learning_rate": 7.494646680942184e-08,
"logits/chosen": -2.092482089996338,
"logits/rejected": -2.0639469623565674,
"logps/chosen": -364.37774658203125,
"logps/rejected": -419.9878845214844,
"loss": 0.2908,
"rewards/accuracies": 0.925000011920929,
"rewards/chosen": 0.16728416085243225,
"rewards/margins": 1.9277279376983643,
"rewards/rejected": -1.760443925857544,
"step": 140
},
{
"epoch": 0.1,
"learning_rate": 8.029978586723767e-08,
"logits/chosen": -2.035637378692627,
"logits/rejected": -2.007483959197998,
"logps/chosen": -364.07745361328125,
"logps/rejected": -378.3775939941406,
"loss": 0.2519,
"rewards/accuracies": 0.949999988079071,
"rewards/chosen": -0.009305549785494804,
"rewards/margins": 1.7874408960342407,
"rewards/rejected": -1.7967464923858643,
"step": 150
},
{
"epoch": 0.1,
"learning_rate": 8.565310492505352e-08,
"logits/chosen": -1.9844402074813843,
"logits/rejected": -1.9815524816513062,
"logps/chosen": -384.30645751953125,
"logps/rejected": -454.4593811035156,
"loss": 0.2155,
"rewards/accuracies": 0.9375,
"rewards/chosen": -0.009780190885066986,
"rewards/margins": 2.3053088188171387,
"rewards/rejected": -2.315088987350464,
"step": 160
},
{
"epoch": 0.11,
"learning_rate": 9.100642398286937e-08,
"logits/chosen": -1.8912779092788696,
"logits/rejected": -1.8273910284042358,
"logps/chosen": -378.23199462890625,
"logps/rejected": -344.0122985839844,
"loss": 0.2089,
"rewards/accuracies": 0.9375,
"rewards/chosen": -0.13415472209453583,
"rewards/margins": 2.2053399085998535,
"rewards/rejected": -2.3394949436187744,
"step": 170
},
{
"epoch": 0.12,
"learning_rate": 9.635974304068522e-08,
"logits/chosen": -1.8499170541763306,
"logits/rejected": -1.8102867603302002,
"logps/chosen": -337.0228271484375,
"logps/rejected": -421.8775329589844,
"loss": 0.1974,
"rewards/accuracies": 0.9375,
"rewards/chosen": -0.24989835917949677,
"rewards/margins": 2.7448952198028564,
"rewards/rejected": -2.99479341506958,
"step": 180
},
{
"epoch": 0.12,
"learning_rate": 1.0171306209850107e-07,
"logits/chosen": -1.8178815841674805,
"logits/rejected": -1.8075618743896484,
"logps/chosen": -420.0069885253906,
"logps/rejected": -451.9561462402344,
"loss": 0.1646,
"rewards/accuracies": 0.9375,
"rewards/chosen": -0.2596689462661743,
"rewards/margins": 2.7779135704040527,
"rewards/rejected": -3.0375826358795166,
"step": 190
},
{
"epoch": 0.13,
"learning_rate": 1.0706638115631692e-07,
"logits/chosen": -1.8774926662445068,
"logits/rejected": -1.7711089849472046,
"logps/chosen": -368.1988830566406,
"logps/rejected": -364.1625671386719,
"loss": 0.1791,
"rewards/accuracies": 0.949999988079071,
"rewards/chosen": -0.20116904377937317,
"rewards/margins": 3.0269274711608887,
"rewards/rejected": -3.2280964851379395,
"step": 200
},
{
"epoch": 0.13,
"eval_logits/chosen": -1.9170525074005127,
"eval_logits/rejected": -1.8318464756011963,
"eval_logps/chosen": -330.80859375,
"eval_logps/rejected": -343.30322265625,
"eval_loss": 0.19462376832962036,
"eval_rewards/accuracies": 0.90625,
"eval_rewards/chosen": -0.14455503225326538,
"eval_rewards/margins": 2.5506958961486816,
"eval_rewards/rejected": -2.695250988006592,
"eval_runtime": 76.4342,
"eval_samples_per_second": 13.083,
"eval_steps_per_second": 0.419,
"step": 200
},
{
"epoch": 0.13,
"learning_rate": 1.1241970021413276e-07,
"logits/chosen": -1.870164155960083,
"logits/rejected": -1.7529096603393555,
"logps/chosen": -394.24310302734375,
"logps/rejected": -359.798828125,
"loss": 0.1689,
"rewards/accuracies": 0.949999988079071,
"rewards/chosen": -0.23141837120056152,
"rewards/margins": 2.955934524536133,
"rewards/rejected": -3.1873526573181152,
"step": 210
},
{
"epoch": 0.14,
"learning_rate": 1.177730192719486e-07,
"logits/chosen": -1.7515084743499756,
"logits/rejected": -1.6350901126861572,
"logps/chosen": -399.573486328125,
"logps/rejected": -395.66864013671875,
"loss": 0.1476,
"rewards/accuracies": 0.949999988079071,
"rewards/chosen": -0.43568697571754456,
"rewards/margins": 3.202346086502075,
"rewards/rejected": -3.638033390045166,
"step": 220
},
{
"epoch": 0.15,
"learning_rate": 1.2312633832976445e-07,
"logits/chosen": -1.7598545551300049,
"logits/rejected": -1.6209675073623657,
"logps/chosen": -397.47747802734375,
"logps/rejected": -407.18780517578125,
"loss": 0.1534,
"rewards/accuracies": 0.949999988079071,
"rewards/chosen": -0.2008744478225708,
"rewards/margins": 3.397059917449951,
"rewards/rejected": -3.5979347229003906,
"step": 230
},
{
"epoch": 0.15,
"learning_rate": 1.284796573875803e-07,
"logits/chosen": -1.7311824560165405,
"logits/rejected": -1.6193158626556396,
"logps/chosen": -379.2996520996094,
"logps/rejected": -434.6549377441406,
"loss": 0.136,
"rewards/accuracies": 0.9375,
"rewards/chosen": -0.4003673493862152,
"rewards/margins": 3.5797741413116455,
"rewards/rejected": -3.9801411628723145,
"step": 240
},
{
"epoch": 0.16,
"learning_rate": 1.3383297644539615e-07,
"logits/chosen": -1.7749998569488525,
"logits/rejected": -1.6336084604263306,
"logps/chosen": -384.36016845703125,
"logps/rejected": -412.198974609375,
"loss": 0.1424,
"rewards/accuracies": 0.9375,
"rewards/chosen": -0.44941583275794983,
"rewards/margins": 4.18085241317749,
"rewards/rejected": -4.630268096923828,
"step": 250
},
{
"epoch": 0.17,
"learning_rate": 1.3918629550321198e-07,
"logits/chosen": -1.6577208042144775,
"logits/rejected": -1.4876978397369385,
"logps/chosen": -368.1956481933594,
"logps/rejected": -311.98223876953125,
"loss": 0.1379,
"rewards/accuracies": 0.9750000238418579,
"rewards/chosen": -0.5439226031303406,
"rewards/margins": 3.506497621536255,
"rewards/rejected": -4.050419807434082,
"step": 260
},
{
"epoch": 0.17,
"learning_rate": 1.4453961456102785e-07,
"logits/chosen": -1.6528087854385376,
"logits/rejected": -1.5587131977081299,
"logps/chosen": -357.02655029296875,
"logps/rejected": -449.92718505859375,
"loss": 0.1273,
"rewards/accuracies": 0.9375,
"rewards/chosen": -0.7018794417381287,
"rewards/margins": 4.135760307312012,
"rewards/rejected": -4.837639808654785,
"step": 270
},
{
"epoch": 0.18,
"learning_rate": 1.4989293361884367e-07,
"logits/chosen": -1.6728260517120361,
"logits/rejected": -1.4406911134719849,
"logps/chosen": -383.44635009765625,
"logps/rejected": -405.33428955078125,
"loss": 0.1286,
"rewards/accuracies": 0.9125000238418579,
"rewards/chosen": -0.7234546542167664,
"rewards/margins": 4.43070125579834,
"rewards/rejected": -5.154156684875488,
"step": 280
},
{
"epoch": 0.19,
"learning_rate": 1.5524625267665952e-07,
"logits/chosen": -1.5811737775802612,
"logits/rejected": -1.4415075778961182,
"logps/chosen": -380.16302490234375,
"logps/rejected": -385.4619140625,
"loss": 0.1184,
"rewards/accuracies": 0.949999988079071,
"rewards/chosen": -0.6038004159927368,
"rewards/margins": 4.522972106933594,
"rewards/rejected": -5.126772403717041,
"step": 290
},
{
"epoch": 0.19,
"learning_rate": 1.6059957173447535e-07,
"logits/chosen": -1.5964694023132324,
"logits/rejected": -1.5073761940002441,
"logps/chosen": -348.82110595703125,
"logps/rejected": -396.21234130859375,
"loss": 0.1218,
"rewards/accuracies": 0.949999988079071,
"rewards/chosen": -0.7199587821960449,
"rewards/margins": 4.322809219360352,
"rewards/rejected": -5.042768478393555,
"step": 300
},
{
"epoch": 0.19,
"eval_logits/chosen": -1.7016242742538452,
"eval_logits/rejected": -1.5659476518630981,
"eval_logps/chosen": -336.16741943359375,
"eval_logps/rejected": -363.2919006347656,
"eval_loss": 0.1464938521385193,
"eval_rewards/accuracies": 0.921875,
"eval_rewards/chosen": -0.6804376840591431,
"eval_rewards/margins": 4.013677597045898,
"eval_rewards/rejected": -4.69411563873291,
"eval_runtime": 76.4822,
"eval_samples_per_second": 13.075,
"eval_steps_per_second": 0.418,
"step": 300
},
{
"epoch": 0.2,
"learning_rate": 1.6595289079229122e-07,
"logits/chosen": -1.5078377723693848,
"logits/rejected": -1.4056973457336426,
"logps/chosen": -412.2068786621094,
"logps/rejected": -462.7405700683594,
"loss": 0.1094,
"rewards/accuracies": 0.987500011920929,
"rewards/chosen": -0.718084454536438,
"rewards/margins": 5.0283284187316895,
"rewards/rejected": -5.746413230895996,
"step": 310
},
{
"epoch": 0.21,
"learning_rate": 1.7130620985010704e-07,
"logits/chosen": -1.586196780204773,
"logits/rejected": -1.4498493671417236,
"logps/chosen": -351.50775146484375,
"logps/rejected": -403.0026550292969,
"loss": 0.1016,
"rewards/accuracies": 0.9624999761581421,
"rewards/chosen": -0.6627265214920044,
"rewards/margins": 4.463755130767822,
"rewards/rejected": -5.126482009887695,
"step": 320
},
{
"epoch": 0.21,
"learning_rate": 1.766595289079229e-07,
"logits/chosen": -1.5722558498382568,
"logits/rejected": -1.340435266494751,
"logps/chosen": -337.4762878417969,
"logps/rejected": -419.3818359375,
"loss": 0.1221,
"rewards/accuracies": 0.9624999761581421,
"rewards/chosen": -0.869371235370636,
"rewards/margins": 5.570626258850098,
"rewards/rejected": -6.439997673034668,
"step": 330
},
{
"epoch": 0.22,
"learning_rate": 1.8201284796573874e-07,
"logits/chosen": -1.563241958618164,
"logits/rejected": -1.43202805519104,
"logps/chosen": -332.5054016113281,
"logps/rejected": -401.9011535644531,
"loss": 0.1141,
"rewards/accuracies": 0.949999988079071,
"rewards/chosen": -0.7410942912101746,
"rewards/margins": 5.449254512786865,
"rewards/rejected": -6.1903486251831055,
"step": 340
},
{
"epoch": 0.22,
"learning_rate": 1.873661670235546e-07,
"logits/chosen": -1.509913682937622,
"logits/rejected": -1.352683186531067,
"logps/chosen": -428.9737243652344,
"logps/rejected": -455.664306640625,
"loss": 0.1092,
"rewards/accuracies": 0.9624999761581421,
"rewards/chosen": -0.7532263398170471,
"rewards/margins": 5.451329708099365,
"rewards/rejected": -6.204555988311768,
"step": 350
},
{
"epoch": 0.23,
"learning_rate": 1.9271948608137044e-07,
"logits/chosen": -1.6219438314437866,
"logits/rejected": -1.3939545154571533,
"logps/chosen": -389.11297607421875,
"logps/rejected": -411.80035400390625,
"loss": 0.1218,
"rewards/accuracies": 0.9375,
"rewards/chosen": -1.0884325504302979,
"rewards/margins": 5.317915439605713,
"rewards/rejected": -6.406347751617432,
"step": 360
},
{
"epoch": 0.24,
"learning_rate": 1.980728051391863e-07,
"logits/chosen": -1.465427041053772,
"logits/rejected": -1.3383185863494873,
"logps/chosen": -373.2828369140625,
"logps/rejected": -433.7022399902344,
"loss": 0.1098,
"rewards/accuracies": 0.9624999761581421,
"rewards/chosen": -1.5387589931488037,
"rewards/margins": 5.308978080749512,
"rewards/rejected": -6.8477373123168945,
"step": 370
},
{
"epoch": 0.24,
"learning_rate": 2.0342612419700214e-07,
"logits/chosen": -1.4776766300201416,
"logits/rejected": -1.390604019165039,
"logps/chosen": -384.6776428222656,
"logps/rejected": -409.8622131347656,
"loss": 0.1068,
"rewards/accuracies": 0.9624999761581421,
"rewards/chosen": -1.1275439262390137,
"rewards/margins": 5.649918079376221,
"rewards/rejected": -6.777462005615234,
"step": 380
},
{
"epoch": 0.25,
"learning_rate": 2.0877944325481796e-07,
"logits/chosen": -1.496004343032837,
"logits/rejected": -1.2643308639526367,
"logps/chosen": -319.6006774902344,
"logps/rejected": -435.0767517089844,
"loss": 0.0963,
"rewards/accuracies": 0.9750000238418579,
"rewards/chosen": -0.9759249687194824,
"rewards/margins": 5.894278526306152,
"rewards/rejected": -6.870204925537109,
"step": 390
},
{
"epoch": 0.26,
"learning_rate": 2.1413276231263384e-07,
"logits/chosen": -1.373808741569519,
"logits/rejected": -1.2059423923492432,
"logps/chosen": -355.2186584472656,
"logps/rejected": -375.163330078125,
"loss": 0.1065,
"rewards/accuracies": 0.949999988079071,
"rewards/chosen": -1.117621660232544,
"rewards/margins": 5.132685661315918,
"rewards/rejected": -6.250307559967041,
"step": 400
},
{
"epoch": 0.26,
"eval_logits/chosen": -1.5739349126815796,
"eval_logits/rejected": -1.3864831924438477,
"eval_logps/chosen": -342.44061279296875,
"eval_logps/rejected": -380.4227600097656,
"eval_loss": 0.1241927221417427,
"eval_rewards/accuracies": 0.921875,
"eval_rewards/chosen": -1.3077539205551147,
"eval_rewards/margins": 5.099446773529053,
"eval_rewards/rejected": -6.407200336456299,
"eval_runtime": 76.309,
"eval_samples_per_second": 13.105,
"eval_steps_per_second": 0.419,
"step": 400
},
{
"epoch": 0.26,
"learning_rate": 2.1948608137044966e-07,
"logits/chosen": -1.453360915184021,
"logits/rejected": -1.215315341949463,
"logps/chosen": -309.32196044921875,
"logps/rejected": -383.0122985839844,
"loss": 0.0932,
"rewards/accuracies": 0.9375,
"rewards/chosen": -1.0941340923309326,
"rewards/margins": 5.864490985870361,
"rewards/rejected": -6.958624839782715,
"step": 410
},
{
"epoch": 0.27,
"learning_rate": 2.248394004282655e-07,
"logits/chosen": -1.6208438873291016,
"logits/rejected": -1.3579902648925781,
"logps/chosen": -420.99951171875,
"logps/rejected": -458.4291076660156,
"loss": 0.0972,
"rewards/accuracies": 0.9750000238418579,
"rewards/chosen": -0.9325690269470215,
"rewards/margins": 6.371849536895752,
"rewards/rejected": -7.304419040679932,
"step": 420
},
{
"epoch": 0.28,
"learning_rate": 2.3019271948608136e-07,
"logits/chosen": -1.5590957403182983,
"logits/rejected": -1.3185454607009888,
"logps/chosen": -433.14007568359375,
"logps/rejected": -423.8736877441406,
"loss": 0.0955,
"rewards/accuracies": 0.9375,
"rewards/chosen": -0.888096034526825,
"rewards/margins": 6.284546852111816,
"rewards/rejected": -7.172643184661865,
"step": 430
},
{
"epoch": 0.28,
"learning_rate": 2.355460385438972e-07,
"logits/chosen": -1.5379421710968018,
"logits/rejected": -1.287793755531311,
"logps/chosen": -384.4920959472656,
"logps/rejected": -459.51495361328125,
"loss": 0.0925,
"rewards/accuracies": 0.9750000238418579,
"rewards/chosen": -0.9522300958633423,
"rewards/margins": 6.38286828994751,
"rewards/rejected": -7.3350982666015625,
"step": 440
},
{
"epoch": 0.29,
"learning_rate": 2.4089935760171303e-07,
"logits/chosen": -1.539567232131958,
"logits/rejected": -1.2881194353103638,
"logps/chosen": -360.4696960449219,
"logps/rejected": -421.70953369140625,
"loss": 0.0983,
"rewards/accuracies": 0.949999988079071,
"rewards/chosen": -1.032198190689087,
"rewards/margins": 6.628798484802246,
"rewards/rejected": -7.660996437072754,
"step": 450
},
{
"epoch": 0.3,
"learning_rate": 2.462526766595289e-07,
"logits/chosen": -1.5270180702209473,
"logits/rejected": -1.213521122932434,
"logps/chosen": -341.48370361328125,
"logps/rejected": -416.1814880371094,
"loss": 0.0862,
"rewards/accuracies": 0.925000011920929,
"rewards/chosen": -1.2647520303726196,
"rewards/margins": 7.327714443206787,
"rewards/rejected": -8.592466354370117,
"step": 460
},
{
"epoch": 0.3,
"learning_rate": 2.5160599571734473e-07,
"logits/chosen": -1.539104700088501,
"logits/rejected": -1.304837942123413,
"logps/chosen": -412.41192626953125,
"logps/rejected": -530.3709106445312,
"loss": 0.0685,
"rewards/accuracies": 0.9750000238418579,
"rewards/chosen": -1.0689477920532227,
"rewards/margins": 7.198890686035156,
"rewards/rejected": -8.267837524414062,
"step": 470
},
{
"epoch": 0.31,
"learning_rate": 2.569593147751606e-07,
"logits/chosen": -1.4452335834503174,
"logits/rejected": -1.177215576171875,
"logps/chosen": -356.6268005371094,
"logps/rejected": -386.1933288574219,
"loss": 0.0906,
"rewards/accuracies": 0.987500011920929,
"rewards/chosen": -0.5285716652870178,
"rewards/margins": 7.398108005523682,
"rewards/rejected": -7.9266791343688965,
"step": 480
},
{
"epoch": 0.31,
"learning_rate": 2.6231263383297643e-07,
"logits/chosen": -1.299133062362671,
"logits/rejected": -1.2396031618118286,
"logps/chosen": -354.5841979980469,
"logps/rejected": -451.559326171875,
"loss": 0.0801,
"rewards/accuracies": 0.9750000238418579,
"rewards/chosen": -0.7138751745223999,
"rewards/margins": 6.743910789489746,
"rewards/rejected": -7.457786560058594,
"step": 490
},
{
"epoch": 0.32,
"learning_rate": 2.676659528907923e-07,
"logits/chosen": -1.2933090925216675,
"logits/rejected": -1.0351635217666626,
"logps/chosen": -403.50091552734375,
"logps/rejected": -444.50811767578125,
"loss": 0.0804,
"rewards/accuracies": 1.0,
"rewards/chosen": -1.2215214967727661,
"rewards/margins": 7.684378623962402,
"rewards/rejected": -8.905900955200195,
"step": 500
},
{
"epoch": 0.32,
"eval_logits/chosen": -1.5204213857650757,
"eval_logits/rejected": -1.313443899154663,
"eval_logps/chosen": -341.24200439453125,
"eval_logps/rejected": -387.53155517578125,
"eval_loss": 0.11293376982212067,
"eval_rewards/accuracies": 0.9296875,
"eval_rewards/chosen": -1.1878938674926758,
"eval_rewards/margins": 5.9301862716674805,
"eval_rewards/rejected": -7.118079662322998,
"eval_runtime": 76.7239,
"eval_samples_per_second": 13.034,
"eval_steps_per_second": 0.417,
"step": 500
},
{
"epoch": 0.33,
"learning_rate": 2.7301927194860813e-07,
"logits/chosen": -1.4917323589324951,
"logits/rejected": -1.2333643436431885,
"logps/chosen": -391.20941162109375,
"logps/rejected": -462.3330993652344,
"loss": 0.0677,
"rewards/accuracies": 0.987500011920929,
"rewards/chosen": -0.9355085492134094,
"rewards/margins": 8.421406745910645,
"rewards/rejected": -9.356914520263672,
"step": 510
},
{
"epoch": 0.33,
"learning_rate": 2.7837259100642395e-07,
"logits/chosen": -1.3520570993423462,
"logits/rejected": -1.1625728607177734,
"logps/chosen": -392.84210205078125,
"logps/rejected": -441.2123107910156,
"loss": 0.084,
"rewards/accuracies": 0.987500011920929,
"rewards/chosen": -1.286471962928772,
"rewards/margins": 7.407790184020996,
"rewards/rejected": -8.69426155090332,
"step": 520
},
{
"epoch": 0.34,
"learning_rate": 2.8372591006423977e-07,
"logits/chosen": -1.2561280727386475,
"logits/rejected": -1.0563820600509644,
"logps/chosen": -355.0276794433594,
"logps/rejected": -434.96099853515625,
"loss": 0.0754,
"rewards/accuracies": 0.925000011920929,
"rewards/chosen": -1.644857406616211,
"rewards/margins": 7.365194797515869,
"rewards/rejected": -9.010051727294922,
"step": 530
},
{
"epoch": 0.35,
"learning_rate": 2.890792291220557e-07,
"logits/chosen": -1.3786545991897583,
"logits/rejected": -1.1711790561676025,
"logps/chosen": -343.2646179199219,
"logps/rejected": -414.624267578125,
"loss": 0.0644,
"rewards/accuracies": 0.9624999761581421,
"rewards/chosen": -1.0238958597183228,
"rewards/margins": 7.4112067222595215,
"rewards/rejected": -8.435102462768555,
"step": 540
},
{
"epoch": 0.35,
"learning_rate": 2.944325481798715e-07,
"logits/chosen": -1.483944296836853,
"logits/rejected": -1.2009809017181396,
"logps/chosen": -397.4231872558594,
"logps/rejected": -453.69903564453125,
"loss": 0.0742,
"rewards/accuracies": 0.987500011920929,
"rewards/chosen": -0.842347264289856,
"rewards/margins": 7.386562347412109,
"rewards/rejected": -8.228910446166992,
"step": 550
},
{
"epoch": 0.36,
"learning_rate": 2.9978586723768735e-07,
"logits/chosen": -1.4178438186645508,
"logits/rejected": -1.243758201599121,
"logps/chosen": -412.52105712890625,
"logps/rejected": -464.28741455078125,
"loss": 0.0944,
"rewards/accuracies": 0.949999988079071,
"rewards/chosen": -1.449329137802124,
"rewards/margins": 8.025833129882812,
"rewards/rejected": -9.475163459777832,
"step": 560
},
{
"epoch": 0.37,
"learning_rate": 3.051391862955032e-07,
"logits/chosen": -1.3710880279541016,
"logits/rejected": -1.218766450881958,
"logps/chosen": -344.7862548828125,
"logps/rejected": -453.5824279785156,
"loss": 0.0435,
"rewards/accuracies": 0.987500011920929,
"rewards/chosen": -1.8545820713043213,
"rewards/margins": 7.798386573791504,
"rewards/rejected": -9.652968406677246,
"step": 570
},
{
"epoch": 0.37,
"learning_rate": 3.1049250535331905e-07,
"logits/chosen": -1.461828589439392,
"logits/rejected": -1.0271979570388794,
"logps/chosen": -387.4298095703125,
"logps/rejected": -435.98663330078125,
"loss": 0.0867,
"rewards/accuracies": 0.987500011920929,
"rewards/chosen": -1.7421836853027344,
"rewards/margins": 7.6707444190979,
"rewards/rejected": -9.412927627563477,
"step": 580
},
{
"epoch": 0.38,
"learning_rate": 3.1584582441113487e-07,
"logits/chosen": -1.3038650751113892,
"logits/rejected": -1.07460618019104,
"logps/chosen": -324.758544921875,
"logps/rejected": -503.78387451171875,
"loss": 0.0716,
"rewards/accuracies": 0.9375,
"rewards/chosen": -2.3957910537719727,
"rewards/margins": 8.855663299560547,
"rewards/rejected": -11.25145435333252,
"step": 590
},
{
"epoch": 0.39,
"learning_rate": 3.211991434689507e-07,
"logits/chosen": -1.3503267765045166,
"logits/rejected": -1.0638809204101562,
"logps/chosen": -374.38348388671875,
"logps/rejected": -516.7820434570312,
"loss": 0.0767,
"rewards/accuracies": 0.9624999761581421,
"rewards/chosen": -2.1524558067321777,
"rewards/margins": 8.407071113586426,
"rewards/rejected": -10.559527397155762,
"step": 600
},
{
"epoch": 0.39,
"eval_logits/chosen": -1.4297420978546143,
"eval_logits/rejected": -1.150868535041809,
"eval_logps/chosen": -353.238037109375,
"eval_logps/rejected": -407.5989990234375,
"eval_loss": 0.13097576797008514,
"eval_rewards/accuracies": 0.8984375,
"eval_rewards/chosen": -2.387495994567871,
"eval_rewards/margins": 6.737332820892334,
"eval_rewards/rejected": -9.124829292297363,
"eval_runtime": 76.6752,
"eval_samples_per_second": 13.042,
"eval_steps_per_second": 0.417,
"step": 600
},
{
"epoch": 0.39,
"learning_rate": 3.265524625267666e-07,
"logits/chosen": -1.1903326511383057,
"logits/rejected": -0.9525307416915894,
"logps/chosen": -419.54608154296875,
"logps/rejected": -444.11383056640625,
"loss": 0.0831,
"rewards/accuracies": 0.949999988079071,
"rewards/chosen": -2.369199752807617,
"rewards/margins": 8.518608093261719,
"rewards/rejected": -10.887807846069336,
"step": 610
},
{
"epoch": 0.4,
"learning_rate": 3.3190578158458244e-07,
"logits/chosen": -1.3438398838043213,
"logits/rejected": -1.143065333366394,
"logps/chosen": -369.13970947265625,
"logps/rejected": -439.51715087890625,
"loss": 0.0779,
"rewards/accuracies": 0.9750000238418579,
"rewards/chosen": -1.6147950887680054,
"rewards/margins": 7.42797327041626,
"rewards/rejected": -9.042768478393555,
"step": 620
},
{
"epoch": 0.4,
"learning_rate": 3.3725910064239827e-07,
"logits/chosen": -1.4843438863754272,
"logits/rejected": -1.0890620946884155,
"logps/chosen": -409.73583984375,
"logps/rejected": -517.4986572265625,
"loss": 0.0703,
"rewards/accuracies": 1.0,
"rewards/chosen": -1.1902521848678589,
"rewards/margins": 9.822367668151855,
"rewards/rejected": -11.01262092590332,
"step": 630
},
{
"epoch": 0.41,
"learning_rate": 3.426124197002141e-07,
"logits/chosen": -1.3501672744750977,
"logits/rejected": -1.0996173620224,
"logps/chosen": -360.51531982421875,
"logps/rejected": -489.69061279296875,
"loss": 0.0968,
"rewards/accuracies": 0.9750000238418579,
"rewards/chosen": -1.6506973505020142,
"rewards/margins": 8.347593307495117,
"rewards/rejected": -9.998289108276367,
"step": 640
},
{
"epoch": 0.42,
"learning_rate": 3.4796573875802996e-07,
"logits/chosen": -1.2105796337127686,
"logits/rejected": -0.9770170450210571,
"logps/chosen": -335.6686096191406,
"logps/rejected": -449.52447509765625,
"loss": 0.0659,
"rewards/accuracies": 0.9624999761581421,
"rewards/chosen": -2.2099552154541016,
"rewards/margins": 7.764392852783203,
"rewards/rejected": -9.974349021911621,
"step": 650
},
{
"epoch": 0.42,
"learning_rate": 3.533190578158458e-07,
"logits/chosen": -1.4429116249084473,
"logits/rejected": -1.06112539768219,
"logps/chosen": -420.8074645996094,
"logps/rejected": -442.88568115234375,
"loss": 0.0967,
"rewards/accuracies": 0.9624999761581421,
"rewards/chosen": -1.681353211402893,
"rewards/margins": 7.297033786773682,
"rewards/rejected": -8.978387832641602,
"step": 660
},
{
"epoch": 0.43,
"learning_rate": 3.5867237687366166e-07,
"logits/chosen": -1.4578073024749756,
"logits/rejected": -1.1529022455215454,
"logps/chosen": -362.67230224609375,
"logps/rejected": -455.08367919921875,
"loss": 0.0694,
"rewards/accuracies": 0.9624999761581421,
"rewards/chosen": -1.878594994544983,
"rewards/margins": 8.333452224731445,
"rewards/rejected": -10.212045669555664,
"step": 670
},
{
"epoch": 0.44,
"learning_rate": 3.640256959314775e-07,
"logits/chosen": -1.34915292263031,
"logits/rejected": -1.125140905380249,
"logps/chosen": -364.0298156738281,
"logps/rejected": -492.75909423828125,
"loss": 0.0744,
"rewards/accuracies": 0.9750000238418579,
"rewards/chosen": -2.043875217437744,
"rewards/margins": 9.902002334594727,
"rewards/rejected": -11.945877075195312,
"step": 680
},
{
"epoch": 0.44,
"learning_rate": 3.6937901498929336e-07,
"logits/chosen": -1.2846548557281494,
"logits/rejected": -0.9907848238945007,
"logps/chosen": -423.0797424316406,
"logps/rejected": -499.71142578125,
"loss": 0.0603,
"rewards/accuracies": 1.0,
"rewards/chosen": -2.9811477661132812,
"rewards/margins": 9.256712913513184,
"rewards/rejected": -12.237860679626465,
"step": 690
},
{
"epoch": 0.45,
"learning_rate": 3.747323340471092e-07,
"logits/chosen": -1.3538507223129272,
"logits/rejected": -1.1379244327545166,
"logps/chosen": -329.5379638671875,
"logps/rejected": -413.8023986816406,
"loss": 0.0759,
"rewards/accuracies": 0.9750000238418579,
"rewards/chosen": -2.2614781856536865,
"rewards/margins": 8.729515075683594,
"rewards/rejected": -10.990991592407227,
"step": 700
},
{
"epoch": 0.45,
"eval_logits/chosen": -1.4886287450790405,
"eval_logits/rejected": -1.2180323600769043,
"eval_logps/chosen": -348.9230041503906,
"eval_logps/rejected": -409.6312255859375,
"eval_loss": 0.12044651806354523,
"eval_rewards/accuracies": 0.9296875,
"eval_rewards/chosen": -1.9559952020645142,
"eval_rewards/margins": 7.372057914733887,
"eval_rewards/rejected": -9.328052520751953,
"eval_runtime": 76.578,
"eval_samples_per_second": 13.059,
"eval_steps_per_second": 0.418,
"step": 700
},
{
"epoch": 0.46,
"learning_rate": 3.80085653104925e-07,
"logits/chosen": -1.3313727378845215,
"logits/rejected": -0.9123377799987793,
"logps/chosen": -442.8067321777344,
"logps/rejected": -468.52984619140625,
"loss": 0.0905,
"rewards/accuracies": 0.9375,
"rewards/chosen": -1.6124906539916992,
"rewards/margins": 8.923405647277832,
"rewards/rejected": -10.535896301269531,
"step": 710
},
{
"epoch": 0.46,
"learning_rate": 3.854389721627409e-07,
"logits/chosen": -1.4308016300201416,
"logits/rejected": -1.0905249118804932,
"logps/chosen": -378.02545166015625,
"logps/rejected": -479.7505798339844,
"loss": 0.0561,
"rewards/accuracies": 0.9750000238418579,
"rewards/chosen": -1.7049999237060547,
"rewards/margins": 8.536346435546875,
"rewards/rejected": -10.241347312927246,
"step": 720
},
{
"epoch": 0.47,
"learning_rate": 3.9079229122055676e-07,
"logits/chosen": -1.3243951797485352,
"logits/rejected": -0.9832109212875366,
"logps/chosen": -431.05645751953125,
"logps/rejected": -432.52886962890625,
"loss": 0.0581,
"rewards/accuracies": 0.9624999761581421,
"rewards/chosen": -2.067716121673584,
"rewards/margins": 8.114839553833008,
"rewards/rejected": -10.18255615234375,
"step": 730
},
{
"epoch": 0.48,
"learning_rate": 3.961456102783726e-07,
"logits/chosen": -1.258172869682312,
"logits/rejected": -1.0175604820251465,
"logps/chosen": -404.3645935058594,
"logps/rejected": -519.955322265625,
"loss": 0.0452,
"rewards/accuracies": 1.0,
"rewards/chosen": -1.6845811605453491,
"rewards/margins": 10.622332572937012,
"rewards/rejected": -12.306914329528809,
"step": 740
},
{
"epoch": 0.48,
"learning_rate": 4.014989293361884e-07,
"logits/chosen": -1.28511381149292,
"logits/rejected": -1.0871832370758057,
"logps/chosen": -378.40478515625,
"logps/rejected": -460.6148376464844,
"loss": 0.0669,
"rewards/accuracies": 0.9750000238418579,
"rewards/chosen": -1.8259923458099365,
"rewards/margins": 8.321816444396973,
"rewards/rejected": -10.147809982299805,
"step": 750
},
{
"epoch": 0.49,
"learning_rate": 4.068522483940043e-07,
"logits/chosen": -1.5064611434936523,
"logits/rejected": -1.0779759883880615,
"logps/chosen": -371.971435546875,
"logps/rejected": -416.6922302246094,
"loss": 0.0762,
"rewards/accuracies": 0.9624999761581421,
"rewards/chosen": -1.135917067527771,
"rewards/margins": 10.22150993347168,
"rewards/rejected": -11.357427597045898,
"step": 760
},
{
"epoch": 0.49,
"learning_rate": 4.122055674518201e-07,
"logits/chosen": -1.429022192955017,
"logits/rejected": -1.0790882110595703,
"logps/chosen": -394.6370544433594,
"logps/rejected": -461.9344787597656,
"loss": 0.0662,
"rewards/accuracies": 0.987500011920929,
"rewards/chosen": -2.259974956512451,
"rewards/margins": 9.807271003723145,
"rewards/rejected": -12.067245483398438,
"step": 770
},
{
"epoch": 0.5,
"learning_rate": 4.175588865096359e-07,
"logits/chosen": -1.296064853668213,
"logits/rejected": -0.9478403925895691,
"logps/chosen": -376.76727294921875,
"logps/rejected": -470.43743896484375,
"loss": 0.0513,
"rewards/accuracies": 0.9750000238418579,
"rewards/chosen": -1.549682378768921,
"rewards/margins": 9.523083686828613,
"rewards/rejected": -11.072766304016113,
"step": 780
},
{
"epoch": 0.51,
"learning_rate": 4.2291220556745175e-07,
"logits/chosen": -1.309073567390442,
"logits/rejected": -0.8560987710952759,
"logps/chosen": -457.4974670410156,
"logps/rejected": -528.8920288085938,
"loss": 0.0616,
"rewards/accuracies": 0.9750000238418579,
"rewards/chosen": -2.5288264751434326,
"rewards/margins": 9.966497421264648,
"rewards/rejected": -12.495325088500977,
"step": 790
},
{
"epoch": 0.51,
"learning_rate": 4.282655246252677e-07,
"logits/chosen": -1.2481553554534912,
"logits/rejected": -0.764441728591919,
"logps/chosen": -306.9283142089844,
"logps/rejected": -417.22491455078125,
"loss": 0.0681,
"rewards/accuracies": 1.0,
"rewards/chosen": -1.760056734085083,
"rewards/margins": 10.229304313659668,
"rewards/rejected": -11.989361763000488,
"step": 800
},
{
"epoch": 0.51,
"eval_logits/chosen": -1.4168956279754639,
"eval_logits/rejected": -1.0999643802642822,
"eval_logps/chosen": -350.21661376953125,
"eval_logps/rejected": -414.49041748046875,
"eval_loss": 0.1091269925236702,
"eval_rewards/accuracies": 0.9609375,
"eval_rewards/chosen": -2.085355281829834,
"eval_rewards/margins": 7.728612422943115,
"eval_rewards/rejected": -9.81396770477295,
"eval_runtime": 76.655,
"eval_samples_per_second": 13.045,
"eval_steps_per_second": 0.417,
"step": 800
},
{
"epoch": 0.52,
"learning_rate": 4.336188436830835e-07,
"logits/chosen": -1.092874526977539,
"logits/rejected": -0.8753455281257629,
"logps/chosen": -367.61846923828125,
"logps/rejected": -431.23834228515625,
"loss": 0.0496,
"rewards/accuracies": 0.987500011920929,
"rewards/chosen": -2.793146848678589,
"rewards/margins": 8.786299705505371,
"rewards/rejected": -11.579446792602539,
"step": 810
},
{
"epoch": 0.53,
"learning_rate": 4.389721627408993e-07,
"logits/chosen": -1.02623450756073,
"logits/rejected": -0.5294391512870789,
"logps/chosen": -348.81463623046875,
"logps/rejected": -391.2160339355469,
"loss": 0.0634,
"rewards/accuracies": 0.9750000238418579,
"rewards/chosen": -2.466538906097412,
"rewards/margins": 9.582557678222656,
"rewards/rejected": -12.049097061157227,
"step": 820
},
{
"epoch": 0.53,
"learning_rate": 4.443254817987152e-07,
"logits/chosen": -1.1547324657440186,
"logits/rejected": -0.6096884608268738,
"logps/chosen": -426.49798583984375,
"logps/rejected": -467.9422302246094,
"loss": 0.072,
"rewards/accuracies": 0.9375,
"rewards/chosen": -2.5411438941955566,
"rewards/margins": 9.401772499084473,
"rewards/rejected": -11.942916870117188,
"step": 830
},
{
"epoch": 0.54,
"learning_rate": 4.49678800856531e-07,
"logits/chosen": -1.247434377670288,
"logits/rejected": -0.7016826868057251,
"logps/chosen": -393.85430908203125,
"logps/rejected": -482.9002990722656,
"loss": 0.0467,
"rewards/accuracies": 0.9750000238418579,
"rewards/chosen": -2.051135301589966,
"rewards/margins": 10.815892219543457,
"rewards/rejected": -12.867027282714844,
"step": 840
},
{
"epoch": 0.55,
"learning_rate": 4.5503211991434684e-07,
"logits/chosen": -1.2653485536575317,
"logits/rejected": -0.7517842054367065,
"logps/chosen": -393.53643798828125,
"logps/rejected": -458.5020446777344,
"loss": 0.0962,
"rewards/accuracies": 0.987500011920929,
"rewards/chosen": -1.8538535833358765,
"rewards/margins": 9.80111312866211,
"rewards/rejected": -11.654967308044434,
"step": 850
},
{
"epoch": 0.55,
"learning_rate": 4.603854389721627e-07,
"logits/chosen": -1.0140388011932373,
"logits/rejected": -0.5941162109375,
"logps/chosen": -431.3036193847656,
"logps/rejected": -522.5137939453125,
"loss": 0.073,
"rewards/accuracies": 0.987500011920929,
"rewards/chosen": -2.430864095687866,
"rewards/margins": 10.6370267868042,
"rewards/rejected": -13.067891120910645,
"step": 860
},
{
"epoch": 0.56,
"learning_rate": 4.657387580299786e-07,
"logits/chosen": -0.8724175691604614,
"logits/rejected": -0.424823522567749,
"logps/chosen": -405.14544677734375,
"logps/rejected": -458.218994140625,
"loss": 0.0896,
"rewards/accuracies": 0.9375,
"rewards/chosen": -3.2016243934631348,
"rewards/margins": 10.069561958312988,
"rewards/rejected": -13.271186828613281,
"step": 870
},
{
"epoch": 0.57,
"learning_rate": 4.710920770877944e-07,
"logits/chosen": -0.9462097883224487,
"logits/rejected": -0.5895959138870239,
"logps/chosen": -384.04046630859375,
"logps/rejected": -430.6656188964844,
"loss": 0.0703,
"rewards/accuracies": 0.925000011920929,
"rewards/chosen": -2.604841947555542,
"rewards/margins": 8.405369758605957,
"rewards/rejected": -11.010213851928711,
"step": 880
},
{
"epoch": 0.57,
"learning_rate": 4.7644539614561024e-07,
"logits/chosen": -1.0459139347076416,
"logits/rejected": -0.5180732011795044,
"logps/chosen": -349.8458557128906,
"logps/rejected": -463.24993896484375,
"loss": 0.068,
"rewards/accuracies": 0.987500011920929,
"rewards/chosen": -2.0111186504364014,
"rewards/margins": 10.823290824890137,
"rewards/rejected": -12.8344087600708,
"step": 890
},
{
"epoch": 0.58,
"learning_rate": 4.817987152034261e-07,
"logits/chosen": -1.0149575471878052,
"logits/rejected": -0.3724205493927002,
"logps/chosen": -396.1781311035156,
"logps/rejected": -512.572265625,
"loss": 0.0494,
"rewards/accuracies": 1.0,
"rewards/chosen": -2.9190399646759033,
"rewards/margins": 12.252616882324219,
"rewards/rejected": -15.171656608581543,
"step": 900
},
{
"epoch": 0.58,
"eval_logits/chosen": -1.1149048805236816,
"eval_logits/rejected": -0.723557710647583,
"eval_logps/chosen": -371.6729736328125,
"eval_logps/rejected": -447.3422546386719,
"eval_loss": 0.14773131906986237,
"eval_rewards/accuracies": 0.890625,
"eval_rewards/chosen": -4.230995178222656,
"eval_rewards/margins": 8.868158340454102,
"eval_rewards/rejected": -13.099154472351074,
"eval_runtime": 76.5391,
"eval_samples_per_second": 13.065,
"eval_steps_per_second": 0.418,
"step": 900
},
{
"epoch": 0.58,
"learning_rate": 4.871520342612419e-07,
"logits/chosen": -1.0406441688537598,
"logits/rejected": -0.7672456502914429,
"logps/chosen": -367.68988037109375,
"logps/rejected": -497.24542236328125,
"loss": 0.0497,
"rewards/accuracies": 1.0,
"rewards/chosen": -2.375262498855591,
"rewards/margins": 11.108481407165527,
"rewards/rejected": -13.483744621276855,
"step": 910
},
{
"epoch": 0.59,
"learning_rate": 4.925053533190578e-07,
"logits/chosen": -1.3684611320495605,
"logits/rejected": -0.7619699239730835,
"logps/chosen": -395.2074279785156,
"logps/rejected": -435.02490234375,
"loss": 0.0771,
"rewards/accuracies": 0.9750000238418579,
"rewards/chosen": -2.186323642730713,
"rewards/margins": 9.919301986694336,
"rewards/rejected": -12.10562515258789,
"step": 920
},
{
"epoch": 0.6,
"learning_rate": 4.978586723768736e-07,
"logits/chosen": -1.1214783191680908,
"logits/rejected": -0.5191805362701416,
"logps/chosen": -415.1065979003906,
"logps/rejected": -468.7010192871094,
"loss": 0.046,
"rewards/accuracies": 1.0,
"rewards/chosen": -3.4811503887176514,
"rewards/margins": 9.906661987304688,
"rewards/rejected": -13.387812614440918,
"step": 930
},
{
"epoch": 0.6,
"learning_rate": 4.996429421566293e-07,
"logits/chosen": -1.2635540962219238,
"logits/rejected": -0.628160834312439,
"logps/chosen": -416.54644775390625,
"logps/rejected": -465.719970703125,
"loss": 0.1156,
"rewards/accuracies": 0.949999988079071,
"rewards/chosen": -3.1887431144714355,
"rewards/margins": 9.464393615722656,
"rewards/rejected": -12.65313720703125,
"step": 940
},
{
"epoch": 0.61,
"learning_rate": 4.990478457510116e-07,
"logits/chosen": -1.3375509977340698,
"logits/rejected": -0.5667593479156494,
"logps/chosen": -430.92742919921875,
"logps/rejected": -532.2154541015625,
"loss": 0.0961,
"rewards/accuracies": 0.9375,
"rewards/chosen": -2.9553847312927246,
"rewards/margins": 12.077500343322754,
"rewards/rejected": -15.03288745880127,
"step": 950
},
{
"epoch": 0.62,
"learning_rate": 4.98452749345394e-07,
"logits/chosen": -1.117200493812561,
"logits/rejected": -0.6453494429588318,
"logps/chosen": -424.7369689941406,
"logps/rejected": -452.61749267578125,
"loss": 0.0766,
"rewards/accuracies": 0.9624999761581421,
"rewards/chosen": -3.5312678813934326,
"rewards/margins": 10.04582405090332,
"rewards/rejected": -13.577092170715332,
"step": 960
},
{
"epoch": 0.62,
"learning_rate": 4.978576529397762e-07,
"logits/chosen": -1.1340444087982178,
"logits/rejected": -0.656354546546936,
"logps/chosen": -389.5958557128906,
"logps/rejected": -524.0951538085938,
"loss": 0.0653,
"rewards/accuracies": 0.987500011920929,
"rewards/chosen": -5.064024448394775,
"rewards/margins": 11.442460060119629,
"rewards/rejected": -16.506484985351562,
"step": 970
},
{
"epoch": 0.63,
"learning_rate": 4.972625565341585e-07,
"logits/chosen": -1.2187734842300415,
"logits/rejected": -0.6018660664558411,
"logps/chosen": -386.0228271484375,
"logps/rejected": -454.13140869140625,
"loss": 0.0838,
"rewards/accuracies": 0.9624999761581421,
"rewards/chosen": -3.2799274921417236,
"rewards/margins": 9.308359146118164,
"rewards/rejected": -12.588286399841309,
"step": 980
},
{
"epoch": 0.64,
"learning_rate": 4.966674601285408e-07,
"logits/chosen": -1.111574411392212,
"logits/rejected": -0.7768339514732361,
"logps/chosen": -366.3212585449219,
"logps/rejected": -501.3627014160156,
"loss": 0.0817,
"rewards/accuracies": 0.9375,
"rewards/chosen": -2.8082051277160645,
"rewards/margins": 10.757050514221191,
"rewards/rejected": -13.565256118774414,
"step": 990
},
{
"epoch": 0.64,
"learning_rate": 4.960723637229232e-07,
"logits/chosen": -0.855319619178772,
"logits/rejected": -0.4498973786830902,
"logps/chosen": -356.251708984375,
"logps/rejected": -479.78094482421875,
"loss": 0.0735,
"rewards/accuracies": 0.9624999761581421,
"rewards/chosen": -4.355088710784912,
"rewards/margins": 9.392342567443848,
"rewards/rejected": -13.747430801391602,
"step": 1000
},
{
"epoch": 0.64,
"eval_logits/chosen": -1.069393277168274,
"eval_logits/rejected": -0.698665976524353,
"eval_logps/chosen": -367.884033203125,
"eval_logps/rejected": -449.0148620605469,
"eval_loss": 0.12433216720819473,
"eval_rewards/accuracies": 0.9609375,
"eval_rewards/chosen": -3.8520963191986084,
"eval_rewards/margins": 9.414312362670898,
"eval_rewards/rejected": -13.26640796661377,
"eval_runtime": 76.7244,
"eval_samples_per_second": 13.034,
"eval_steps_per_second": 0.417,
"step": 1000
},
{
"epoch": 0.65,
"learning_rate": 4.954772673173054e-07,
"logits/chosen": -0.8880437612533569,
"logits/rejected": -0.5571062564849854,
"logps/chosen": -427.5099182128906,
"logps/rejected": -511.27398681640625,
"loss": 0.1211,
"rewards/accuracies": 0.949999988079071,
"rewards/chosen": -3.051557779312134,
"rewards/margins": 10.788399696350098,
"rewards/rejected": -13.839956283569336,
"step": 1010
},
{
"epoch": 0.66,
"learning_rate": 4.948821709116876e-07,
"logits/chosen": -1.3221044540405273,
"logits/rejected": -0.6521639823913574,
"logps/chosen": -380.6373596191406,
"logps/rejected": -449.31817626953125,
"loss": 0.0824,
"rewards/accuracies": 0.987500011920929,
"rewards/chosen": -2.038402795791626,
"rewards/margins": 11.031317710876465,
"rewards/rejected": -13.069720268249512,
"step": 1020
},
{
"epoch": 0.66,
"learning_rate": 4.9428707450607e-07,
"logits/chosen": -1.1821175813674927,
"logits/rejected": -0.7060034871101379,
"logps/chosen": -366.3950500488281,
"logps/rejected": -507.5702209472656,
"loss": 0.078,
"rewards/accuracies": 0.9624999761581421,
"rewards/chosen": -3.4800446033477783,
"rewards/margins": 10.978177070617676,
"rewards/rejected": -14.458221435546875,
"step": 1030
},
{
"epoch": 0.67,
"learning_rate": 4.936919781004522e-07,
"logits/chosen": -1.2339891195297241,
"logits/rejected": -0.8903535604476929,
"logps/chosen": -411.30615234375,
"logps/rejected": -553.6360473632812,
"loss": 0.0829,
"rewards/accuracies": 0.9750000238418579,
"rewards/chosen": -3.9073569774627686,
"rewards/margins": 10.883973121643066,
"rewards/rejected": -14.79133129119873,
"step": 1040
},
{
"epoch": 0.67,
"learning_rate": 4.930968816948346e-07,
"logits/chosen": -1.3873369693756104,
"logits/rejected": -0.8341131210327148,
"logps/chosen": -424.88543701171875,
"logps/rejected": -477.947021484375,
"loss": 0.0747,
"rewards/accuracies": 0.9624999761581421,
"rewards/chosen": -2.1361687183380127,
"rewards/margins": 11.415266036987305,
"rewards/rejected": -13.551434516906738,
"step": 1050
},
{
"epoch": 0.68,
"learning_rate": 4.925017852892168e-07,
"logits/chosen": -1.1863329410552979,
"logits/rejected": -0.6122242212295532,
"logps/chosen": -400.68865966796875,
"logps/rejected": -515.4220581054688,
"loss": 0.0844,
"rewards/accuracies": 0.987500011920929,
"rewards/chosen": -2.35847544670105,
"rewards/margins": 13.144007682800293,
"rewards/rejected": -15.502484321594238,
"step": 1060
},
{
"epoch": 0.69,
"learning_rate": 4.919066888835991e-07,
"logits/chosen": -1.1419920921325684,
"logits/rejected": -0.5117210149765015,
"logps/chosen": -370.2403564453125,
"logps/rejected": -593.1256103515625,
"loss": 0.0685,
"rewards/accuracies": 0.987500011920929,
"rewards/chosen": -2.8190958499908447,
"rewards/margins": 14.543069839477539,
"rewards/rejected": -17.362167358398438,
"step": 1070
},
{
"epoch": 0.69,
"learning_rate": 4.913115924779814e-07,
"logits/chosen": -1.1333223581314087,
"logits/rejected": -0.5887588858604431,
"logps/chosen": -438.794677734375,
"logps/rejected": -556.837890625,
"loss": 0.0702,
"rewards/accuracies": 0.9624999761581421,
"rewards/chosen": -3.482754945755005,
"rewards/margins": 12.69272232055664,
"rewards/rejected": -16.175479888916016,
"step": 1080
},
{
"epoch": 0.7,
"learning_rate": 4.907164960723638e-07,
"logits/chosen": -1.0477075576782227,
"logits/rejected": -0.5950930714607239,
"logps/chosen": -385.4281005859375,
"logps/rejected": -448.23211669921875,
"loss": 0.0694,
"rewards/accuracies": 0.9375,
"rewards/chosen": -4.315608501434326,
"rewards/margins": 10.84793472290039,
"rewards/rejected": -15.163541793823242,
"step": 1090
},
{
"epoch": 0.71,
"learning_rate": 4.90121399666746e-07,
"logits/chosen": -0.8860788345336914,
"logits/rejected": -0.4812610149383545,
"logps/chosen": -356.5136413574219,
"logps/rejected": -509.6454162597656,
"loss": 0.0806,
"rewards/accuracies": 0.9624999761581421,
"rewards/chosen": -4.0979533195495605,
"rewards/margins": 11.64047908782959,
"rewards/rejected": -15.738431930541992,
"step": 1100
},
{
"epoch": 0.71,
"eval_logits/chosen": -1.1228324174880981,
"eval_logits/rejected": -0.7543247938156128,
"eval_logps/chosen": -354.41119384765625,
"eval_logps/rejected": -438.4850158691406,
"eval_loss": 0.11537463963031769,
"eval_rewards/accuracies": 0.9765625,
"eval_rewards/chosen": -2.5048139095306396,
"eval_rewards/margins": 9.708612442016602,
"eval_rewards/rejected": -12.213427543640137,
"eval_runtime": 76.7376,
"eval_samples_per_second": 13.031,
"eval_steps_per_second": 0.417,
"step": 1100
},
{
"epoch": 0.71,
"learning_rate": 4.895263032611282e-07,
"logits/chosen": -1.2738934755325317,
"logits/rejected": -0.6076444387435913,
"logps/chosen": -452.056396484375,
"logps/rejected": -505.551513671875,
"loss": 0.0556,
"rewards/accuracies": 0.949999988079071,
"rewards/chosen": -3.597477674484253,
"rewards/margins": 11.379976272583008,
"rewards/rejected": -14.977453231811523,
"step": 1110
},
{
"epoch": 0.72,
"learning_rate": 4.889312068555106e-07,
"logits/chosen": -1.174073576927185,
"logits/rejected": -0.7190831303596497,
"logps/chosen": -400.78656005859375,
"logps/rejected": -528.5311889648438,
"loss": 0.0635,
"rewards/accuracies": 0.9750000238418579,
"rewards/chosen": -3.351175308227539,
"rewards/margins": 11.898660659790039,
"rewards/rejected": -15.249837875366211,
"step": 1120
},
{
"epoch": 0.73,
"learning_rate": 4.883361104498928e-07,
"logits/chosen": -1.2661216259002686,
"logits/rejected": -0.7576111555099487,
"logps/chosen": -395.97833251953125,
"logps/rejected": -508.68719482421875,
"loss": 0.0564,
"rewards/accuracies": 1.0,
"rewards/chosen": -3.6894619464874268,
"rewards/margins": 13.860685348510742,
"rewards/rejected": -17.550146102905273,
"step": 1130
},
{
"epoch": 0.73,
"learning_rate": 4.877410140442752e-07,
"logits/chosen": -1.3575642108917236,
"logits/rejected": -0.8006072044372559,
"logps/chosen": -423.7298889160156,
"logps/rejected": -535.811767578125,
"loss": 0.0727,
"rewards/accuracies": 0.9624999761581421,
"rewards/chosen": -1.8236665725708008,
"rewards/margins": 12.405186653137207,
"rewards/rejected": -14.228856086730957,
"step": 1140
},
{
"epoch": 0.74,
"learning_rate": 4.871459176386574e-07,
"logits/chosen": -1.1852174997329712,
"logits/rejected": -0.7334953546524048,
"logps/chosen": -447.42724609375,
"logps/rejected": -565.5848388671875,
"loss": 0.0614,
"rewards/accuracies": 0.987500011920929,
"rewards/chosen": -3.648531436920166,
"rewards/margins": 13.974637031555176,
"rewards/rejected": -17.6231689453125,
"step": 1150
},
{
"epoch": 0.75,
"learning_rate": 4.865508212330398e-07,
"logits/chosen": -1.1278337240219116,
"logits/rejected": -0.5487757325172424,
"logps/chosen": -394.2236022949219,
"logps/rejected": -564.2678833007812,
"loss": 0.05,
"rewards/accuracies": 0.9750000238418579,
"rewards/chosen": -2.3238348960876465,
"rewards/margins": 13.824040412902832,
"rewards/rejected": -16.147876739501953,
"step": 1160
},
{
"epoch": 0.75,
"learning_rate": 4.85955724827422e-07,
"logits/chosen": -0.7651220560073853,
"logits/rejected": -0.06668927520513535,
"logps/chosen": -444.3988342285156,
"logps/rejected": -548.660888671875,
"loss": 0.0583,
"rewards/accuracies": 0.9750000238418579,
"rewards/chosen": -2.920553684234619,
"rewards/margins": 13.667207717895508,
"rewards/rejected": -16.5877628326416,
"step": 1170
},
{
"epoch": 0.76,
"learning_rate": 4.853606284218044e-07,
"logits/chosen": -1.0830278396606445,
"logits/rejected": -0.4501362442970276,
"logps/chosen": -332.0751037597656,
"logps/rejected": -418.5309143066406,
"loss": 0.0789,
"rewards/accuracies": 0.9375,
"rewards/chosen": -1.7456166744232178,
"rewards/margins": 11.250526428222656,
"rewards/rejected": -12.996142387390137,
"step": 1180
},
{
"epoch": 0.76,
"learning_rate": 4.847655320161866e-07,
"logits/chosen": -0.6781784296035767,
"logits/rejected": -0.27196237444877625,
"logps/chosen": -396.753662109375,
"logps/rejected": -491.14892578125,
"loss": 0.0668,
"rewards/accuracies": 0.9750000238418579,
"rewards/chosen": -2.8804516792297363,
"rewards/margins": 11.853898048400879,
"rewards/rejected": -14.734350204467773,
"step": 1190
},
{
"epoch": 0.77,
"learning_rate": 4.841704356105689e-07,
"logits/chosen": -0.7234100103378296,
"logits/rejected": -0.2980794310569763,
"logps/chosen": -367.587158203125,
"logps/rejected": -433.124755859375,
"loss": 0.0822,
"rewards/accuracies": 0.949999988079071,
"rewards/chosen": -3.162593364715576,
"rewards/margins": 10.242734909057617,
"rewards/rejected": -13.405328750610352,
"step": 1200
},
{
"epoch": 0.77,
"eval_logits/chosen": -1.0001459121704102,
"eval_logits/rejected": -0.6194829940795898,
"eval_logps/chosen": -365.28955078125,
"eval_logps/rejected": -458.57135009765625,
"eval_loss": 0.13022372126579285,
"eval_rewards/accuracies": 0.96875,
"eval_rewards/chosen": -3.5926475524902344,
"eval_rewards/margins": 10.629414558410645,
"eval_rewards/rejected": -14.222061157226562,
"eval_runtime": 76.5043,
"eval_samples_per_second": 13.071,
"eval_steps_per_second": 0.418,
"step": 1200
},
{
"epoch": 0.78,
"learning_rate": 4.835753392049512e-07,
"logits/chosen": -0.6337307691574097,
"logits/rejected": -0.23596186935901642,
"logps/chosen": -356.19122314453125,
"logps/rejected": -514.327392578125,
"loss": 0.1085,
"rewards/accuracies": 0.949999988079071,
"rewards/chosen": -3.958400249481201,
"rewards/margins": 13.92591667175293,
"rewards/rejected": -17.884319305419922,
"step": 1210
},
{
"epoch": 0.78,
"learning_rate": 4.829802427993334e-07,
"logits/chosen": -0.6189112663269043,
"logits/rejected": -0.34567025303840637,
"logps/chosen": -374.00286865234375,
"logps/rejected": -474.2106018066406,
"loss": 0.1089,
"rewards/accuracies": 0.9375,
"rewards/chosen": -3.2338080406188965,
"rewards/margins": 9.657389640808105,
"rewards/rejected": -12.891199111938477,
"step": 1220
},
{
"epoch": 0.79,
"learning_rate": 4.823851463937158e-07,
"logits/chosen": -0.3865962028503418,
"logits/rejected": 0.04075580835342407,
"logps/chosen": -359.3363342285156,
"logps/rejected": -501.40240478515625,
"loss": 0.1142,
"rewards/accuracies": 0.9750000238418579,
"rewards/chosen": -4.306814193725586,
"rewards/margins": 13.71052074432373,
"rewards/rejected": -18.017335891723633,
"step": 1230
},
{
"epoch": 0.8,
"learning_rate": 4.81790049988098e-07,
"logits/chosen": -0.7158625721931458,
"logits/rejected": -0.21917279064655304,
"logps/chosen": -434.91253662109375,
"logps/rejected": -528.3048095703125,
"loss": 0.0757,
"rewards/accuracies": 0.987500011920929,
"rewards/chosen": -3.414928913116455,
"rewards/margins": 12.52367115020752,
"rewards/rejected": -15.938600540161133,
"step": 1240
},
{
"epoch": 0.8,
"learning_rate": 4.811949535824804e-07,
"logits/chosen": -0.7639582753181458,
"logits/rejected": -0.4316403269767761,
"logps/chosen": -365.72784423828125,
"logps/rejected": -567.1033935546875,
"loss": 0.084,
"rewards/accuracies": 0.9750000238418579,
"rewards/chosen": -3.063331127166748,
"rewards/margins": 12.180209159851074,
"rewards/rejected": -15.243539810180664,
"step": 1250
},
{
"epoch": 0.81,
"learning_rate": 4.805998571768626e-07,
"logits/chosen": -1.2161794900894165,
"logits/rejected": -0.6932582259178162,
"logps/chosen": -406.9449768066406,
"logps/rejected": -520.5137939453125,
"loss": 0.0951,
"rewards/accuracies": 0.9624999761581421,
"rewards/chosen": -3.868889331817627,
"rewards/margins": 10.655603408813477,
"rewards/rejected": -14.524490356445312,
"step": 1260
},
{
"epoch": 0.82,
"learning_rate": 4.80004760771245e-07,
"logits/chosen": -0.9595744013786316,
"logits/rejected": -0.43417948484420776,
"logps/chosen": -411.44677734375,
"logps/rejected": -526.41845703125,
"loss": 0.0691,
"rewards/accuracies": 0.9750000238418579,
"rewards/chosen": -3.0373916625976562,
"rewards/margins": 9.937090873718262,
"rewards/rejected": -12.97448444366455,
"step": 1270
},
{
"epoch": 0.82,
"learning_rate": 4.794096643656272e-07,
"logits/chosen": -1.0694881677627563,
"logits/rejected": -0.5757830142974854,
"logps/chosen": -382.9389343261719,
"logps/rejected": -500.0220642089844,
"loss": 0.0743,
"rewards/accuracies": 0.9624999761581421,
"rewards/chosen": -3.738356113433838,
"rewards/margins": 10.426973342895508,
"rewards/rejected": -14.165328979492188,
"step": 1280
},
{
"epoch": 0.83,
"learning_rate": 4.788145679600095e-07,
"logits/chosen": -0.9496662020683289,
"logits/rejected": -0.3601847290992737,
"logps/chosen": -358.3319396972656,
"logps/rejected": -560.3468627929688,
"loss": 0.0742,
"rewards/accuracies": 1.0,
"rewards/chosen": -2.929766893386841,
"rewards/margins": 13.46239185333252,
"rewards/rejected": -16.39215660095215,
"step": 1290
},
{
"epoch": 0.84,
"learning_rate": 4.782194715543918e-07,
"logits/chosen": -0.6319989562034607,
"logits/rejected": -0.09417597949504852,
"logps/chosen": -379.7413024902344,
"logps/rejected": -522.1488037109375,
"loss": 0.063,
"rewards/accuracies": 0.9750000238418579,
"rewards/chosen": -4.54395055770874,
"rewards/margins": 13.589025497436523,
"rewards/rejected": -18.132978439331055,
"step": 1300
},
{
"epoch": 0.84,
"eval_logits/chosen": -0.9766608476638794,
"eval_logits/rejected": -0.467482328414917,
"eval_logps/chosen": -374.5415954589844,
"eval_logps/rejected": -472.6826477050781,
"eval_loss": 0.18041668832302094,
"eval_rewards/accuracies": 0.9375,
"eval_rewards/chosen": -4.517853260040283,
"eval_rewards/margins": 11.115335464477539,
"eval_rewards/rejected": -15.633190155029297,
"eval_runtime": 76.7717,
"eval_samples_per_second": 13.026,
"eval_steps_per_second": 0.417,
"step": 1300
},
{
"epoch": 0.84,
"learning_rate": 4.77624375148774e-07,
"logits/chosen": -0.6732112169265747,
"logits/rejected": 0.014696260914206505,
"logps/chosen": -417.83074951171875,
"logps/rejected": -488.5224609375,
"loss": 0.0634,
"rewards/accuracies": 0.9624999761581421,
"rewards/chosen": -4.793654441833496,
"rewards/margins": 13.286532402038574,
"rewards/rejected": -18.08018684387207,
"step": 1310
},
{
"epoch": 0.85,
"learning_rate": 4.770292787431564e-07,
"logits/chosen": -0.3928489685058594,
"logits/rejected": -0.07063998281955719,
"logps/chosen": -365.6458435058594,
"logps/rejected": -472.38043212890625,
"loss": 0.0885,
"rewards/accuracies": 0.925000011920929,
"rewards/chosen": -4.527663707733154,
"rewards/margins": 11.80773639678955,
"rewards/rejected": -16.335399627685547,
"step": 1320
},
{
"epoch": 0.85,
"learning_rate": 4.764341823375387e-07,
"logits/chosen": -0.5924497842788696,
"logits/rejected": 0.10169048607349396,
"logps/chosen": -394.43292236328125,
"logps/rejected": -514.439697265625,
"loss": 0.0762,
"rewards/accuracies": 0.9624999761581421,
"rewards/chosen": -3.806927442550659,
"rewards/margins": 12.637249946594238,
"rewards/rejected": -16.444177627563477,
"step": 1330
},
{
"epoch": 0.86,
"learning_rate": 4.7583908593192097e-07,
"logits/chosen": -0.6815664172172546,
"logits/rejected": -0.29060396552085876,
"logps/chosen": -420.83221435546875,
"logps/rejected": -551.6920776367188,
"loss": 0.0532,
"rewards/accuracies": 0.987500011920929,
"rewards/chosen": -3.1024906635284424,
"rewards/margins": 13.700825691223145,
"rewards/rejected": -16.80331802368164,
"step": 1340
},
{
"epoch": 0.87,
"learning_rate": 4.752439895263032e-07,
"logits/chosen": -0.689470112323761,
"logits/rejected": -0.20767569541931152,
"logps/chosen": -385.32037353515625,
"logps/rejected": -533.0687255859375,
"loss": 0.0878,
"rewards/accuracies": 0.925000011920929,
"rewards/chosen": -3.7341361045837402,
"rewards/margins": 14.102251052856445,
"rewards/rejected": -17.83638572692871,
"step": 1350
},
{
"epoch": 0.87,
"learning_rate": 4.746488931206855e-07,
"logits/chosen": -0.7405373454093933,
"logits/rejected": -0.07327382266521454,
"logps/chosen": -377.8034973144531,
"logps/rejected": -466.72784423828125,
"loss": 0.0851,
"rewards/accuracies": 0.949999988079071,
"rewards/chosen": -3.5931084156036377,
"rewards/margins": 12.507670402526855,
"rewards/rejected": -16.100778579711914,
"step": 1360
},
{
"epoch": 0.88,
"learning_rate": 4.7405379671506785e-07,
"logits/chosen": -0.5811715126037598,
"logits/rejected": -0.06195932626724243,
"logps/chosen": -411.6004943847656,
"logps/rejected": -548.6055908203125,
"loss": 0.1093,
"rewards/accuracies": 0.9125000238418579,
"rewards/chosen": -4.2639055252075195,
"rewards/margins": 11.680307388305664,
"rewards/rejected": -15.944211959838867,
"step": 1370
},
{
"epoch": 0.89,
"learning_rate": 4.734587003094501e-07,
"logits/chosen": -0.8370053172111511,
"logits/rejected": -0.14809687435626984,
"logps/chosen": -447.0265197753906,
"logps/rejected": -535.3525390625,
"loss": 0.081,
"rewards/accuracies": 0.949999988079071,
"rewards/chosen": -4.550225734710693,
"rewards/margins": 12.257158279418945,
"rewards/rejected": -16.807382583618164,
"step": 1380
},
{
"epoch": 0.89,
"learning_rate": 4.728636039038324e-07,
"logits/chosen": -0.7418749928474426,
"logits/rejected": -0.25607532262802124,
"logps/chosen": -374.479248046875,
"logps/rejected": -532.3735961914062,
"loss": 0.0738,
"rewards/accuracies": 0.987500011920929,
"rewards/chosen": -5.4016432762146,
"rewards/margins": 13.285806655883789,
"rewards/rejected": -18.687450408935547,
"step": 1390
},
{
"epoch": 0.9,
"learning_rate": 4.722685074982147e-07,
"logits/chosen": -0.8056036233901978,
"logits/rejected": -0.2640933394432068,
"logps/chosen": -372.2264709472656,
"logps/rejected": -565.7047729492188,
"loss": 0.0648,
"rewards/accuracies": 0.987500011920929,
"rewards/chosen": -3.8431191444396973,
"rewards/margins": 13.19017505645752,
"rewards/rejected": -17.033292770385742,
"step": 1400
},
{
"epoch": 0.9,
"eval_logits/chosen": -1.118211030960083,
"eval_logits/rejected": -0.5943832397460938,
"eval_logps/chosen": -354.5495300292969,
"eval_logps/rejected": -446.5892333984375,
"eval_loss": 0.13570235669612885,
"eval_rewards/accuracies": 0.953125,
"eval_rewards/chosen": -2.518648147583008,
"eval_rewards/margins": 10.50519847869873,
"eval_rewards/rejected": -13.023846626281738,
"eval_runtime": 76.6799,
"eval_samples_per_second": 13.041,
"eval_steps_per_second": 0.417,
"step": 1400
},
{
"epoch": 0.91,
"learning_rate": 4.7167341109259703e-07,
"logits/chosen": -1.0566179752349854,
"logits/rejected": -0.2900046706199646,
"logps/chosen": -398.8577575683594,
"logps/rejected": -454.0634765625,
"loss": 0.094,
"rewards/accuracies": 0.987500011920929,
"rewards/chosen": -2.2813262939453125,
"rewards/margins": 11.221616744995117,
"rewards/rejected": -13.502942085266113,
"step": 1410
},
{
"epoch": 0.91,
"learning_rate": 4.710783146869793e-07,
"logits/chosen": -0.9073772430419922,
"logits/rejected": -0.400244802236557,
"logps/chosen": -357.8586730957031,
"logps/rejected": -546.3787841796875,
"loss": 0.0928,
"rewards/accuracies": 0.9375,
"rewards/chosen": -3.4375388622283936,
"rewards/margins": 11.90378475189209,
"rewards/rejected": -15.341323852539062,
"step": 1420
},
{
"epoch": 0.92,
"learning_rate": 4.7048321828136157e-07,
"logits/chosen": -0.6786088347434998,
"logits/rejected": -0.10559716075658798,
"logps/chosen": -417.87939453125,
"logps/rejected": -530.9561767578125,
"loss": 0.042,
"rewards/accuracies": 0.987500011920929,
"rewards/chosen": -5.6138105392456055,
"rewards/margins": 13.705339431762695,
"rewards/rejected": -19.319150924682617,
"step": 1430
},
{
"epoch": 0.93,
"learning_rate": 4.698881218757438e-07,
"logits/chosen": -0.5866619348526001,
"logits/rejected": -0.026242520660161972,
"logps/chosen": -422.72613525390625,
"logps/rejected": -541.8472900390625,
"loss": 0.1036,
"rewards/accuracies": 0.9375,
"rewards/chosen": -4.567929267883301,
"rewards/margins": 11.613755226135254,
"rewards/rejected": -16.181686401367188,
"step": 1440
},
{
"epoch": 0.93,
"learning_rate": 4.692930254701261e-07,
"logits/chosen": -0.7376483082771301,
"logits/rejected": -0.09872325509786606,
"logps/chosen": -425.328857421875,
"logps/rejected": -529.0191650390625,
"loss": 0.0628,
"rewards/accuracies": 0.9375,
"rewards/chosen": -4.07589054107666,
"rewards/margins": 12.2763032913208,
"rewards/rejected": -16.352191925048828,
"step": 1450
},
{
"epoch": 0.94,
"learning_rate": 4.6869792906450845e-07,
"logits/chosen": -0.5996996760368347,
"logits/rejected": 0.09782281517982483,
"logps/chosen": -426.3692321777344,
"logps/rejected": -495.6768493652344,
"loss": 0.0448,
"rewards/accuracies": 0.987500011920929,
"rewards/chosen": -4.688590049743652,
"rewards/margins": 13.677145004272461,
"rewards/rejected": -18.36573600769043,
"step": 1460
},
{
"epoch": 0.94,
"learning_rate": 4.6810283265889075e-07,
"logits/chosen": -0.6974693536758423,
"logits/rejected": 0.09173402935266495,
"logps/chosen": -361.08746337890625,
"logps/rejected": -483.73211669921875,
"loss": 0.0571,
"rewards/accuracies": 1.0,
"rewards/chosen": -4.233609199523926,
"rewards/margins": 13.777618408203125,
"rewards/rejected": -18.011228561401367,
"step": 1470
},
{
"epoch": 0.95,
"learning_rate": 4.67507736253273e-07,
"logits/chosen": -0.8001763224601746,
"logits/rejected": -0.1673848032951355,
"logps/chosen": -368.32196044921875,
"logps/rejected": -508.84368896484375,
"loss": 0.0676,
"rewards/accuracies": 1.0,
"rewards/chosen": -2.7094593048095703,
"rewards/margins": 13.913045883178711,
"rewards/rejected": -16.622507095336914,
"step": 1480
},
{
"epoch": 0.96,
"learning_rate": 4.669126398476553e-07,
"logits/chosen": -0.589401125907898,
"logits/rejected": 0.09411343187093735,
"logps/chosen": -448.2190856933594,
"logps/rejected": -579.1790161132812,
"loss": 0.07,
"rewards/accuracies": 0.9750000238418579,
"rewards/chosen": -3.9501609802246094,
"rewards/margins": 15.417715072631836,
"rewards/rejected": -19.367877960205078,
"step": 1490
},
{
"epoch": 0.96,
"learning_rate": 4.6631754344203763e-07,
"logits/chosen": -0.6391473412513733,
"logits/rejected": -0.1050008162856102,
"logps/chosen": -376.50579833984375,
"logps/rejected": -544.2982177734375,
"loss": 0.0714,
"rewards/accuracies": 0.949999988079071,
"rewards/chosen": -3.8969063758850098,
"rewards/margins": 13.172216415405273,
"rewards/rejected": -17.069122314453125,
"step": 1500
},
{
"epoch": 0.96,
"eval_logits/chosen": -1.0033119916915894,
"eval_logits/rejected": -0.4300505220890045,
"eval_logps/chosen": -374.3122863769531,
"eval_logps/rejected": -470.21783447265625,
"eval_loss": 0.12438826262950897,
"eval_rewards/accuracies": 0.9453125,
"eval_rewards/chosen": -4.494921684265137,
"eval_rewards/margins": 10.89178466796875,
"eval_rewards/rejected": -15.38670539855957,
"eval_runtime": 76.5689,
"eval_samples_per_second": 13.06,
"eval_steps_per_second": 0.418,
"step": 1500
},
{
"epoch": 0.97,
"learning_rate": 4.657224470364199e-07,
"logits/chosen": -0.43436574935913086,
"logits/rejected": 0.07220065593719482,
"logps/chosen": -391.37725830078125,
"logps/rejected": -588.6260986328125,
"loss": 0.0546,
"rewards/accuracies": 0.987500011920929,
"rewards/chosen": -5.231613636016846,
"rewards/margins": 13.30431842803955,
"rewards/rejected": -18.535930633544922,
"step": 1510
},
{
"epoch": 0.98,
"learning_rate": 4.6512735063080217e-07,
"logits/chosen": -0.6157822608947754,
"logits/rejected": 0.005131366662681103,
"logps/chosen": -411.13946533203125,
"logps/rejected": -483.9326171875,
"loss": 0.0527,
"rewards/accuracies": 0.9624999761581421,
"rewards/chosen": -5.003513336181641,
"rewards/margins": 10.31053638458252,
"rewards/rejected": -15.314050674438477,
"step": 1520
},
{
"epoch": 0.98,
"learning_rate": 4.6453225422518447e-07,
"logits/chosen": -0.6011049747467041,
"logits/rejected": -0.10970073938369751,
"logps/chosen": -406.9378356933594,
"logps/rejected": -468.97314453125,
"loss": 0.0739,
"rewards/accuracies": 0.987500011920929,
"rewards/chosen": -4.8249969482421875,
"rewards/margins": 11.424562454223633,
"rewards/rejected": -16.24955940246582,
"step": 1530
},
{
"epoch": 0.99,
"learning_rate": 4.6393715781956676e-07,
"logits/chosen": -0.6942557096481323,
"logits/rejected": 0.16295239329338074,
"logps/chosen": -407.1282653808594,
"logps/rejected": -560.19970703125,
"loss": 0.0629,
"rewards/accuracies": 0.987500011920929,
"rewards/chosen": -4.603503704071045,
"rewards/margins": 13.417546272277832,
"rewards/rejected": -18.02104949951172,
"step": 1540
},
{
"epoch": 1.0,
"learning_rate": 4.6334206141394905e-07,
"logits/chosen": -0.3027026057243347,
"logits/rejected": 0.10327012836933136,
"logps/chosen": -405.692626953125,
"logps/rejected": -576.5963134765625,
"loss": 0.0673,
"rewards/accuracies": 0.987500011920929,
"rewards/chosen": -3.6844029426574707,
"rewards/margins": 13.106195449829102,
"rewards/rejected": -16.790597915649414,
"step": 1550
},
{
"epoch": 1.0,
"learning_rate": 4.6274696500833135e-07,
"logits/chosen": -0.5037144422531128,
"logits/rejected": 0.06204764172434807,
"logps/chosen": -412.0623474121094,
"logps/rejected": -525.1038818359375,
"loss": 0.061,
"rewards/accuracies": 0.987500011920929,
"rewards/chosen": -2.6065430641174316,
"rewards/margins": 14.261393547058105,
"rewards/rejected": -16.867937088012695,
"step": 1560
},
{
"epoch": 1.01,
"learning_rate": 4.621518686027136e-07,
"logits/chosen": -0.526481568813324,
"logits/rejected": 0.24668976664543152,
"logps/chosen": -410.71746826171875,
"logps/rejected": -473.8052673339844,
"loss": 0.026,
"rewards/accuracies": 0.9624999761581421,
"rewards/chosen": -2.839338779449463,
"rewards/margins": 13.472638130187988,
"rewards/rejected": -16.31197738647461,
"step": 1570
},
{
"epoch": 1.02,
"learning_rate": 4.6155677219709594e-07,
"logits/chosen": -0.2807101607322693,
"logits/rejected": 0.3707982003688812,
"logps/chosen": -430.853759765625,
"logps/rejected": -465.86328125,
"loss": 0.016,
"rewards/accuracies": 0.987500011920929,
"rewards/chosen": -3.3595848083496094,
"rewards/margins": 12.908388137817383,
"rewards/rejected": -16.267974853515625,
"step": 1580
},
{
"epoch": 1.02,
"learning_rate": 4.6096167579147823e-07,
"logits/chosen": -0.2973923683166504,
"logits/rejected": 0.30676135420799255,
"logps/chosen": -384.2185974121094,
"logps/rejected": -527.7244873046875,
"loss": 0.0162,
"rewards/accuracies": 1.0,
"rewards/chosen": -3.731484889984131,
"rewards/margins": 15.676348686218262,
"rewards/rejected": -19.407833099365234,
"step": 1590
},
{
"epoch": 1.03,
"learning_rate": 4.603665793858605e-07,
"logits/chosen": -0.5543831586837769,
"logits/rejected": 0.3015816807746887,
"logps/chosen": -395.59368896484375,
"logps/rejected": -544.031494140625,
"loss": 0.0095,
"rewards/accuracies": 0.987500011920929,
"rewards/chosen": -3.2728145122528076,
"rewards/margins": 15.476274490356445,
"rewards/rejected": -18.74909019470215,
"step": 1600
},
{
"epoch": 1.03,
"eval_logits/chosen": -0.7791086435317993,
"eval_logits/rejected": -0.20377439260482788,
"eval_logps/chosen": -380.9189453125,
"eval_logps/rejected": -480.3507995605469,
"eval_loss": 0.10078604519367218,
"eval_rewards/accuracies": 0.9375,
"eval_rewards/chosen": -5.155591011047363,
"eval_rewards/margins": 11.244410514831543,
"eval_rewards/rejected": -16.40000343322754,
"eval_runtime": 76.553,
"eval_samples_per_second": 13.063,
"eval_steps_per_second": 0.418,
"step": 1600
},
{
"epoch": 1.03,
"learning_rate": 4.5977148298024277e-07,
"logits/chosen": -0.5526180267333984,
"logits/rejected": 0.3482429087162018,
"logps/chosen": -391.2712707519531,
"logps/rejected": -509.62847900390625,
"loss": 0.0064,
"rewards/accuracies": 1.0,
"rewards/chosen": -3.7447268962860107,
"rewards/margins": 15.208663940429688,
"rewards/rejected": -18.95339012145996,
"step": 1610
},
{
"epoch": 1.04,
"learning_rate": 4.5917638657462507e-07,
"logits/chosen": -0.4088626801967621,
"logits/rejected": 0.3109140992164612,
"logps/chosen": -426.03643798828125,
"logps/rejected": -556.555908203125,
"loss": 0.0053,
"rewards/accuracies": 0.987500011920929,
"rewards/chosen": -4.894583702087402,
"rewards/margins": 14.70109748840332,
"rewards/rejected": -19.595678329467773,
"step": 1620
},
{
"epoch": 1.05,
"learning_rate": 4.5858129016900736e-07,
"logits/chosen": -0.5371668934822083,
"logits/rejected": 0.3167404532432556,
"logps/chosen": -393.68524169921875,
"logps/rejected": -562.7149658203125,
"loss": 0.0129,
"rewards/accuracies": 0.987500011920929,
"rewards/chosen": -4.771970272064209,
"rewards/margins": 15.018495559692383,
"rewards/rejected": -19.79046630859375,
"step": 1630
},
{
"epoch": 1.05,
"learning_rate": 4.5798619376338966e-07,
"logits/chosen": -0.247820645570755,
"logits/rejected": 0.3391680121421814,
"logps/chosen": -370.0538024902344,
"logps/rejected": -491.40252685546875,
"loss": 0.0067,
"rewards/accuracies": 0.987500011920929,
"rewards/chosen": -4.405149936676025,
"rewards/margins": 14.295089721679688,
"rewards/rejected": -18.700239181518555,
"step": 1640
},
{
"epoch": 1.06,
"learning_rate": 4.5739109735777195e-07,
"logits/chosen": -0.2800091505050659,
"logits/rejected": 0.2654980719089508,
"logps/chosen": -439.1228942871094,
"logps/rejected": -581.3944091796875,
"loss": 0.0115,
"rewards/accuracies": 0.9750000238418579,
"rewards/chosen": -6.183625221252441,
"rewards/margins": 16.414287567138672,
"rewards/rejected": -22.597911834716797,
"step": 1650
},
{
"epoch": 1.07,
"learning_rate": 4.567960009521542e-07,
"logits/chosen": -0.32536178827285767,
"logits/rejected": 0.2308044731616974,
"logps/chosen": -362.37945556640625,
"logps/rejected": -542.333984375,
"loss": 0.0122,
"rewards/accuracies": 0.987500011920929,
"rewards/chosen": -3.910571575164795,
"rewards/margins": 15.097010612487793,
"rewards/rejected": -19.007579803466797,
"step": 1660
},
{
"epoch": 1.07,
"learning_rate": 4.5620090454653654e-07,
"logits/chosen": -0.44402211904525757,
"logits/rejected": 0.07348278164863586,
"logps/chosen": -377.78759765625,
"logps/rejected": -574.8585205078125,
"loss": 0.0117,
"rewards/accuracies": 1.0,
"rewards/chosen": -5.063399314880371,
"rewards/margins": 16.107744216918945,
"rewards/rejected": -21.171140670776367,
"step": 1670
},
{
"epoch": 1.08,
"learning_rate": 4.5560580814091884e-07,
"logits/chosen": -0.6580984592437744,
"logits/rejected": 0.15282706916332245,
"logps/chosen": -388.4255065917969,
"logps/rejected": -543.6886596679688,
"loss": 0.0331,
"rewards/accuracies": 0.9750000238418579,
"rewards/chosen": -4.285706996917725,
"rewards/margins": 15.84802532196045,
"rewards/rejected": -20.13373374938965,
"step": 1680
},
{
"epoch": 1.09,
"learning_rate": 4.550107117353011e-07,
"logits/chosen": -0.5916538834571838,
"logits/rejected": 0.042367033660411835,
"logps/chosen": -425.89013671875,
"logps/rejected": -556.3162841796875,
"loss": 0.0158,
"rewards/accuracies": 1.0,
"rewards/chosen": -4.548110008239746,
"rewards/margins": 14.834815979003906,
"rewards/rejected": -19.38292694091797,
"step": 1690
},
{
"epoch": 1.09,
"learning_rate": 4.5441561532968337e-07,
"logits/chosen": -0.6858727335929871,
"logits/rejected": -0.02261565811932087,
"logps/chosen": -391.042724609375,
"logps/rejected": -572.4518432617188,
"loss": 0.0096,
"rewards/accuracies": 1.0,
"rewards/chosen": -3.9488537311553955,
"rewards/margins": 17.172945022583008,
"rewards/rejected": -21.121801376342773,
"step": 1700
},
{
"epoch": 1.09,
"eval_logits/chosen": -0.9992736577987671,
"eval_logits/rejected": -0.40412917733192444,
"eval_logps/chosen": -377.3610534667969,
"eval_logps/rejected": -483.4149169921875,
"eval_loss": 0.1330709010362625,
"eval_rewards/accuracies": 0.953125,
"eval_rewards/chosen": -4.799799919128418,
"eval_rewards/margins": 11.906618118286133,
"eval_rewards/rejected": -16.706417083740234,
"eval_runtime": 76.6396,
"eval_samples_per_second": 13.048,
"eval_steps_per_second": 0.418,
"step": 1700
},
{
"epoch": 1.1,
"learning_rate": 4.538205189240657e-07,
"logits/chosen": -0.4306742548942566,
"logits/rejected": 0.3547573983669281,
"logps/chosen": -398.1122741699219,
"logps/rejected": -550.4100341796875,
"loss": 0.0132,
"rewards/accuracies": 1.0,
"rewards/chosen": -5.38922643661499,
"rewards/margins": 16.458080291748047,
"rewards/rejected": -21.847307205200195,
"step": 1710
},
{
"epoch": 1.11,
"learning_rate": 4.5322542251844796e-07,
"logits/chosen": -0.5155312418937683,
"logits/rejected": -0.017366236075758934,
"logps/chosen": -410.91412353515625,
"logps/rejected": -553.8331298828125,
"loss": 0.0058,
"rewards/accuracies": 0.987500011920929,
"rewards/chosen": -4.796280384063721,
"rewards/margins": 16.963947296142578,
"rewards/rejected": -21.76022720336914,
"step": 1720
},
{
"epoch": 1.11,
"learning_rate": 4.5263032611283026e-07,
"logits/chosen": -0.12997198104858398,
"logits/rejected": 0.2565564215183258,
"logps/chosen": -405.981201171875,
"logps/rejected": -620.6024169921875,
"loss": 0.014,
"rewards/accuracies": 0.987500011920929,
"rewards/chosen": -4.828683376312256,
"rewards/margins": 18.5574893951416,
"rewards/rejected": -23.386173248291016,
"step": 1730
},
{
"epoch": 1.12,
"learning_rate": 4.5203522970721255e-07,
"logits/chosen": -0.0680803433060646,
"logits/rejected": 0.4802146852016449,
"logps/chosen": -421.7552185058594,
"logps/rejected": -488.4523010253906,
"loss": 0.0131,
"rewards/accuracies": 1.0,
"rewards/chosen": -3.658025026321411,
"rewards/margins": 15.093057632446289,
"rewards/rejected": -18.751081466674805,
"step": 1740
},
{
"epoch": 1.12,
"learning_rate": 4.5144013330159485e-07,
"logits/chosen": -0.3530596196651459,
"logits/rejected": 0.2572212815284729,
"logps/chosen": -413.51629638671875,
"logps/rejected": -577.3765869140625,
"loss": 0.0179,
"rewards/accuracies": 0.987500011920929,
"rewards/chosen": -4.011878967285156,
"rewards/margins": 15.384442329406738,
"rewards/rejected": -19.39632225036621,
"step": 1750
},
{
"epoch": 1.13,
"learning_rate": 4.5084503689597714e-07,
"logits/chosen": -0.28345853090286255,
"logits/rejected": 0.42750459909439087,
"logps/chosen": -437.33941650390625,
"logps/rejected": -554.0641479492188,
"loss": 0.0124,
"rewards/accuracies": 1.0,
"rewards/chosen": -4.84152889251709,
"rewards/margins": 14.463783264160156,
"rewards/rejected": -19.305313110351562,
"step": 1760
},
{
"epoch": 1.14,
"learning_rate": 4.5024994049035944e-07,
"logits/chosen": -0.09664113819599152,
"logits/rejected": 0.22168950736522675,
"logps/chosen": -378.29632568359375,
"logps/rejected": -565.90185546875,
"loss": 0.0134,
"rewards/accuracies": 1.0,
"rewards/chosen": -4.932016372680664,
"rewards/margins": 17.979869842529297,
"rewards/rejected": -22.911888122558594,
"step": 1770
},
{
"epoch": 1.14,
"learning_rate": 4.496548440847417e-07,
"logits/chosen": -0.049262501299381256,
"logits/rejected": 0.4335893988609314,
"logps/chosen": -402.48980712890625,
"logps/rejected": -552.8167114257812,
"loss": 0.0279,
"rewards/accuracies": 0.9750000238418579,
"rewards/chosen": -3.7815442085266113,
"rewards/margins": 15.186810493469238,
"rewards/rejected": -18.968353271484375,
"step": 1780
},
{
"epoch": 1.15,
"learning_rate": 4.49059747679124e-07,
"logits/chosen": -0.28909042477607727,
"logits/rejected": 0.3799073398113251,
"logps/chosen": -430.60760498046875,
"logps/rejected": -537.1256713867188,
"loss": 0.0168,
"rewards/accuracies": 1.0,
"rewards/chosen": -4.874712944030762,
"rewards/margins": 15.4542875289917,
"rewards/rejected": -20.32900047302246,
"step": 1790
},
{
"epoch": 1.16,
"learning_rate": 4.484646512735063e-07,
"logits/chosen": -0.3206165134906769,
"logits/rejected": 0.06656259298324585,
"logps/chosen": -471.58001708984375,
"logps/rejected": -507.40301513671875,
"loss": 0.0293,
"rewards/accuracies": 0.987500011920929,
"rewards/chosen": -4.206088066101074,
"rewards/margins": 13.663434982299805,
"rewards/rejected": -17.869524002075195,
"step": 1800
},
{
"epoch": 1.16,
"eval_logits/chosen": -0.8347997665405273,
"eval_logits/rejected": -0.41860231757164,
"eval_logps/chosen": -372.9380187988281,
"eval_logps/rejected": -465.8204345703125,
"eval_loss": 0.14275716245174408,
"eval_rewards/accuracies": 0.9609375,
"eval_rewards/chosen": -4.35749626159668,
"eval_rewards/margins": 10.589475631713867,
"eval_rewards/rejected": -14.94697093963623,
"eval_runtime": 76.7934,
"eval_samples_per_second": 13.022,
"eval_steps_per_second": 0.417,
"step": 1800
},
{
"epoch": 1.16,
"learning_rate": 4.4786955486788856e-07,
"logits/chosen": -0.3631385266780853,
"logits/rejected": 0.07067543268203735,
"logps/chosen": -377.46551513671875,
"logps/rejected": -519.6616821289062,
"loss": 0.0359,
"rewards/accuracies": 1.0,
"rewards/chosen": -3.011387586593628,
"rewards/margins": 15.720315933227539,
"rewards/rejected": -18.73170280456543,
"step": 1810
},
{
"epoch": 1.17,
"learning_rate": 4.4727445846227086e-07,
"logits/chosen": -0.36144647002220154,
"logits/rejected": -0.008104220032691956,
"logps/chosen": -397.82940673828125,
"logps/rejected": -593.34521484375,
"loss": 0.0099,
"rewards/accuracies": 1.0,
"rewards/chosen": -5.28765869140625,
"rewards/margins": 15.561511039733887,
"rewards/rejected": -20.849170684814453,
"step": 1820
},
{
"epoch": 1.18,
"learning_rate": 4.4667936205665315e-07,
"logits/chosen": -0.1907232105731964,
"logits/rejected": 0.313555508852005,
"logps/chosen": -393.31719970703125,
"logps/rejected": -510.4048767089844,
"loss": 0.0247,
"rewards/accuracies": 1.0,
"rewards/chosen": -4.2387800216674805,
"rewards/margins": 14.292640686035156,
"rewards/rejected": -18.531421661376953,
"step": 1830
},
{
"epoch": 1.18,
"learning_rate": 4.4608426565103545e-07,
"logits/chosen": -0.6258490085601807,
"logits/rejected": 0.040771596133708954,
"logps/chosen": -419.44549560546875,
"logps/rejected": -544.8959350585938,
"loss": 0.0159,
"rewards/accuracies": 1.0,
"rewards/chosen": -3.875296115875244,
"rewards/margins": 15.84538459777832,
"rewards/rejected": -19.720678329467773,
"step": 1840
},
{
"epoch": 1.19,
"learning_rate": 4.4548916924541774e-07,
"logits/chosen": -0.3880535066127777,
"logits/rejected": -0.07375472038984299,
"logps/chosen": -354.2195739746094,
"logps/rejected": -516.4483642578125,
"loss": 0.024,
"rewards/accuracies": 0.987500011920929,
"rewards/chosen": -2.647956609725952,
"rewards/margins": 14.303915023803711,
"rewards/rejected": -16.95186996459961,
"step": 1850
},
{
"epoch": 1.2,
"learning_rate": 4.4489407283980004e-07,
"logits/chosen": 0.03841676935553551,
"logits/rejected": 0.44405698776245117,
"logps/chosen": -405.54351806640625,
"logps/rejected": -583.1129760742188,
"loss": 0.0113,
"rewards/accuracies": 0.9750000238418579,
"rewards/chosen": -5.536320686340332,
"rewards/margins": 15.479736328125,
"rewards/rejected": -21.016056060791016,
"step": 1860
},
{
"epoch": 1.2,
"learning_rate": 4.442989764341823e-07,
"logits/chosen": 0.409410297870636,
"logits/rejected": 0.6078455448150635,
"logps/chosen": -425.523193359375,
"logps/rejected": -559.0198364257812,
"loss": 0.0116,
"rewards/accuracies": 1.0,
"rewards/chosen": -7.118569374084473,
"rewards/margins": 15.779667854309082,
"rewards/rejected": -22.898235321044922,
"step": 1870
},
{
"epoch": 1.21,
"learning_rate": 4.437038800285646e-07,
"logits/chosen": 0.023495376110076904,
"logits/rejected": 0.8307549357414246,
"logps/chosen": -367.8580017089844,
"logps/rejected": -482.8976135253906,
"loss": 0.0228,
"rewards/accuracies": 1.0,
"rewards/chosen": -4.339402198791504,
"rewards/margins": 14.32616901397705,
"rewards/rejected": -18.665569305419922,
"step": 1880
},
{
"epoch": 1.21,
"learning_rate": 4.431087836229469e-07,
"logits/chosen": -0.1776140034198761,
"logits/rejected": 0.4008842408657074,
"logps/chosen": -368.744873046875,
"logps/rejected": -584.6207885742188,
"loss": 0.0183,
"rewards/accuracies": 0.987500011920929,
"rewards/chosen": -4.053328514099121,
"rewards/margins": 16.808618545532227,
"rewards/rejected": -20.861948013305664,
"step": 1890
},
{
"epoch": 1.22,
"learning_rate": 4.4251368721732916e-07,
"logits/chosen": -0.10556875169277191,
"logits/rejected": 0.6400087475776672,
"logps/chosen": -395.7705993652344,
"logps/rejected": -568.189697265625,
"loss": 0.0327,
"rewards/accuracies": 1.0,
"rewards/chosen": -4.5353851318359375,
"rewards/margins": 15.502912521362305,
"rewards/rejected": -20.038297653198242,
"step": 1900
},
{
"epoch": 1.22,
"eval_logits/chosen": -0.3349679708480835,
"eval_logits/rejected": 0.1110042855143547,
"eval_logps/chosen": -412.0855712890625,
"eval_logps/rejected": -525.799560546875,
"eval_loss": 0.16337917745113373,
"eval_rewards/accuracies": 0.90625,
"eval_rewards/chosen": -8.27225112915039,
"eval_rewards/margins": 12.672636032104492,
"eval_rewards/rejected": -20.94488525390625,
"eval_runtime": 76.7623,
"eval_samples_per_second": 13.027,
"eval_steps_per_second": 0.417,
"step": 1900
},
{
"epoch": 1.23,
"learning_rate": 4.4191859081171146e-07,
"logits/chosen": 0.13517367839813232,
"logits/rejected": 0.3502965569496155,
"logps/chosen": -432.887451171875,
"logps/rejected": -610.2424926757812,
"loss": 0.0176,
"rewards/accuracies": 1.0,
"rewards/chosen": -5.2362380027771,
"rewards/margins": 17.042871475219727,
"rewards/rejected": -22.279109954833984,
"step": 1910
},
{
"epoch": 1.23,
"learning_rate": 4.413234944060938e-07,
"logits/chosen": -0.011228932067751884,
"logits/rejected": 0.4079880714416504,
"logps/chosen": -397.6374206542969,
"logps/rejected": -609.8255615234375,
"loss": 0.0084,
"rewards/accuracies": 1.0,
"rewards/chosen": -3.824807643890381,
"rewards/margins": 15.99610424041748,
"rewards/rejected": -19.820913314819336,
"step": 1920
},
{
"epoch": 1.24,
"learning_rate": 4.4072839800047605e-07,
"logits/chosen": -0.048855990171432495,
"logits/rejected": 0.2614760994911194,
"logps/chosen": -425.06182861328125,
"logps/rejected": -693.6744384765625,
"loss": 0.0236,
"rewards/accuracies": 0.9750000238418579,
"rewards/chosen": -4.437471866607666,
"rewards/margins": 17.300281524658203,
"rewards/rejected": -21.73775291442871,
"step": 1930
},
{
"epoch": 1.25,
"learning_rate": 4.4013330159485834e-07,
"logits/chosen": -0.6388794183731079,
"logits/rejected": -0.16498331725597382,
"logps/chosen": -455.348388671875,
"logps/rejected": -607.502685546875,
"loss": 0.0343,
"rewards/accuracies": 1.0,
"rewards/chosen": -2.9607555866241455,
"rewards/margins": 16.582752227783203,
"rewards/rejected": -19.543508529663086,
"step": 1940
},
{
"epoch": 1.25,
"learning_rate": 4.3953820518924064e-07,
"logits/chosen": -0.7284079194068909,
"logits/rejected": -0.07122499495744705,
"logps/chosen": -416.238525390625,
"logps/rejected": -586.7880249023438,
"loss": 0.0142,
"rewards/accuracies": 0.987500011920929,
"rewards/chosen": -4.067997932434082,
"rewards/margins": 16.282215118408203,
"rewards/rejected": -20.350215911865234,
"step": 1950
},
{
"epoch": 1.26,
"learning_rate": 4.3894310878362293e-07,
"logits/chosen": -0.7452508211135864,
"logits/rejected": 0.1260381042957306,
"logps/chosen": -449.06268310546875,
"logps/rejected": -529.99072265625,
"loss": 0.0258,
"rewards/accuracies": 0.987500011920929,
"rewards/chosen": -2.9167428016662598,
"rewards/margins": 15.735280990600586,
"rewards/rejected": -18.65202522277832,
"step": 1960
},
{
"epoch": 1.27,
"learning_rate": 4.3834801237800523e-07,
"logits/chosen": -0.20991234481334686,
"logits/rejected": 0.1470331847667694,
"logps/chosen": -391.4617919921875,
"logps/rejected": -590.724365234375,
"loss": 0.0332,
"rewards/accuracies": 0.987500011920929,
"rewards/chosen": -4.234528541564941,
"rewards/margins": 17.127511978149414,
"rewards/rejected": -21.362041473388672,
"step": 1970
},
{
"epoch": 1.27,
"learning_rate": 4.377529159723875e-07,
"logits/chosen": -0.1375136375427246,
"logits/rejected": 0.26706498861312866,
"logps/chosen": -444.4725646972656,
"logps/rejected": -529.6869506835938,
"loss": 0.0262,
"rewards/accuracies": 1.0,
"rewards/chosen": -5.4430928230285645,
"rewards/margins": 15.911623001098633,
"rewards/rejected": -21.354717254638672,
"step": 1980
},
{
"epoch": 1.28,
"learning_rate": 4.3715781956676976e-07,
"logits/chosen": -0.21475636959075928,
"logits/rejected": 0.3777112364768982,
"logps/chosen": -406.0322570800781,
"logps/rejected": -613.7669067382812,
"loss": 0.0202,
"rewards/accuracies": 1.0,
"rewards/chosen": -4.220062732696533,
"rewards/margins": 17.0797119140625,
"rewards/rejected": -21.299776077270508,
"step": 1990
},
{
"epoch": 1.29,
"learning_rate": 4.365627231611521e-07,
"logits/chosen": -0.42644554376602173,
"logits/rejected": 0.14130757749080658,
"logps/chosen": -414.5654296875,
"logps/rejected": -642.5784912109375,
"loss": 0.0182,
"rewards/accuracies": 1.0,
"rewards/chosen": -3.209155321121216,
"rewards/margins": 17.73525619506836,
"rewards/rejected": -20.944412231445312,
"step": 2000
},
{
"epoch": 1.29,
"eval_logits/chosen": -0.6391128301620483,
"eval_logits/rejected": -0.05341078341007233,
"eval_logps/chosen": -384.2879638671875,
"eval_logps/rejected": -496.6387023925781,
"eval_loss": 0.15107305347919464,
"eval_rewards/accuracies": 0.953125,
"eval_rewards/chosen": -5.492494106292725,
"eval_rewards/margins": 12.536298751831055,
"eval_rewards/rejected": -18.028793334960938,
"eval_runtime": 76.6391,
"eval_samples_per_second": 13.048,
"eval_steps_per_second": 0.418,
"step": 2000
},
{
"epoch": 1.29,
"learning_rate": 4.359676267555344e-07,
"logits/chosen": -0.516070544719696,
"logits/rejected": 0.47864165902137756,
"logps/chosen": -344.65240478515625,
"logps/rejected": -633.1707763671875,
"loss": 0.0116,
"rewards/accuracies": 0.9750000238418579,
"rewards/chosen": -4.215503692626953,
"rewards/margins": 20.52490234375,
"rewards/rejected": -24.740406036376953,
"step": 2010
},
{
"epoch": 1.3,
"learning_rate": 4.3537253034991665e-07,
"logits/chosen": -0.2483837604522705,
"logits/rejected": 0.22826921939849854,
"logps/chosen": -390.77874755859375,
"logps/rejected": -561.1676025390625,
"loss": 0.0096,
"rewards/accuracies": 1.0,
"rewards/chosen": -4.410299777984619,
"rewards/margins": 18.708133697509766,
"rewards/rejected": -23.11842918395996,
"step": 2020
},
{
"epoch": 1.3,
"learning_rate": 4.3477743394429894e-07,
"logits/chosen": -0.058499228209257126,
"logits/rejected": 0.5287091135978699,
"logps/chosen": -399.7431640625,
"logps/rejected": -509.27593994140625,
"loss": 0.0097,
"rewards/accuracies": 1.0,
"rewards/chosen": -4.519265651702881,
"rewards/margins": 16.500688552856445,
"rewards/rejected": -21.01995277404785,
"step": 2030
},
{
"epoch": 1.31,
"learning_rate": 4.3418233753868124e-07,
"logits/chosen": 0.22348590195178986,
"logits/rejected": 0.7664919495582581,
"logps/chosen": -411.91668701171875,
"logps/rejected": -567.1552124023438,
"loss": 0.0393,
"rewards/accuracies": 0.987500011920929,
"rewards/chosen": -5.37440299987793,
"rewards/margins": 16.45658302307129,
"rewards/rejected": -21.830982208251953,
"step": 2040
},
{
"epoch": 1.32,
"learning_rate": 4.3358724113306353e-07,
"logits/chosen": 0.23079347610473633,
"logits/rejected": 0.849484920501709,
"logps/chosen": -399.9824523925781,
"logps/rejected": -545.28271484375,
"loss": 0.0132,
"rewards/accuracies": 1.0,
"rewards/chosen": -5.708091735839844,
"rewards/margins": 18.776620864868164,
"rewards/rejected": -24.484712600708008,
"step": 2050
},
{
"epoch": 1.32,
"learning_rate": 4.3299214472744583e-07,
"logits/chosen": 0.5551694631576538,
"logits/rejected": 1.1838432550430298,
"logps/chosen": -406.03460693359375,
"logps/rejected": -533.4246826171875,
"loss": 0.0198,
"rewards/accuracies": 0.987500011920929,
"rewards/chosen": -6.767632961273193,
"rewards/margins": 15.5044527053833,
"rewards/rejected": -22.27208709716797,
"step": 2060
},
{
"epoch": 1.33,
"learning_rate": 4.323970483218281e-07,
"logits/chosen": 0.5585372447967529,
"logits/rejected": 1.1072343587875366,
"logps/chosen": -390.73870849609375,
"logps/rejected": -581.7123413085938,
"loss": 0.0109,
"rewards/accuracies": 1.0,
"rewards/chosen": -5.473759651184082,
"rewards/margins": 17.37258529663086,
"rewards/rejected": -22.846345901489258,
"step": 2070
},
{
"epoch": 1.34,
"learning_rate": 4.3180195191621036e-07,
"logits/chosen": 0.19852328300476074,
"logits/rejected": 0.6014672517776489,
"logps/chosen": -408.9750061035156,
"logps/rejected": -573.5753784179688,
"loss": 0.0317,
"rewards/accuracies": 0.9750000238418579,
"rewards/chosen": -3.9805541038513184,
"rewards/margins": 16.70760154724121,
"rewards/rejected": -20.688152313232422,
"step": 2080
},
{
"epoch": 1.34,
"learning_rate": 4.312068555105927e-07,
"logits/chosen": 0.2863103151321411,
"logits/rejected": 1.09657883644104,
"logps/chosen": -437.265380859375,
"logps/rejected": -549.7987060546875,
"loss": 0.013,
"rewards/accuracies": 1.0,
"rewards/chosen": -4.587510108947754,
"rewards/margins": 17.130268096923828,
"rewards/rejected": -21.7177791595459,
"step": 2090
},
{
"epoch": 1.35,
"learning_rate": 4.30611759104975e-07,
"logits/chosen": 0.16401129961013794,
"logits/rejected": 0.43381816148757935,
"logps/chosen": -367.1972351074219,
"logps/rejected": -574.0045776367188,
"loss": 0.0181,
"rewards/accuracies": 0.987500011920929,
"rewards/chosen": -3.683870315551758,
"rewards/margins": 18.48448944091797,
"rewards/rejected": -22.168359756469727,
"step": 2100
},
{
"epoch": 1.35,
"eval_logits/chosen": -0.38096773624420166,
"eval_logits/rejected": 0.2048163115978241,
"eval_logps/chosen": -379.0364074707031,
"eval_logps/rejected": -488.2851867675781,
"eval_loss": 0.16009199619293213,
"eval_rewards/accuracies": 0.9140625,
"eval_rewards/chosen": -4.9673357009887695,
"eval_rewards/margins": 12.22611141204834,
"eval_rewards/rejected": -17.19344711303711,
"eval_runtime": 77.0334,
"eval_samples_per_second": 12.981,
"eval_steps_per_second": 0.415,
"step": 2100
},
{
"epoch": 1.36,
"learning_rate": 4.3001666269935725e-07,
"logits/chosen": 0.4979768395423889,
"logits/rejected": 0.9932268261909485,
"logps/chosen": -439.8809509277344,
"logps/rejected": -572.9110107421875,
"loss": 0.0196,
"rewards/accuracies": 0.987500011920929,
"rewards/chosen": -4.661294937133789,
"rewards/margins": 16.660076141357422,
"rewards/rejected": -21.32137107849121,
"step": 2110
},
{
"epoch": 1.36,
"learning_rate": 4.2942156629373954e-07,
"logits/chosen": 0.38851994276046753,
"logits/rejected": 1.0913450717926025,
"logps/chosen": -391.9513854980469,
"logps/rejected": -557.0726318359375,
"loss": 0.0222,
"rewards/accuracies": 1.0,
"rewards/chosen": -4.788923263549805,
"rewards/margins": 16.37869644165039,
"rewards/rejected": -21.167619705200195,
"step": 2120
},
{
"epoch": 1.37,
"learning_rate": 4.288264698881219e-07,
"logits/chosen": 0.7406013607978821,
"logits/rejected": 1.1288832426071167,
"logps/chosen": -434.83880615234375,
"logps/rejected": -587.9083251953125,
"loss": 0.0055,
"rewards/accuracies": 1.0,
"rewards/chosen": -5.380326271057129,
"rewards/margins": 18.681997299194336,
"rewards/rejected": -24.06232261657715,
"step": 2130
},
{
"epoch": 1.38,
"learning_rate": 4.2823137348250413e-07,
"logits/chosen": 0.521850049495697,
"logits/rejected": 0.9274239540100098,
"logps/chosen": -380.3211669921875,
"logps/rejected": -581.7938232421875,
"loss": 0.0179,
"rewards/accuracies": 1.0,
"rewards/chosen": -4.798120021820068,
"rewards/margins": 17.523136138916016,
"rewards/rejected": -22.321256637573242,
"step": 2140
},
{
"epoch": 1.38,
"learning_rate": 4.2763627707688643e-07,
"logits/chosen": 0.12944528460502625,
"logits/rejected": 0.8914273381233215,
"logps/chosen": -443.59954833984375,
"logps/rejected": -575.8558349609375,
"loss": 0.0128,
"rewards/accuracies": 1.0,
"rewards/chosen": -4.318188190460205,
"rewards/margins": 17.411693572998047,
"rewards/rejected": -21.729881286621094,
"step": 2150
},
{
"epoch": 1.39,
"learning_rate": 4.270411806712687e-07,
"logits/chosen": -0.12583580613136292,
"logits/rejected": 0.797269344329834,
"logps/chosen": -445.8939514160156,
"logps/rejected": -576.5032958984375,
"loss": 0.0453,
"rewards/accuracies": 0.987500011920929,
"rewards/chosen": -6.454132080078125,
"rewards/margins": 17.303661346435547,
"rewards/rejected": -23.757795333862305,
"step": 2160
},
{
"epoch": 1.39,
"learning_rate": 4.26446084265651e-07,
"logits/chosen": -0.158376082777977,
"logits/rejected": 0.6286741495132446,
"logps/chosen": -425.6141662597656,
"logps/rejected": -587.5155029296875,
"loss": 0.004,
"rewards/accuracies": 1.0,
"rewards/chosen": -6.210710048675537,
"rewards/margins": 16.975194931030273,
"rewards/rejected": -23.18590545654297,
"step": 2170
},
{
"epoch": 1.4,
"learning_rate": 4.258509878600333e-07,
"logits/chosen": -0.22286149859428406,
"logits/rejected": 0.8965922594070435,
"logps/chosen": -482.9183654785156,
"logps/rejected": -620.5008544921875,
"loss": 0.0323,
"rewards/accuracies": 0.9750000238418579,
"rewards/chosen": -7.59103536605835,
"rewards/margins": 16.49752426147461,
"rewards/rejected": -24.088560104370117,
"step": 2180
},
{
"epoch": 1.41,
"learning_rate": 4.252558914544156e-07,
"logits/chosen": 0.22360090911388397,
"logits/rejected": 0.7913219928741455,
"logps/chosen": -467.44757080078125,
"logps/rejected": -694.8656616210938,
"loss": 0.0094,
"rewards/accuracies": 1.0,
"rewards/chosen": -8.326881408691406,
"rewards/margins": 19.117225646972656,
"rewards/rejected": -27.444107055664062,
"step": 2190
},
{
"epoch": 1.41,
"learning_rate": 4.2466079504879785e-07,
"logits/chosen": -0.1699143350124359,
"logits/rejected": 0.9544417262077332,
"logps/chosen": -444.05224609375,
"logps/rejected": -574.4688110351562,
"loss": 0.0107,
"rewards/accuracies": 0.987500011920929,
"rewards/chosen": -7.666506767272949,
"rewards/margins": 15.971659660339355,
"rewards/rejected": -23.638164520263672,
"step": 2200
},
{
"epoch": 1.41,
"eval_logits/chosen": -0.5077850222587585,
"eval_logits/rejected": 0.1793254315853119,
"eval_logps/chosen": -393.3812561035156,
"eval_logps/rejected": -497.8894958496094,
"eval_loss": 0.1433764100074768,
"eval_rewards/accuracies": 0.9296875,
"eval_rewards/chosen": -6.401820182800293,
"eval_rewards/margins": 11.752058029174805,
"eval_rewards/rejected": -18.15387725830078,
"eval_runtime": 76.5649,
"eval_samples_per_second": 13.061,
"eval_steps_per_second": 0.418,
"step": 2200
},
{
"epoch": 1.42,
"learning_rate": 4.240656986431802e-07,
"logits/chosen": 0.25252458453178406,
"logits/rejected": 0.6517130136489868,
"logps/chosen": -439.65478515625,
"logps/rejected": -589.1119995117188,
"loss": 0.023,
"rewards/accuracies": 1.0,
"rewards/chosen": -5.0924177169799805,
"rewards/margins": 17.32897186279297,
"rewards/rejected": -22.421390533447266,
"step": 2210
},
{
"epoch": 1.43,
"learning_rate": 4.234706022375625e-07,
"logits/chosen": 0.16541361808776855,
"logits/rejected": 0.7442089319229126,
"logps/chosen": -340.91363525390625,
"logps/rejected": -542.9158935546875,
"loss": 0.0378,
"rewards/accuracies": 0.9750000238418579,
"rewards/chosen": -6.545586585998535,
"rewards/margins": 16.971027374267578,
"rewards/rejected": -23.51661491394043,
"step": 2220
},
{
"epoch": 1.43,
"learning_rate": 4.2287550583194473e-07,
"logits/chosen": 0.12309785187244415,
"logits/rejected": 0.7068944573402405,
"logps/chosen": -416.27166748046875,
"logps/rejected": -671.1712646484375,
"loss": 0.0373,
"rewards/accuracies": 0.9750000238418579,
"rewards/chosen": -6.756946563720703,
"rewards/margins": 19.396137237548828,
"rewards/rejected": -26.1530818939209,
"step": 2230
},
{
"epoch": 1.44,
"learning_rate": 4.2228040942632703e-07,
"logits/chosen": -0.17779667675495148,
"logits/rejected": 0.7514945268630981,
"logps/chosen": -387.315185546875,
"logps/rejected": -554.4407348632812,
"loss": 0.0103,
"rewards/accuracies": 1.0,
"rewards/chosen": -6.591644287109375,
"rewards/margins": 16.485776901245117,
"rewards/rejected": -23.07741928100586,
"step": 2240
},
{
"epoch": 1.45,
"learning_rate": 4.216853130207093e-07,
"logits/chosen": -0.4565064311027527,
"logits/rejected": 0.8017956018447876,
"logps/chosen": -407.85028076171875,
"logps/rejected": -557.0631103515625,
"loss": 0.0244,
"rewards/accuracies": 1.0,
"rewards/chosen": -4.639117240905762,
"rewards/margins": 16.890506744384766,
"rewards/rejected": -21.529621124267578,
"step": 2250
},
{
"epoch": 1.45,
"learning_rate": 4.210902166150916e-07,
"logits/chosen": -0.1683514416217804,
"logits/rejected": 0.797057569026947,
"logps/chosen": -448.7789611816406,
"logps/rejected": -617.1752319335938,
"loss": 0.0067,
"rewards/accuracies": 0.987500011920929,
"rewards/chosen": -6.5801100730896,
"rewards/margins": 17.799306869506836,
"rewards/rejected": -24.379417419433594,
"step": 2260
},
{
"epoch": 1.46,
"learning_rate": 4.204951202094739e-07,
"logits/chosen": 0.12932386994361877,
"logits/rejected": 0.9783223867416382,
"logps/chosen": -392.2380676269531,
"logps/rejected": -563.818603515625,
"loss": 0.0214,
"rewards/accuracies": 1.0,
"rewards/chosen": -5.884858131408691,
"rewards/margins": 17.019224166870117,
"rewards/rejected": -22.904081344604492,
"step": 2270
},
{
"epoch": 1.47,
"learning_rate": 4.199000238038562e-07,
"logits/chosen": 0.17704737186431885,
"logits/rejected": 0.9605581164360046,
"logps/chosen": -393.4834289550781,
"logps/rejected": -520.2174072265625,
"loss": 0.0254,
"rewards/accuracies": 1.0,
"rewards/chosen": -4.810626983642578,
"rewards/margins": 17.23440170288086,
"rewards/rejected": -22.045028686523438,
"step": 2280
},
{
"epoch": 1.47,
"learning_rate": 4.1930492739823845e-07,
"logits/chosen": 0.23696310818195343,
"logits/rejected": 0.8548523783683777,
"logps/chosen": -413.873046875,
"logps/rejected": -593.3323974609375,
"loss": 0.0362,
"rewards/accuracies": 1.0,
"rewards/chosen": -5.712353229522705,
"rewards/margins": 18.40825653076172,
"rewards/rejected": -24.120609283447266,
"step": 2290
},
{
"epoch": 1.48,
"learning_rate": 4.187098309926208e-07,
"logits/chosen": 0.11162440478801727,
"logits/rejected": 0.8643198013305664,
"logps/chosen": -446.72216796875,
"logps/rejected": -609.0733642578125,
"loss": 0.0207,
"rewards/accuracies": 0.987500011920929,
"rewards/chosen": -7.024069786071777,
"rewards/margins": 17.165082931518555,
"rewards/rejected": -24.18915367126465,
"step": 2300
},
{
"epoch": 1.48,
"eval_logits/chosen": -0.3969336152076721,
"eval_logits/rejected": 0.30485790967941284,
"eval_logps/chosen": -409.54925537109375,
"eval_logps/rejected": -527.1227416992188,
"eval_loss": 0.1328170895576477,
"eval_rewards/accuracies": 0.9609375,
"eval_rewards/chosen": -8.018616676330566,
"eval_rewards/margins": 13.058581352233887,
"eval_rewards/rejected": -21.077199935913086,
"eval_runtime": 76.7293,
"eval_samples_per_second": 13.033,
"eval_steps_per_second": 0.417,
"step": 2300
},
{
"epoch": 1.48,
"learning_rate": 4.181147345870031e-07,
"logits/chosen": -0.14238300919532776,
"logits/rejected": 0.5450645685195923,
"logps/chosen": -449.2244567871094,
"logps/rejected": -655.3807373046875,
"loss": 0.0182,
"rewards/accuracies": 0.987500011920929,
"rewards/chosen": -6.570483207702637,
"rewards/margins": 16.494831085205078,
"rewards/rejected": -23.065311431884766,
"step": 2310
},
{
"epoch": 1.49,
"learning_rate": 4.1751963818138534e-07,
"logits/chosen": -0.011528102681040764,
"logits/rejected": 0.896866500377655,
"logps/chosen": -477.01226806640625,
"logps/rejected": -661.6298217773438,
"loss": 0.0098,
"rewards/accuracies": 0.987500011920929,
"rewards/chosen": -6.549506187438965,
"rewards/margins": 19.142070770263672,
"rewards/rejected": -25.691574096679688,
"step": 2320
},
{
"epoch": 1.5,
"learning_rate": 4.1692454177576763e-07,
"logits/chosen": -0.2801293432712555,
"logits/rejected": 0.5617603063583374,
"logps/chosen": -442.40167236328125,
"logps/rejected": -573.85205078125,
"loss": 0.0289,
"rewards/accuracies": 0.987500011920929,
"rewards/chosen": -6.618128776550293,
"rewards/margins": 17.062252044677734,
"rewards/rejected": -23.680381774902344,
"step": 2330
},
{
"epoch": 1.5,
"learning_rate": 4.1632944537015e-07,
"logits/chosen": -0.1246495246887207,
"logits/rejected": 0.732117772102356,
"logps/chosen": -445.9453125,
"logps/rejected": -648.7326049804688,
"loss": 0.0567,
"rewards/accuracies": 0.987500011920929,
"rewards/chosen": -7.793832302093506,
"rewards/margins": 16.911283493041992,
"rewards/rejected": -24.705114364624023,
"step": 2340
},
{
"epoch": 1.51,
"learning_rate": 4.157343489645322e-07,
"logits/chosen": -0.13531716167926788,
"logits/rejected": 0.6813434362411499,
"logps/chosen": -500.760986328125,
"logps/rejected": -748.8970947265625,
"loss": 0.0249,
"rewards/accuracies": 0.987500011920929,
"rewards/chosen": -8.698297500610352,
"rewards/margins": 22.551307678222656,
"rewards/rejected": -31.249608993530273,
"step": 2350
},
{
"epoch": 1.52,
"learning_rate": 4.151392525589145e-07,
"logits/chosen": -0.17930714786052704,
"logits/rejected": 0.5675514340400696,
"logps/chosen": -507.093994140625,
"logps/rejected": -617.2818603515625,
"loss": 0.0077,
"rewards/accuracies": 0.987500011920929,
"rewards/chosen": -9.938199043273926,
"rewards/margins": 18.702774047851562,
"rewards/rejected": -28.640972137451172,
"step": 2360
},
{
"epoch": 1.52,
"learning_rate": 4.145441561532968e-07,
"logits/chosen": -0.02535531297326088,
"logits/rejected": 0.5533861517906189,
"logps/chosen": -441.52362060546875,
"logps/rejected": -658.696044921875,
"loss": 0.0219,
"rewards/accuracies": 0.987500011920929,
"rewards/chosen": -7.094923496246338,
"rewards/margins": 19.90808868408203,
"rewards/rejected": -27.00301170349121,
"step": 2370
},
{
"epoch": 1.53,
"learning_rate": 4.139490597476791e-07,
"logits/chosen": -0.01725180074572563,
"logits/rejected": 0.545805811882019,
"logps/chosen": -431.83197021484375,
"logps/rejected": -565.3300170898438,
"loss": 0.0116,
"rewards/accuracies": 1.0,
"rewards/chosen": -4.328795909881592,
"rewards/margins": 17.926631927490234,
"rewards/rejected": -22.255428314208984,
"step": 2380
},
{
"epoch": 1.54,
"learning_rate": 4.133539633420614e-07,
"logits/chosen": -0.21750584244728088,
"logits/rejected": 0.8260287046432495,
"logps/chosen": -426.7081604003906,
"logps/rejected": -571.2562255859375,
"loss": 0.0139,
"rewards/accuracies": 1.0,
"rewards/chosen": -4.599783897399902,
"rewards/margins": 18.57124900817871,
"rewards/rejected": -23.17103385925293,
"step": 2390
},
{
"epoch": 1.54,
"learning_rate": 4.127588669364437e-07,
"logits/chosen": 0.12132026255130768,
"logits/rejected": 0.586942195892334,
"logps/chosen": -387.7074890136719,
"logps/rejected": -575.88623046875,
"loss": 0.0237,
"rewards/accuracies": 0.987500011920929,
"rewards/chosen": -5.400722980499268,
"rewards/margins": 19.196186065673828,
"rewards/rejected": -24.596908569335938,
"step": 2400
},
{
"epoch": 1.54,
"eval_logits/chosen": -0.4777054190635681,
"eval_logits/rejected": 0.22216691076755524,
"eval_logps/chosen": -392.7734069824219,
"eval_logps/rejected": -515.349365234375,
"eval_loss": 0.1322525590658188,
"eval_rewards/accuracies": 0.9765625,
"eval_rewards/chosen": -6.341035842895508,
"eval_rewards/margins": 13.558823585510254,
"eval_rewards/rejected": -19.899858474731445,
"eval_runtime": 76.575,
"eval_samples_per_second": 13.059,
"eval_steps_per_second": 0.418,
"step": 2400
},
{
"epoch": 1.55,
"learning_rate": 4.1216377053082594e-07,
"logits/chosen": 0.19748568534851074,
"logits/rejected": 0.791664719581604,
"logps/chosen": -461.057861328125,
"logps/rejected": -645.6085205078125,
"loss": 0.0145,
"rewards/accuracies": 1.0,
"rewards/chosen": -6.15015172958374,
"rewards/margins": 19.398887634277344,
"rewards/rejected": -25.549039840698242,
"step": 2410
},
{
"epoch": 1.56,
"learning_rate": 4.115686741252083e-07,
"logits/chosen": 0.5446011424064636,
"logits/rejected": 0.8287714123725891,
"logps/chosen": -428.01068115234375,
"logps/rejected": -596.6788330078125,
"loss": 0.0192,
"rewards/accuracies": 0.987500011920929,
"rewards/chosen": -6.512643337249756,
"rewards/margins": 18.827342987060547,
"rewards/rejected": -25.33998680114746,
"step": 2420
},
{
"epoch": 1.56,
"learning_rate": 4.109735777195906e-07,
"logits/chosen": 0.2738257646560669,
"logits/rejected": 0.6724745035171509,
"logps/chosen": -443.29852294921875,
"logps/rejected": -617.8185424804688,
"loss": 0.0311,
"rewards/accuracies": 1.0,
"rewards/chosen": -4.934741497039795,
"rewards/margins": 18.175350189208984,
"rewards/rejected": -23.110088348388672,
"step": 2430
},
{
"epoch": 1.57,
"learning_rate": 4.103784813139728e-07,
"logits/chosen": -0.21786899864673615,
"logits/rejected": 0.508806049823761,
"logps/chosen": -386.5865173339844,
"logps/rejected": -540.3848876953125,
"loss": 0.011,
"rewards/accuracies": 1.0,
"rewards/chosen": -3.1630804538726807,
"rewards/margins": 18.559303283691406,
"rewards/rejected": -21.72238540649414,
"step": 2440
},
{
"epoch": 1.57,
"learning_rate": 4.097833849083551e-07,
"logits/chosen": 0.27359846234321594,
"logits/rejected": 0.8219190835952759,
"logps/chosen": -489.2318420410156,
"logps/rejected": -654.9146118164062,
"loss": 0.0167,
"rewards/accuracies": 0.987500011920929,
"rewards/chosen": -6.5973005294799805,
"rewards/margins": 19.05999183654785,
"rewards/rejected": -25.65729331970215,
"step": 2450
},
{
"epoch": 1.58,
"learning_rate": 4.091882885027374e-07,
"logits/chosen": 0.23141400516033173,
"logits/rejected": 0.8404422998428345,
"logps/chosen": -470.26300048828125,
"logps/rejected": -647.7288208007812,
"loss": 0.0143,
"rewards/accuracies": 1.0,
"rewards/chosen": -5.572993278503418,
"rewards/margins": 19.47740364074707,
"rewards/rejected": -25.050395965576172,
"step": 2460
},
{
"epoch": 1.59,
"learning_rate": 4.0859319209711976e-07,
"logits/chosen": 0.21321940422058105,
"logits/rejected": 0.9413207769393921,
"logps/chosen": -481.8892517089844,
"logps/rejected": -568.3132934570312,
"loss": 0.0106,
"rewards/accuracies": 0.987500011920929,
"rewards/chosen": -5.117605686187744,
"rewards/margins": 17.3239688873291,
"rewards/rejected": -22.441572189331055,
"step": 2470
},
{
"epoch": 1.59,
"learning_rate": 4.07998095691502e-07,
"logits/chosen": 0.34440717101097107,
"logits/rejected": 1.0420324802398682,
"logps/chosen": -412.06756591796875,
"logps/rejected": -661.225830078125,
"loss": 0.0085,
"rewards/accuracies": 0.987500011920929,
"rewards/chosen": -5.927117347717285,
"rewards/margins": 19.294017791748047,
"rewards/rejected": -25.22113609313965,
"step": 2480
},
{
"epoch": 1.6,
"learning_rate": 4.074029992858843e-07,
"logits/chosen": 0.617647647857666,
"logits/rejected": 1.0356745719909668,
"logps/chosen": -466.6419372558594,
"logps/rejected": -593.55517578125,
"loss": 0.027,
"rewards/accuracies": 1.0,
"rewards/chosen": -5.356731414794922,
"rewards/margins": 18.503705978393555,
"rewards/rejected": -23.860441207885742,
"step": 2490
},
{
"epoch": 1.61,
"learning_rate": 4.0680790288026654e-07,
"logits/chosen": 0.6595112681388855,
"logits/rejected": 1.4305124282836914,
"logps/chosen": -434.178466796875,
"logps/rejected": -573.2113647460938,
"loss": 0.0269,
"rewards/accuracies": 0.987500011920929,
"rewards/chosen": -6.551410675048828,
"rewards/margins": 15.67224407196045,
"rewards/rejected": -22.223651885986328,
"step": 2500
},
{
"epoch": 1.61,
"eval_logits/chosen": -0.08062884956598282,
"eval_logits/rejected": 0.4895774722099304,
"eval_logps/chosen": -393.9366149902344,
"eval_logps/rejected": -509.43963623046875,
"eval_loss": 0.151441290974617,
"eval_rewards/accuracies": 0.96875,
"eval_rewards/chosen": -6.457356929779053,
"eval_rewards/margins": 12.851531982421875,
"eval_rewards/rejected": -19.308889389038086,
"eval_runtime": 76.6294,
"eval_samples_per_second": 13.05,
"eval_steps_per_second": 0.418,
"step": 2500
},
{
"epoch": 1.61,
"learning_rate": 4.062128064746489e-07,
"logits/chosen": 0.7047534584999084,
"logits/rejected": 1.2956479787826538,
"logps/chosen": -373.82366943359375,
"logps/rejected": -593.6276245117188,
"loss": 0.0182,
"rewards/accuracies": 0.987500011920929,
"rewards/chosen": -6.4311394691467285,
"rewards/margins": 15.736666679382324,
"rewards/rejected": -22.167804718017578,
"step": 2510
},
{
"epoch": 1.62,
"learning_rate": 4.056177100690312e-07,
"logits/chosen": 0.23054015636444092,
"logits/rejected": 0.9308466911315918,
"logps/chosen": -403.5622253417969,
"logps/rejected": -522.812744140625,
"loss": 0.03,
"rewards/accuracies": 0.949999988079071,
"rewards/chosen": -7.561907768249512,
"rewards/margins": 15.371617317199707,
"rewards/rejected": -22.933523178100586,
"step": 2520
},
{
"epoch": 1.63,
"learning_rate": 4.050226136634135e-07,
"logits/chosen": 0.061969585716724396,
"logits/rejected": 0.7286633253097534,
"logps/chosen": -414.0792541503906,
"logps/rejected": -600.1736450195312,
"loss": 0.0162,
"rewards/accuracies": 0.9750000238418579,
"rewards/chosen": -5.574122428894043,
"rewards/margins": 19.657848358154297,
"rewards/rejected": -25.231969833374023,
"step": 2530
},
{
"epoch": 1.63,
"learning_rate": 4.044275172577957e-07,
"logits/chosen": 0.05762529373168945,
"logits/rejected": 1.0145411491394043,
"logps/chosen": -445.4334411621094,
"logps/rejected": -645.9708251953125,
"loss": 0.0138,
"rewards/accuracies": 0.9750000238418579,
"rewards/chosen": -7.179165840148926,
"rewards/margins": 19.949363708496094,
"rewards/rejected": -27.128530502319336,
"step": 2540
},
{
"epoch": 1.64,
"learning_rate": 4.0383242085217806e-07,
"logits/chosen": 0.2469167709350586,
"logits/rejected": 0.8062774538993835,
"logps/chosen": -454.03668212890625,
"logps/rejected": -624.8360595703125,
"loss": 0.0212,
"rewards/accuracies": 0.987500011920929,
"rewards/chosen": -6.777960777282715,
"rewards/margins": 17.905128479003906,
"rewards/rejected": -24.68309211730957,
"step": 2550
},
{
"epoch": 1.65,
"learning_rate": 4.0323732444656036e-07,
"logits/chosen": 0.52375328540802,
"logits/rejected": 1.2141704559326172,
"logps/chosen": -423.7889099121094,
"logps/rejected": -609.6292114257812,
"loss": 0.0147,
"rewards/accuracies": 1.0,
"rewards/chosen": -6.806221961975098,
"rewards/margins": 18.522531509399414,
"rewards/rejected": -25.328754425048828,
"step": 2560
},
{
"epoch": 1.65,
"learning_rate": 4.026422280409426e-07,
"logits/chosen": 0.5363117456436157,
"logits/rejected": 0.9718011617660522,
"logps/chosen": -423.1117248535156,
"logps/rejected": -559.2742309570312,
"loss": 0.0108,
"rewards/accuracies": 1.0,
"rewards/chosen": -5.507771015167236,
"rewards/margins": 17.65149688720703,
"rewards/rejected": -23.159269332885742,
"step": 2570
},
{
"epoch": 1.66,
"learning_rate": 4.020471316353249e-07,
"logits/chosen": 0.7948285341262817,
"logits/rejected": 1.6965904235839844,
"logps/chosen": -450.24945068359375,
"logps/rejected": -587.7679443359375,
"loss": 0.0174,
"rewards/accuracies": 1.0,
"rewards/chosen": -7.323611259460449,
"rewards/margins": 19.197834014892578,
"rewards/rejected": -26.521448135375977,
"step": 2580
},
{
"epoch": 1.66,
"learning_rate": 4.0145203522970724e-07,
"logits/chosen": 0.6111919283866882,
"logits/rejected": 1.2076199054718018,
"logps/chosen": -393.1330871582031,
"logps/rejected": -658.733154296875,
"loss": 0.0212,
"rewards/accuracies": 1.0,
"rewards/chosen": -6.268210411071777,
"rewards/margins": 22.285709381103516,
"rewards/rejected": -28.553918838500977,
"step": 2590
},
{
"epoch": 1.67,
"learning_rate": 4.008569388240895e-07,
"logits/chosen": 0.8838823437690735,
"logits/rejected": 1.2930363416671753,
"logps/chosen": -433.79248046875,
"logps/rejected": -662.343994140625,
"loss": 0.0183,
"rewards/accuracies": 1.0,
"rewards/chosen": -6.181797027587891,
"rewards/margins": 20.533702850341797,
"rewards/rejected": -26.715499877929688,
"step": 2600
},
{
"epoch": 1.67,
"eval_logits/chosen": 0.019558865576982498,
"eval_logits/rejected": 0.6054279804229736,
"eval_logps/chosen": -407.05767822265625,
"eval_logps/rejected": -531.6161499023438,
"eval_loss": 0.177625834941864,
"eval_rewards/accuracies": 0.9375,
"eval_rewards/chosen": -7.769463539123535,
"eval_rewards/margins": 13.75707721710205,
"eval_rewards/rejected": -21.526540756225586,
"eval_runtime": 76.8929,
"eval_samples_per_second": 13.005,
"eval_steps_per_second": 0.416,
"step": 2600
},
{
"epoch": 1.68,
"learning_rate": 4.002618424184718e-07,
"logits/chosen": 0.37892434000968933,
"logits/rejected": 0.9920595288276672,
"logps/chosen": -391.6972351074219,
"logps/rejected": -585.7908935546875,
"loss": 0.0218,
"rewards/accuracies": 1.0,
"rewards/chosen": -5.025783538818359,
"rewards/margins": 17.12643814086914,
"rewards/rejected": -22.1522216796875,
"step": 2610
},
{
"epoch": 1.68,
"learning_rate": 3.996667460128541e-07,
"logits/chosen": 0.08298696577548981,
"logits/rejected": 0.9364360570907593,
"logps/chosen": -374.4390563964844,
"logps/rejected": -576.79541015625,
"loss": 0.0334,
"rewards/accuracies": 0.9624999761581421,
"rewards/chosen": -5.216673851013184,
"rewards/margins": 18.307117462158203,
"rewards/rejected": -23.523792266845703,
"step": 2620
},
{
"epoch": 1.69,
"learning_rate": 3.990716496072363e-07,
"logits/chosen": 0.30024105310440063,
"logits/rejected": 0.9240363836288452,
"logps/chosen": -419.29510498046875,
"logps/rejected": -590.0726928710938,
"loss": 0.0232,
"rewards/accuracies": 1.0,
"rewards/chosen": -6.619715690612793,
"rewards/margins": 17.221851348876953,
"rewards/rejected": -23.841564178466797,
"step": 2630
},
{
"epoch": 1.7,
"learning_rate": 3.9847655320161867e-07,
"logits/chosen": 0.2850233018398285,
"logits/rejected": 0.6386219263076782,
"logps/chosen": -397.0573425292969,
"logps/rejected": -577.4563598632812,
"loss": 0.0089,
"rewards/accuracies": 0.9750000238418579,
"rewards/chosen": -6.049335479736328,
"rewards/margins": 16.51738739013672,
"rewards/rejected": -22.56671905517578,
"step": 2640
},
{
"epoch": 1.7,
"learning_rate": 3.9788145679600096e-07,
"logits/chosen": 0.2091820240020752,
"logits/rejected": 0.9494321942329407,
"logps/chosen": -415.09869384765625,
"logps/rejected": -584.553466796875,
"loss": 0.0211,
"rewards/accuracies": 0.987500011920929,
"rewards/chosen": -5.250141620635986,
"rewards/margins": 17.63558578491211,
"rewards/rejected": -22.88572883605957,
"step": 2650
},
{
"epoch": 1.71,
"learning_rate": 3.972863603903832e-07,
"logits/chosen": 0.15588752925395966,
"logits/rejected": 0.9101377725601196,
"logps/chosen": -440.03546142578125,
"logps/rejected": -556.76953125,
"loss": 0.0258,
"rewards/accuracies": 0.987500011920929,
"rewards/chosen": -6.330071449279785,
"rewards/margins": 16.938446044921875,
"rewards/rejected": -23.26851463317871,
"step": 2660
},
{
"epoch": 1.72,
"learning_rate": 3.966912639847655e-07,
"logits/chosen": 0.43947821855545044,
"logits/rejected": 1.1439796686172485,
"logps/chosen": -408.59869384765625,
"logps/rejected": -561.8624267578125,
"loss": 0.0274,
"rewards/accuracies": 0.9750000238418579,
"rewards/chosen": -6.567991733551025,
"rewards/margins": 18.473587036132812,
"rewards/rejected": -25.041580200195312,
"step": 2670
},
{
"epoch": 1.72,
"learning_rate": 3.9609616757914784e-07,
"logits/chosen": 0.639157772064209,
"logits/rejected": 1.2126656770706177,
"logps/chosen": -433.49432373046875,
"logps/rejected": -545.498779296875,
"loss": 0.0441,
"rewards/accuracies": 0.987500011920929,
"rewards/chosen": -6.019906044006348,
"rewards/margins": 16.54349708557129,
"rewards/rejected": -22.563400268554688,
"step": 2680
},
{
"epoch": 1.73,
"learning_rate": 3.955010711735301e-07,
"logits/chosen": 1.0705523490905762,
"logits/rejected": 1.574342966079712,
"logps/chosen": -402.82073974609375,
"logps/rejected": -556.3751831054688,
"loss": 0.0333,
"rewards/accuracies": 0.987500011920929,
"rewards/chosen": -6.881136417388916,
"rewards/margins": 16.096803665161133,
"rewards/rejected": -22.977941513061523,
"step": 2690
},
{
"epoch": 1.74,
"learning_rate": 3.949059747679124e-07,
"logits/chosen": 0.6093908548355103,
"logits/rejected": 1.4320948123931885,
"logps/chosen": -416.2469177246094,
"logps/rejected": -563.9771118164062,
"loss": 0.0219,
"rewards/accuracies": 1.0,
"rewards/chosen": -5.77686071395874,
"rewards/margins": 17.765539169311523,
"rewards/rejected": -23.54239845275879,
"step": 2700
},
{
"epoch": 1.74,
"eval_logits/chosen": -0.009464547038078308,
"eval_logits/rejected": 0.5373153686523438,
"eval_logps/chosen": -408.0279541015625,
"eval_logps/rejected": -531.8214721679688,
"eval_loss": 0.14935775101184845,
"eval_rewards/accuracies": 0.9296875,
"eval_rewards/chosen": -7.866490364074707,
"eval_rewards/margins": 13.680585861206055,
"eval_rewards/rejected": -21.547077178955078,
"eval_runtime": 76.6739,
"eval_samples_per_second": 13.042,
"eval_steps_per_second": 0.417,
"step": 2700
},
{
"epoch": 1.74,
"learning_rate": 3.943108783622947e-07,
"logits/chosen": 0.5636851787567139,
"logits/rejected": 0.969277024269104,
"logps/chosen": -465.9778747558594,
"logps/rejected": -580.30810546875,
"loss": 0.0097,
"rewards/accuracies": 1.0,
"rewards/chosen": -7.8137688636779785,
"rewards/margins": 17.516647338867188,
"rewards/rejected": -25.330415725708008,
"step": 2710
},
{
"epoch": 1.75,
"learning_rate": 3.9371578195667697e-07,
"logits/chosen": 0.28570881485939026,
"logits/rejected": 1.0974262952804565,
"logps/chosen": -420.02520751953125,
"logps/rejected": -557.4937744140625,
"loss": 0.0242,
"rewards/accuracies": 0.987500011920929,
"rewards/chosen": -5.895697116851807,
"rewards/margins": 18.642253875732422,
"rewards/rejected": -24.53795051574707,
"step": 2720
},
{
"epoch": 1.75,
"learning_rate": 3.9312068555105927e-07,
"logits/chosen": 0.2833688259124756,
"logits/rejected": 1.2028071880340576,
"logps/chosen": -428.30242919921875,
"logps/rejected": -601.4442138671875,
"loss": 0.022,
"rewards/accuracies": 1.0,
"rewards/chosen": -5.424818992614746,
"rewards/margins": 16.82640838623047,
"rewards/rejected": -22.2512264251709,
"step": 2730
},
{
"epoch": 1.76,
"learning_rate": 3.9252558914544156e-07,
"logits/chosen": -0.17283181846141815,
"logits/rejected": 0.6285573840141296,
"logps/chosen": -407.96063232421875,
"logps/rejected": -656.8048095703125,
"loss": 0.0084,
"rewards/accuracies": 1.0,
"rewards/chosen": -5.4999213218688965,
"rewards/margins": 18.290767669677734,
"rewards/rejected": -23.790691375732422,
"step": 2740
},
{
"epoch": 1.77,
"learning_rate": 3.919304927398238e-07,
"logits/chosen": 0.1602792888879776,
"logits/rejected": 0.6874667406082153,
"logps/chosen": -375.77960205078125,
"logps/rejected": -546.3209228515625,
"loss": 0.0288,
"rewards/accuracies": 0.987500011920929,
"rewards/chosen": -5.720765113830566,
"rewards/margins": 16.913997650146484,
"rewards/rejected": -22.634761810302734,
"step": 2750
},
{
"epoch": 1.77,
"learning_rate": 3.9133539633420615e-07,
"logits/chosen": 0.2454497367143631,
"logits/rejected": 0.6986191272735596,
"logps/chosen": -390.337890625,
"logps/rejected": -560.3748779296875,
"loss": 0.0155,
"rewards/accuracies": 1.0,
"rewards/chosen": -5.7264814376831055,
"rewards/margins": 17.356287002563477,
"rewards/rejected": -23.082767486572266,
"step": 2760
},
{
"epoch": 1.78,
"learning_rate": 3.9074029992858845e-07,
"logits/chosen": 0.38854673504829407,
"logits/rejected": 1.452370047569275,
"logps/chosen": -415.73297119140625,
"logps/rejected": -565.9852294921875,
"loss": 0.016,
"rewards/accuracies": 1.0,
"rewards/chosen": -6.159702777862549,
"rewards/margins": 17.43438148498535,
"rewards/rejected": -23.594083786010742,
"step": 2770
},
{
"epoch": 1.79,
"learning_rate": 3.901452035229707e-07,
"logits/chosen": 0.7071353197097778,
"logits/rejected": 1.089143991470337,
"logps/chosen": -402.5384216308594,
"logps/rejected": -598.9990234375,
"loss": 0.0203,
"rewards/accuracies": 0.987500011920929,
"rewards/chosen": -7.571539402008057,
"rewards/margins": 18.129215240478516,
"rewards/rejected": -25.700754165649414,
"step": 2780
},
{
"epoch": 1.79,
"learning_rate": 3.89550107117353e-07,
"logits/chosen": 0.5019019842147827,
"logits/rejected": 1.2631019353866577,
"logps/chosen": -412.53607177734375,
"logps/rejected": -484.9837341308594,
"loss": 0.0231,
"rewards/accuracies": 0.9750000238418579,
"rewards/chosen": -4.363646984100342,
"rewards/margins": 15.107401847839355,
"rewards/rejected": -19.471050262451172,
"step": 2790
},
{
"epoch": 1.8,
"learning_rate": 3.8895501071173533e-07,
"logits/chosen": 0.5883782505989075,
"logits/rejected": 1.2746624946594238,
"logps/chosen": -431.44287109375,
"logps/rejected": -637.4125366210938,
"loss": 0.0335,
"rewards/accuracies": 0.987500011920929,
"rewards/chosen": -6.525457859039307,
"rewards/margins": 19.470340728759766,
"rewards/rejected": -25.995798110961914,
"step": 2800
},
{
"epoch": 1.8,
"eval_logits/chosen": -0.053867146372795105,
"eval_logits/rejected": 0.6149381399154663,
"eval_logps/chosen": -404.7615051269531,
"eval_logps/rejected": -530.05126953125,
"eval_loss": 0.15537042915821075,
"eval_rewards/accuracies": 0.9609375,
"eval_rewards/chosen": -7.539842128753662,
"eval_rewards/margins": 13.830206871032715,
"eval_rewards/rejected": -21.37004852294922,
"eval_runtime": 76.7186,
"eval_samples_per_second": 13.035,
"eval_steps_per_second": 0.417,
"step": 2800
},
{
"epoch": 1.81,
"learning_rate": 3.8835991430611757e-07,
"logits/chosen": 0.6285505294799805,
"logits/rejected": 1.1924630403518677,
"logps/chosen": -426.795166015625,
"logps/rejected": -537.3714599609375,
"loss": 0.0291,
"rewards/accuracies": 0.9750000238418579,
"rewards/chosen": -6.4925079345703125,
"rewards/margins": 17.20037078857422,
"rewards/rejected": -23.69287872314453,
"step": 2810
},
{
"epoch": 1.81,
"learning_rate": 3.8776481790049987e-07,
"logits/chosen": 0.3737705945968628,
"logits/rejected": 1.088739275932312,
"logps/chosen": -472.11669921875,
"logps/rejected": -613.3414916992188,
"loss": 0.0219,
"rewards/accuracies": 1.0,
"rewards/chosen": -5.82584285736084,
"rewards/margins": 20.265369415283203,
"rewards/rejected": -26.09120750427246,
"step": 2820
},
{
"epoch": 1.82,
"learning_rate": 3.8716972149488216e-07,
"logits/chosen": 0.4724366068840027,
"logits/rejected": 1.5313258171081543,
"logps/chosen": -474.59326171875,
"logps/rejected": -591.4876098632812,
"loss": 0.0295,
"rewards/accuracies": 1.0,
"rewards/chosen": -5.544611930847168,
"rewards/margins": 20.092144012451172,
"rewards/rejected": -25.636754989624023,
"step": 2830
},
{
"epoch": 1.83,
"learning_rate": 3.865746250892644e-07,
"logits/chosen": 0.6960526704788208,
"logits/rejected": 1.0484097003936768,
"logps/chosen": -442.74298095703125,
"logps/rejected": -641.082763671875,
"loss": 0.0359,
"rewards/accuracies": 0.9750000238418579,
"rewards/chosen": -5.976676940917969,
"rewards/margins": 18.82603645324707,
"rewards/rejected": -24.802715301513672,
"step": 2840
},
{
"epoch": 1.83,
"learning_rate": 3.8597952868364675e-07,
"logits/chosen": 0.6734079122543335,
"logits/rejected": 1.2869551181793213,
"logps/chosen": -410.61846923828125,
"logps/rejected": -535.3711547851562,
"loss": 0.0256,
"rewards/accuracies": 0.9750000238418579,
"rewards/chosen": -5.3238630294799805,
"rewards/margins": 16.99979019165039,
"rewards/rejected": -22.323650360107422,
"step": 2850
},
{
"epoch": 1.84,
"learning_rate": 3.8538443227802905e-07,
"logits/chosen": 0.39041590690612793,
"logits/rejected": 1.0508239269256592,
"logps/chosen": -455.080322265625,
"logps/rejected": -550.6773681640625,
"loss": 0.0194,
"rewards/accuracies": 1.0,
"rewards/chosen": -4.037131309509277,
"rewards/margins": 17.071130752563477,
"rewards/rejected": -21.10826301574707,
"step": 2860
},
{
"epoch": 1.84,
"learning_rate": 3.847893358724113e-07,
"logits/chosen": 0.21260789036750793,
"logits/rejected": 1.1434606313705444,
"logps/chosen": -434.0419006347656,
"logps/rejected": -555.1299438476562,
"loss": 0.0233,
"rewards/accuracies": 0.987500011920929,
"rewards/chosen": -5.177916526794434,
"rewards/margins": 15.855340957641602,
"rewards/rejected": -21.03325843811035,
"step": 2870
},
{
"epoch": 1.85,
"learning_rate": 3.841942394667936e-07,
"logits/chosen": 0.26587316393852234,
"logits/rejected": 1.1289548873901367,
"logps/chosen": -388.0738830566406,
"logps/rejected": -598.3447265625,
"loss": 0.0343,
"rewards/accuracies": 0.9624999761581421,
"rewards/chosen": -5.688809394836426,
"rewards/margins": 18.783828735351562,
"rewards/rejected": -24.472637176513672,
"step": 2880
},
{
"epoch": 1.86,
"learning_rate": 3.8359914306117593e-07,
"logits/chosen": 0.9435780644416809,
"logits/rejected": 1.0387550592422485,
"logps/chosen": -420.2930603027344,
"logps/rejected": -602.9673461914062,
"loss": 0.0159,
"rewards/accuracies": 0.987500011920929,
"rewards/chosen": -6.682863712310791,
"rewards/margins": 18.17510986328125,
"rewards/rejected": -24.857975006103516,
"step": 2890
},
{
"epoch": 1.86,
"learning_rate": 3.8300404665555817e-07,
"logits/chosen": 0.5122434496879578,
"logits/rejected": 1.1984080076217651,
"logps/chosen": -479.920654296875,
"logps/rejected": -683.6166381835938,
"loss": 0.0145,
"rewards/accuracies": 1.0,
"rewards/chosen": -6.739544868469238,
"rewards/margins": 21.551074981689453,
"rewards/rejected": -28.290618896484375,
"step": 2900
},
{
"epoch": 1.86,
"eval_logits/chosen": -0.22228558361530304,
"eval_logits/rejected": 0.4165645241737366,
"eval_logps/chosen": -403.52783203125,
"eval_logps/rejected": -530.5889892578125,
"eval_loss": 0.1386074274778366,
"eval_rewards/accuracies": 0.9609375,
"eval_rewards/chosen": -7.416477680206299,
"eval_rewards/margins": 14.007347106933594,
"eval_rewards/rejected": -21.423826217651367,
"eval_runtime": 76.7881,
"eval_samples_per_second": 13.023,
"eval_steps_per_second": 0.417,
"step": 2900
},
{
"epoch": 1.87,
"learning_rate": 3.8240895024994047e-07,
"logits/chosen": 0.35297220945358276,
"logits/rejected": 1.0394313335418701,
"logps/chosen": -442.6025390625,
"logps/rejected": -655.9295654296875,
"loss": 0.0235,
"rewards/accuracies": 1.0,
"rewards/chosen": -6.260979175567627,
"rewards/margins": 20.33395004272461,
"rewards/rejected": -26.59493064880371,
"step": 2910
},
{
"epoch": 1.88,
"learning_rate": 3.8181385384432276e-07,
"logits/chosen": 0.402191162109375,
"logits/rejected": 1.116523265838623,
"logps/chosen": -393.9117126464844,
"logps/rejected": -693.4283447265625,
"loss": 0.0116,
"rewards/accuracies": 1.0,
"rewards/chosen": -4.633070468902588,
"rewards/margins": 21.96987533569336,
"rewards/rejected": -26.602941513061523,
"step": 2920
},
{
"epoch": 1.88,
"learning_rate": 3.8121875743870506e-07,
"logits/chosen": 0.16651079058647156,
"logits/rejected": 0.6804031133651733,
"logps/chosen": -412.31512451171875,
"logps/rejected": -665.4926147460938,
"loss": 0.0146,
"rewards/accuracies": 1.0,
"rewards/chosen": -4.822356700897217,
"rewards/margins": 19.13541030883789,
"rewards/rejected": -23.957767486572266,
"step": 2930
},
{
"epoch": 1.89,
"learning_rate": 3.8062366103308735e-07,
"logits/chosen": 0.1117088571190834,
"logits/rejected": 1.083188772201538,
"logps/chosen": -369.11370849609375,
"logps/rejected": -570.9990844726562,
"loss": 0.0157,
"rewards/accuracies": 1.0,
"rewards/chosen": -3.329820156097412,
"rewards/margins": 19.094078063964844,
"rewards/rejected": -22.423898696899414,
"step": 2940
},
{
"epoch": 1.9,
"learning_rate": 3.8002856462746965e-07,
"logits/chosen": 0.11049242317676544,
"logits/rejected": 0.9105658531188965,
"logps/chosen": -433.6129455566406,
"logps/rejected": -589.2352294921875,
"loss": 0.0138,
"rewards/accuracies": 1.0,
"rewards/chosen": -3.2948660850524902,
"rewards/margins": 18.181903839111328,
"rewards/rejected": -21.47677230834961,
"step": 2950
},
{
"epoch": 1.9,
"learning_rate": 3.794334682218519e-07,
"logits/chosen": -0.10087742656469345,
"logits/rejected": 0.8825875520706177,
"logps/chosen": -401.3361511230469,
"logps/rejected": -503.46710205078125,
"loss": 0.0172,
"rewards/accuracies": 0.987500011920929,
"rewards/chosen": -3.8647804260253906,
"rewards/margins": 15.990776062011719,
"rewards/rejected": -19.85555648803711,
"step": 2960
},
{
"epoch": 1.91,
"learning_rate": 3.7883837181623424e-07,
"logits/chosen": 0.07600893825292587,
"logits/rejected": 0.9553617238998413,
"logps/chosen": -408.1060485839844,
"logps/rejected": -534.631591796875,
"loss": 0.0123,
"rewards/accuracies": 1.0,
"rewards/chosen": -3.9009666442871094,
"rewards/margins": 17.22827911376953,
"rewards/rejected": -21.12924575805664,
"step": 2970
},
{
"epoch": 1.92,
"learning_rate": 3.7824327541061653e-07,
"logits/chosen": 0.2739563286304474,
"logits/rejected": 0.8527243733406067,
"logps/chosen": -437.29638671875,
"logps/rejected": -695.6834106445312,
"loss": 0.0105,
"rewards/accuracies": 1.0,
"rewards/chosen": -5.193564414978027,
"rewards/margins": 21.4495792388916,
"rewards/rejected": -26.643142700195312,
"step": 2980
},
{
"epoch": 1.92,
"learning_rate": 3.776481790049988e-07,
"logits/chosen": 0.43024197220802307,
"logits/rejected": 1.1188406944274902,
"logps/chosen": -386.9818115234375,
"logps/rejected": -554.8717041015625,
"loss": 0.0291,
"rewards/accuracies": 0.987500011920929,
"rewards/chosen": -5.847230911254883,
"rewards/margins": 17.605852127075195,
"rewards/rejected": -23.453083038330078,
"step": 2990
},
{
"epoch": 1.93,
"learning_rate": 3.7705308259938107e-07,
"logits/chosen": 0.6990832090377808,
"logits/rejected": 1.3339582681655884,
"logps/chosen": -375.6474609375,
"logps/rejected": -568.9274291992188,
"loss": 0.0173,
"rewards/accuracies": 1.0,
"rewards/chosen": -3.8585574626922607,
"rewards/margins": 18.85175323486328,
"rewards/rejected": -22.710311889648438,
"step": 3000
},
{
"epoch": 1.93,
"eval_logits/chosen": -0.04803978279232979,
"eval_logits/rejected": 0.6017779111862183,
"eval_logps/chosen": -375.2929992675781,
"eval_logps/rejected": -498.0708312988281,
"eval_loss": 0.1309323012828827,
"eval_rewards/accuracies": 0.953125,
"eval_rewards/chosen": -4.592996120452881,
"eval_rewards/margins": 13.579015731811523,
"eval_rewards/rejected": -18.17201042175293,
"eval_runtime": 76.759,
"eval_samples_per_second": 13.028,
"eval_steps_per_second": 0.417,
"step": 3000
},
{
"epoch": 1.93,
"learning_rate": 3.764579861937634e-07,
"logits/chosen": 0.5929206609725952,
"logits/rejected": 1.196090579032898,
"logps/chosen": -398.5981750488281,
"logps/rejected": -553.6544799804688,
"loss": 0.0145,
"rewards/accuracies": 1.0,
"rewards/chosen": -4.344789028167725,
"rewards/margins": 19.267980575561523,
"rewards/rejected": -23.612768173217773,
"step": 3010
},
{
"epoch": 1.94,
"learning_rate": 3.7586288978814566e-07,
"logits/chosen": 0.46780499815940857,
"logits/rejected": 1.3755557537078857,
"logps/chosen": -408.9537658691406,
"logps/rejected": -639.6270141601562,
"loss": 0.0271,
"rewards/accuracies": 0.987500011920929,
"rewards/chosen": -6.787570953369141,
"rewards/margins": 19.320308685302734,
"rewards/rejected": -26.107879638671875,
"step": 3020
},
{
"epoch": 1.95,
"learning_rate": 3.7526779338252795e-07,
"logits/chosen": 0.47549518942832947,
"logits/rejected": 1.1248539686203003,
"logps/chosen": -458.66864013671875,
"logps/rejected": -612.8165283203125,
"loss": 0.0257,
"rewards/accuracies": 1.0,
"rewards/chosen": -5.648183345794678,
"rewards/margins": 18.065431594848633,
"rewards/rejected": -23.713613510131836,
"step": 3030
},
{
"epoch": 1.95,
"learning_rate": 3.7467269697691025e-07,
"logits/chosen": 0.022746117785573006,
"logits/rejected": 1.0212655067443848,
"logps/chosen": -437.48583984375,
"logps/rejected": -603.9195556640625,
"loss": 0.0137,
"rewards/accuracies": 1.0,
"rewards/chosen": -5.6978535652160645,
"rewards/margins": 19.632654190063477,
"rewards/rejected": -25.330509185791016,
"step": 3040
},
{
"epoch": 1.96,
"learning_rate": 3.740776005712925e-07,
"logits/chosen": -0.12851546704769135,
"logits/rejected": 0.7596645951271057,
"logps/chosen": -444.29461669921875,
"logps/rejected": -648.9785766601562,
"loss": 0.0098,
"rewards/accuracies": 1.0,
"rewards/chosen": -5.788307189941406,
"rewards/margins": 18.562524795532227,
"rewards/rejected": -24.350830078125,
"step": 3050
},
{
"epoch": 1.97,
"learning_rate": 3.7348250416567484e-07,
"logits/chosen": 0.014139672741293907,
"logits/rejected": 0.8716185688972473,
"logps/chosen": -395.7372131347656,
"logps/rejected": -608.2803955078125,
"loss": 0.0161,
"rewards/accuracies": 1.0,
"rewards/chosen": -4.413556098937988,
"rewards/margins": 19.46448516845703,
"rewards/rejected": -23.878040313720703,
"step": 3060
},
{
"epoch": 1.97,
"learning_rate": 3.7288740776005713e-07,
"logits/chosen": 0.32760128378868103,
"logits/rejected": 0.5136176943778992,
"logps/chosen": -420.23480224609375,
"logps/rejected": -669.7601318359375,
"loss": 0.017,
"rewards/accuracies": 1.0,
"rewards/chosen": -4.701635360717773,
"rewards/margins": 19.267168045043945,
"rewards/rejected": -23.96880531311035,
"step": 3070
},
{
"epoch": 1.98,
"learning_rate": 3.722923113544394e-07,
"logits/chosen": 0.4951232373714447,
"logits/rejected": 1.083812952041626,
"logps/chosen": -412.07464599609375,
"logps/rejected": -682.0787353515625,
"loss": 0.0255,
"rewards/accuracies": 1.0,
"rewards/chosen": -4.654479026794434,
"rewards/margins": 18.33094596862793,
"rewards/rejected": -22.985427856445312,
"step": 3080
},
{
"epoch": 1.99,
"learning_rate": 3.7169721494882167e-07,
"logits/chosen": 0.7839171290397644,
"logits/rejected": 1.5858978033065796,
"logps/chosen": -398.9652099609375,
"logps/rejected": -555.3340454101562,
"loss": 0.022,
"rewards/accuracies": 0.987500011920929,
"rewards/chosen": -4.493814945220947,
"rewards/margins": 17.03528594970703,
"rewards/rejected": -21.52910041809082,
"step": 3090
},
{
"epoch": 1.99,
"learning_rate": 3.71102118543204e-07,
"logits/chosen": 0.8546144366264343,
"logits/rejected": 1.6159175634384155,
"logps/chosen": -427.02978515625,
"logps/rejected": -589.3820190429688,
"loss": 0.0187,
"rewards/accuracies": 0.987500011920929,
"rewards/chosen": -6.833371639251709,
"rewards/margins": 18.236379623413086,
"rewards/rejected": -25.069753646850586,
"step": 3100
},
{
"epoch": 1.99,
"eval_logits/chosen": 0.0045381635427474976,
"eval_logits/rejected": 0.7645629048347473,
"eval_logps/chosen": -400.106201171875,
"eval_logps/rejected": -530.0531616210938,
"eval_loss": 0.13586518168449402,
"eval_rewards/accuracies": 0.953125,
"eval_rewards/chosen": -7.074316024780273,
"eval_rewards/margins": 14.295927047729492,
"eval_rewards/rejected": -21.370241165161133,
"eval_runtime": 76.6017,
"eval_samples_per_second": 13.055,
"eval_steps_per_second": 0.418,
"step": 3100
},
{
"epoch": 2.0,
"learning_rate": 3.7050702213758626e-07,
"logits/chosen": 0.6572461128234863,
"logits/rejected": 1.7969856262207031,
"logps/chosen": -398.08453369140625,
"logps/rejected": -607.6263427734375,
"loss": 0.0082,
"rewards/accuracies": 0.987500011920929,
"rewards/chosen": -6.214932918548584,
"rewards/margins": 21.25518035888672,
"rewards/rejected": -27.470117568969727,
"step": 3110
}
],
"logging_steps": 10,
"max_steps": 9336,
"num_train_epochs": 6,
"save_steps": 500,
"total_flos": 0.0,
"trial_name": null,
"trial_params": null
}