tinyllama_mole_dpo_ep3 / trainer_state.json
hushell's picture
Model save
4325427 verified
{
"best_metric": 0.6285176873207092,
"best_model_checkpoint": "data/tinyllama_mole_dpo_ep3/checkpoint-900",
"epoch": 0.9994767137624281,
"eval_steps": 100,
"global_step": 955,
"is_hyper_param_search": false,
"is_local_process_zero": true,
"is_world_process_zero": true,
"log_history": [
{
"epoch": 0.0,
"learning_rate": 5e-09,
"logits/chosen": -1.790444016456604,
"logits/rejected": -1.7375602722167969,
"logps/chosen": -288.11163330078125,
"logps/rejected": -270.18121337890625,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": 0.0,
"rewards/margins": 0.0,
"rewards/rejected": 0.0,
"step": 1
},
{
"epoch": 0.01,
"learning_rate": 5e-08,
"logits/chosen": -1.9029791355133057,
"logits/rejected": -1.6527897119522095,
"logps/chosen": -349.9918518066406,
"logps/rejected": -281.4112243652344,
"loss": 0.693,
"rewards/accuracies": 0.5069444179534912,
"rewards/chosen": 0.0003158848558086902,
"rewards/margins": 0.0004107448039576411,
"rewards/rejected": -9.485996997682378e-05,
"step": 10
},
{
"epoch": 0.02,
"learning_rate": 1e-07,
"logits/chosen": -1.773256540298462,
"logits/rejected": -1.7059326171875,
"logps/chosen": -306.29168701171875,
"logps/rejected": -274.650634765625,
"loss": 0.693,
"rewards/accuracies": 0.5,
"rewards/chosen": 0.0002991842629853636,
"rewards/margins": 0.00016614615742582828,
"rewards/rejected": 0.000133038149215281,
"step": 20
},
{
"epoch": 0.03,
"learning_rate": 1.5e-07,
"logits/chosen": -1.8527847528457642,
"logits/rejected": -1.707421898841858,
"logps/chosen": -339.32342529296875,
"logps/rejected": -297.5170593261719,
"loss": 0.693,
"rewards/accuracies": 0.512499988079071,
"rewards/chosen": -0.0001883753720903769,
"rewards/margins": -0.00028402512543834746,
"rewards/rejected": 9.56498843152076e-05,
"step": 30
},
{
"epoch": 0.04,
"learning_rate": 2e-07,
"logits/chosen": -1.869114875793457,
"logits/rejected": -1.7149085998535156,
"logps/chosen": -329.600830078125,
"logps/rejected": -274.0981140136719,
"loss": 0.693,
"rewards/accuracies": 0.4937500059604645,
"rewards/chosen": 0.0005581147270277143,
"rewards/margins": 0.000760135124437511,
"rewards/rejected": -0.00020202035375405103,
"step": 40
},
{
"epoch": 0.05,
"learning_rate": 2.5e-07,
"logits/chosen": -1.8775875568389893,
"logits/rejected": -1.7631309032440186,
"logps/chosen": -344.46002197265625,
"logps/rejected": -307.79693603515625,
"loss": 0.6928,
"rewards/accuracies": 0.5,
"rewards/chosen": 0.0002629683876875788,
"rewards/margins": 0.0003627274709288031,
"rewards/rejected": -9.97590395854786e-05,
"step": 50
},
{
"epoch": 0.06,
"learning_rate": 3e-07,
"logits/chosen": -1.8707891702651978,
"logits/rejected": -1.711287498474121,
"logps/chosen": -345.78326416015625,
"logps/rejected": -318.0264892578125,
"loss": 0.6926,
"rewards/accuracies": 0.574999988079071,
"rewards/chosen": 0.0013746457407251,
"rewards/margins": 0.0012839403934776783,
"rewards/rejected": 9.070520900422707e-05,
"step": 60
},
{
"epoch": 0.07,
"learning_rate": 3.5e-07,
"logits/chosen": -1.997075080871582,
"logits/rejected": -1.8358447551727295,
"logps/chosen": -379.5538024902344,
"logps/rejected": -332.5546875,
"loss": 0.6926,
"rewards/accuracies": 0.612500011920929,
"rewards/chosen": 0.001883229473605752,
"rewards/margins": 0.0018280971562489867,
"rewards/rejected": 5.51322809769772e-05,
"step": 70
},
{
"epoch": 0.08,
"learning_rate": 4e-07,
"logits/chosen": -1.7132848501205444,
"logits/rejected": -1.5544992685317993,
"logps/chosen": -352.10699462890625,
"logps/rejected": -292.5279846191406,
"loss": 0.6914,
"rewards/accuracies": 0.6812499761581421,
"rewards/chosen": 0.0026874816976487637,
"rewards/margins": 0.0036670640110969543,
"rewards/rejected": -0.000979582779109478,
"step": 80
},
{
"epoch": 0.09,
"learning_rate": 4.5e-07,
"logits/chosen": -1.9277913570404053,
"logits/rejected": -1.8120664358139038,
"logps/chosen": -345.2822570800781,
"logps/rejected": -309.4122314453125,
"loss": 0.6909,
"rewards/accuracies": 0.5562499761581421,
"rewards/chosen": 0.00368087668903172,
"rewards/margins": 0.004829054698348045,
"rewards/rejected": -0.0011481784749776125,
"step": 90
},
{
"epoch": 0.1,
"learning_rate": 5e-07,
"logits/chosen": -1.9077354669570923,
"logits/rejected": -1.8217405080795288,
"logps/chosen": -364.0276184082031,
"logps/rejected": -340.09906005859375,
"loss": 0.6896,
"rewards/accuracies": 0.699999988079071,
"rewards/chosen": 0.00508699519559741,
"rewards/margins": 0.007315085269510746,
"rewards/rejected": -0.002228089142590761,
"step": 100
},
{
"epoch": 0.1,
"eval_logits/chosen": -1.8918097019195557,
"eval_logits/rejected": -1.7573894262313843,
"eval_logps/chosen": -342.0017395019531,
"eval_logps/rejected": -300.80889892578125,
"eval_loss": 0.689919114112854,
"eval_rewards/accuracies": 0.6448412537574768,
"eval_rewards/chosen": 0.006363342050462961,
"eval_rewards/margins": 0.007642398122698069,
"eval_rewards/rejected": -0.0012790567707270384,
"eval_runtime": 339.8787,
"eval_samples_per_second": 5.884,
"eval_steps_per_second": 0.185,
"step": 100
},
{
"epoch": 0.12,
"learning_rate": 4.998312558730158e-07,
"logits/chosen": -1.8893086910247803,
"logits/rejected": -1.6524394750595093,
"logps/chosen": -312.1143493652344,
"logps/rejected": -249.64523315429688,
"loss": 0.6887,
"rewards/accuracies": 0.643750011920929,
"rewards/chosen": 0.005943284835666418,
"rewards/margins": 0.008504782803356647,
"rewards/rejected": -0.002561498200520873,
"step": 110
},
{
"epoch": 0.13,
"learning_rate": 4.993252512887069e-07,
"logits/chosen": -1.8319896459579468,
"logits/rejected": -1.6496975421905518,
"logps/chosen": -358.3703308105469,
"logps/rejected": -289.1592102050781,
"loss": 0.6873,
"rewards/accuracies": 0.6625000238418579,
"rewards/chosen": 0.009710123762488365,
"rewards/margins": 0.017875777557492256,
"rewards/rejected": -0.008165654726326466,
"step": 120
},
{
"epoch": 0.14,
"learning_rate": 4.984826693294873e-07,
"logits/chosen": -1.9560505151748657,
"logits/rejected": -1.7518360614776611,
"logps/chosen": -377.28082275390625,
"logps/rejected": -308.9075927734375,
"loss": 0.6866,
"rewards/accuracies": 0.612500011920929,
"rewards/chosen": 0.009400544688105583,
"rewards/margins": 0.014766154810786247,
"rewards/rejected": -0.005365610122680664,
"step": 130
},
{
"epoch": 0.15,
"learning_rate": 4.973046474414144e-07,
"logits/chosen": -1.7957839965820312,
"logits/rejected": -1.6959806680679321,
"logps/chosen": -283.5320129394531,
"logps/rejected": -259.69097900390625,
"loss": 0.6851,
"rewards/accuracies": 0.6312500238418579,
"rewards/chosen": 0.00773229356855154,
"rewards/margins": 0.01577392965555191,
"rewards/rejected": -0.00804163608700037,
"step": 140
},
{
"epoch": 0.16,
"learning_rate": 4.957927758986888e-07,
"logits/chosen": -1.8573449850082397,
"logits/rejected": -1.7665218114852905,
"logps/chosen": -333.4697265625,
"logps/rejected": -334.0047912597656,
"loss": 0.6864,
"rewards/accuracies": 0.625,
"rewards/chosen": -0.00011548856127774343,
"rewards/margins": 0.005317127797752619,
"rewards/rejected": -0.005432615987956524,
"step": 150
},
{
"epoch": 0.17,
"learning_rate": 4.939490956568589e-07,
"logits/chosen": -1.9507248401641846,
"logits/rejected": -1.7802870273590088,
"logps/chosen": -352.24407958984375,
"logps/rejected": -330.3514099121094,
"loss": 0.6815,
"rewards/accuracies": 0.668749988079071,
"rewards/chosen": 0.0028652914334088564,
"rewards/margins": 0.018842989578843117,
"rewards/rejected": -0.01597769930958748,
"step": 160
},
{
"epoch": 0.18,
"learning_rate": 4.917760955976277e-07,
"logits/chosen": -1.9207957983016968,
"logits/rejected": -1.7402126789093018,
"logps/chosen": -322.8128662109375,
"logps/rejected": -280.6071472167969,
"loss": 0.6807,
"rewards/accuracies": 0.65625,
"rewards/chosen": 0.006607011891901493,
"rewards/margins": 0.029789209365844727,
"rewards/rejected": -0.023182198405265808,
"step": 170
},
{
"epoch": 0.19,
"learning_rate": 4.892767091689785e-07,
"logits/chosen": -1.8039575815200806,
"logits/rejected": -1.6338218450546265,
"logps/chosen": -298.98541259765625,
"logps/rejected": -268.31658935546875,
"loss": 0.6781,
"rewards/accuracies": 0.668749988079071,
"rewards/chosen": -0.010788476094603539,
"rewards/margins": 0.033482082188129425,
"rewards/rejected": -0.044270556420087814,
"step": 180
},
{
"epoch": 0.2,
"learning_rate": 4.864543104251586e-07,
"logits/chosen": -1.9226608276367188,
"logits/rejected": -1.7317349910736084,
"logps/chosen": -342.0778503417969,
"logps/rejected": -300.3584899902344,
"loss": 0.6758,
"rewards/accuracies": 0.675000011920929,
"rewards/chosen": -0.010154007002711296,
"rewards/margins": 0.03691417723894119,
"rewards/rejected": -0.04706818610429764,
"step": 190
},
{
"epoch": 0.21,
"learning_rate": 4.833127094718643e-07,
"logits/chosen": -1.7831227779388428,
"logits/rejected": -1.6590824127197266,
"logps/chosen": -343.6825866699219,
"logps/rejected": -319.09796142578125,
"loss": 0.6762,
"rewards/accuracies": 0.6812499761581421,
"rewards/chosen": -0.012453018687665462,
"rewards/margins": 0.04117157310247421,
"rewards/rejected": -0.05362458899617195,
"step": 200
},
{
"epoch": 0.21,
"eval_logits/chosen": -1.883855938911438,
"eval_logits/rejected": -1.7501325607299805,
"eval_logps/chosen": -345.56878662109375,
"eval_logps/rejected": -307.84234619140625,
"eval_loss": 0.6756463646888733,
"eval_rewards/accuracies": 0.6626983880996704,
"eval_rewards/chosen": -0.029307426884770393,
"eval_rewards/margins": 0.04230639338493347,
"eval_rewards/rejected": -0.07161381840705872,
"eval_runtime": 342.3556,
"eval_samples_per_second": 5.842,
"eval_steps_per_second": 0.184,
"step": 200
},
{
"epoch": 0.22,
"learning_rate": 4.79856147322777e-07,
"logits/chosen": -1.9185651540756226,
"logits/rejected": -1.6988731622695923,
"logps/chosen": -368.73504638671875,
"logps/rejected": -302.3857421875,
"loss": 0.6743,
"rewards/accuracies": 0.643750011920929,
"rewards/chosen": -0.036936286836862564,
"rewards/margins": 0.04566096514463425,
"rewards/rejected": -0.08259725570678711,
"step": 210
},
{
"epoch": 0.23,
"learning_rate": 4.760892901743944e-07,
"logits/chosen": -1.9230194091796875,
"logits/rejected": -1.7622359991073608,
"logps/chosen": -352.57354736328125,
"logps/rejected": -324.182373046875,
"loss": 0.6748,
"rewards/accuracies": 0.65625,
"rewards/chosen": -0.03244791179895401,
"rewards/margins": 0.04869867116212845,
"rewards/rejected": -0.08114659041166306,
"step": 220
},
{
"epoch": 0.24,
"learning_rate": 4.720172231068844e-07,
"logits/chosen": -1.8404433727264404,
"logits/rejected": -1.6695423126220703,
"logps/chosen": -354.01727294921875,
"logps/rejected": -295.2720642089844,
"loss": 0.665,
"rewards/accuracies": 0.668749988079071,
"rewards/chosen": -0.050288014113903046,
"rewards/margins": 0.053208403289318085,
"rewards/rejected": -0.10349641740322113,
"step": 230
},
{
"epoch": 0.25,
"learning_rate": 4.6764544321946557e-07,
"logits/chosen": -1.851320505142212,
"logits/rejected": -1.6485137939453125,
"logps/chosen": -338.0115051269531,
"logps/rejected": -272.17596435546875,
"loss": 0.6675,
"rewards/accuracies": 0.6312500238418579,
"rewards/chosen": -0.04714337736368179,
"rewards/margins": 0.05400116369128227,
"rewards/rejected": -0.10114455223083496,
"step": 240
},
{
"epoch": 0.26,
"learning_rate": 4.6297985220958176e-07,
"logits/chosen": -1.858806848526001,
"logits/rejected": -1.7875276803970337,
"logps/chosen": -323.6578369140625,
"logps/rejected": -327.8287048339844,
"loss": 0.6704,
"rewards/accuracies": 0.668749988079071,
"rewards/chosen": -0.06438130885362625,
"rewards/margins": 0.054077375680208206,
"rewards/rejected": -0.11845867335796356,
"step": 250
},
{
"epoch": 0.27,
"learning_rate": 4.580267484058875e-07,
"logits/chosen": -1.787245512008667,
"logits/rejected": -1.6371400356292725,
"logps/chosen": -346.79132080078125,
"logps/rejected": -315.600341796875,
"loss": 0.6655,
"rewards/accuracies": 0.668749988079071,
"rewards/chosen": -0.07440805435180664,
"rewards/margins": 0.07648531347513199,
"rewards/rejected": -0.15089336037635803,
"step": 260
},
{
"epoch": 0.28,
"learning_rate": 4.527928182658005e-07,
"logits/chosen": -1.9231889247894287,
"logits/rejected": -1.7725601196289062,
"logps/chosen": -357.95367431640625,
"logps/rejected": -305.06793212890625,
"loss": 0.6624,
"rewards/accuracies": 0.7124999761581421,
"rewards/chosen": -0.0633041113615036,
"rewards/margins": 0.09316254407167435,
"rewards/rejected": -0.15646666288375854,
"step": 270
},
{
"epoch": 0.29,
"learning_rate": 4.472851273490984e-07,
"logits/chosen": -1.9144046306610107,
"logits/rejected": -1.8413257598876953,
"logps/chosen": -356.4827575683594,
"logps/rejected": -339.57952880859375,
"loss": 0.6647,
"rewards/accuracies": 0.706250011920929,
"rewards/chosen": -0.07623454183340073,
"rewards/margins": 0.06355991959571838,
"rewards/rejected": -0.1397944837808609,
"step": 280
},
{
"epoch": 0.3,
"learning_rate": 4.415111107797445e-07,
"logits/chosen": -1.7934792041778564,
"logits/rejected": -1.7078644037246704,
"logps/chosen": -343.297607421875,
"logps/rejected": -307.45904541015625,
"loss": 0.6574,
"rewards/accuracies": 0.6000000238418579,
"rewards/chosen": -0.09327341616153717,
"rewards/margins": 0.06554602831602097,
"rewards/rejected": -0.15881945192813873,
"step": 290
},
{
"epoch": 0.31,
"learning_rate": 4.3547856320882036e-07,
"logits/chosen": -1.8048231601715088,
"logits/rejected": -1.560224175453186,
"logps/chosen": -344.6669921875,
"logps/rejected": -290.4841613769531,
"loss": 0.6499,
"rewards/accuracies": 0.65625,
"rewards/chosen": -0.07834906131029129,
"rewards/margins": 0.09436166286468506,
"rewards/rejected": -0.17271070182323456,
"step": 300
},
{
"epoch": 0.31,
"eval_logits/chosen": -1.868820071220398,
"eval_logits/rejected": -1.7357724905014038,
"eval_logps/chosen": -351.38946533203125,
"eval_logps/rejected": -318.8117980957031,
"eval_loss": 0.6586803793907166,
"eval_rewards/accuracies": 0.6686508059501648,
"eval_rewards/chosen": -0.08751402050256729,
"eval_rewards/margins": 0.09379409998655319,
"eval_rewards/rejected": -0.18130813539028168,
"eval_runtime": 336.7839,
"eval_samples_per_second": 5.939,
"eval_steps_per_second": 0.187,
"step": 300
},
{
"epoch": 0.32,
"learning_rate": 4.291956282921128e-07,
"logits/chosen": -1.7621396780014038,
"logits/rejected": -1.651000738143921,
"logps/chosen": -348.511474609375,
"logps/rejected": -330.1929931640625,
"loss": 0.6514,
"rewards/accuracies": 0.6937500238418579,
"rewards/chosen": -0.10528180748224258,
"rewards/margins": 0.11313805729150772,
"rewards/rejected": -0.2184198647737503,
"step": 310
},
{
"epoch": 0.33,
"learning_rate": 4.2267078769656105e-07,
"logits/chosen": -1.8093763589859009,
"logits/rejected": -1.733646035194397,
"logps/chosen": -339.96807861328125,
"logps/rejected": -329.6910705566406,
"loss": 0.6485,
"rewards/accuracies": 0.606249988079071,
"rewards/chosen": -0.10823850333690643,
"rewards/margins": 0.10142701864242554,
"rewards/rejected": -0.20966553688049316,
"step": 320
},
{
"epoch": 0.35,
"learning_rate": 4.159128496504053e-07,
"logits/chosen": -1.8124803304672241,
"logits/rejected": -1.656519889831543,
"logps/chosen": -338.48504638671875,
"logps/rejected": -287.4867248535156,
"loss": 0.6577,
"rewards/accuracies": 0.643750011920929,
"rewards/chosen": -0.09331468492746353,
"rewards/margins": 0.10502304136753082,
"rewards/rejected": -0.19833770394325256,
"step": 330
},
{
"epoch": 0.36,
"learning_rate": 4.0893093705249207e-07,
"logits/chosen": -1.7872120141983032,
"logits/rejected": -1.6502597332000732,
"logps/chosen": -302.8948669433594,
"logps/rejected": -278.6997985839844,
"loss": 0.6597,
"rewards/accuracies": 0.5562499761581421,
"rewards/chosen": -0.1376679241657257,
"rewards/margins": 0.06090687960386276,
"rewards/rejected": -0.19857481122016907,
"step": 340
},
{
"epoch": 0.37,
"learning_rate": 4.0173447515678915e-07,
"logits/chosen": -1.745719313621521,
"logits/rejected": -1.7379214763641357,
"logps/chosen": -341.92218017578125,
"logps/rejected": -346.43121337890625,
"loss": 0.6531,
"rewards/accuracies": 0.6499999761581421,
"rewards/chosen": -0.1528899073600769,
"rewards/margins": 0.10737234354019165,
"rewards/rejected": -0.26026225090026855,
"step": 350
},
{
"epoch": 0.38,
"learning_rate": 3.9433317884873665e-07,
"logits/chosen": -1.896888017654419,
"logits/rejected": -1.7774041891098022,
"logps/chosen": -345.6350402832031,
"logps/rejected": -304.7301025390625,
"loss": 0.6427,
"rewards/accuracies": 0.6625000238418579,
"rewards/chosen": -0.18082933127880096,
"rewards/margins": 0.10347436368465424,
"rewards/rejected": -0.2843037247657776,
"step": 360
},
{
"epoch": 0.39,
"learning_rate": 3.867370395306068e-07,
"logits/chosen": -1.7397247552871704,
"logits/rejected": -1.6811482906341553,
"logps/chosen": -297.6015625,
"logps/rejected": -314.14093017578125,
"loss": 0.6415,
"rewards/accuracies": 0.675000011920929,
"rewards/chosen": -0.17070330679416656,
"rewards/margins": 0.12213484942913055,
"rewards/rejected": -0.2928381562232971,
"step": 370
},
{
"epoch": 0.4,
"learning_rate": 3.78956311633581e-07,
"logits/chosen": -1.9287761449813843,
"logits/rejected": -1.7968651056289673,
"logps/chosen": -369.02142333984375,
"logps/rejected": -335.91986083984375,
"loss": 0.6426,
"rewards/accuracies": 0.668749988079071,
"rewards/chosen": -0.1528002917766571,
"rewards/margins": 0.15230461955070496,
"rewards/rejected": -0.30510494112968445,
"step": 380
},
{
"epoch": 0.41,
"learning_rate": 3.7100149877474976e-07,
"logits/chosen": -1.7732282876968384,
"logits/rejected": -1.6088358163833618,
"logps/chosen": -340.55804443359375,
"logps/rejected": -296.6915283203125,
"loss": 0.6505,
"rewards/accuracies": 0.6937500238418579,
"rewards/chosen": -0.1547761708498001,
"rewards/margins": 0.11781679093837738,
"rewards/rejected": -0.2725929617881775,
"step": 390
},
{
"epoch": 0.42,
"learning_rate": 3.6288333957772234e-07,
"logits/chosen": -1.9192142486572266,
"logits/rejected": -1.7736551761627197,
"logps/chosen": -382.8447265625,
"logps/rejected": -338.8296813964844,
"loss": 0.6374,
"rewards/accuracies": 0.668749988079071,
"rewards/chosen": -0.18712495267391205,
"rewards/margins": 0.15186870098114014,
"rewards/rejected": -0.338993638753891,
"step": 400
},
{
"epoch": 0.42,
"eval_logits/chosen": -1.848189353942871,
"eval_logits/rejected": -1.7164283990859985,
"eval_logps/chosen": -359.8952941894531,
"eval_logps/rejected": -332.86322021484375,
"eval_loss": 0.6451287865638733,
"eval_rewards/accuracies": 0.6746031641960144,
"eval_rewards/chosen": -0.1725723147392273,
"eval_rewards/margins": 0.14925029873847961,
"eval_rewards/rejected": -0.3218226134777069,
"eval_runtime": 336.4318,
"eval_samples_per_second": 5.945,
"eval_steps_per_second": 0.187,
"step": 400
},
{
"epoch": 0.43,
"learning_rate": 3.5461279317599025e-07,
"logits/chosen": -1.9535846710205078,
"logits/rejected": -1.8399330377578735,
"logps/chosen": -380.81396484375,
"logps/rejected": -362.853759765625,
"loss": 0.6402,
"rewards/accuracies": 0.6625000238418579,
"rewards/chosen": -0.1376427710056305,
"rewards/margins": 0.15401354432106018,
"rewards/rejected": -0.2916563153266907,
"step": 410
},
{
"epoch": 0.44,
"learning_rate": 3.4620102441861144e-07,
"logits/chosen": -1.7898231744766235,
"logits/rejected": -1.6419872045516968,
"logps/chosen": -319.6932678222656,
"logps/rejected": -280.15960693359375,
"loss": 0.6387,
"rewards/accuracies": 0.643750011920929,
"rewards/chosen": -0.14749577641487122,
"rewards/margins": 0.12767234444618225,
"rewards/rejected": -0.2751680910587311,
"step": 420
},
{
"epoch": 0.45,
"learning_rate": 3.376593887981886e-07,
"logits/chosen": -1.7821381092071533,
"logits/rejected": -1.6903321743011475,
"logps/chosen": -329.77142333984375,
"logps/rejected": -305.03271484375,
"loss": 0.6447,
"rewards/accuracies": 0.59375,
"rewards/chosen": -0.20932936668395996,
"rewards/margins": 0.10754366964101791,
"rewards/rejected": -0.3168730139732361,
"step": 430
},
{
"epoch": 0.46,
"learning_rate": 3.2899941712148813e-07,
"logits/chosen": -1.7692524194717407,
"logits/rejected": -1.703125,
"logps/chosen": -330.9040222167969,
"logps/rejected": -317.64447021484375,
"loss": 0.6428,
"rewards/accuracies": 0.5874999761581421,
"rewards/chosen": -0.2360539734363556,
"rewards/margins": 0.1059914231300354,
"rewards/rejected": -0.342045396566391,
"step": 440
},
{
"epoch": 0.47,
"learning_rate": 3.2023279994339236e-07,
"logits/chosen": -1.6386339664459229,
"logits/rejected": -1.5296618938446045,
"logps/chosen": -331.60919189453125,
"logps/rejected": -312.84686279296875,
"loss": 0.6366,
"rewards/accuracies": 0.6812499761581421,
"rewards/chosen": -0.18304389715194702,
"rewards/margins": 0.18606719374656677,
"rewards/rejected": -0.3691111207008362,
"step": 450
},
{
"epoch": 0.48,
"learning_rate": 3.1137137178519977e-07,
"logits/chosen": -1.8083422183990479,
"logits/rejected": -1.666670799255371,
"logps/chosen": -337.09991455078125,
"logps/rejected": -324.5543518066406,
"loss": 0.6313,
"rewards/accuracies": 0.581250011920929,
"rewards/chosen": -0.19549410045146942,
"rewards/margins": 0.13338689506053925,
"rewards/rejected": -0.32888099551200867,
"step": 460
},
{
"epoch": 0.49,
"learning_rate": 3.024270951585776e-07,
"logits/chosen": -1.8487087488174438,
"logits/rejected": -1.6575820446014404,
"logps/chosen": -383.4247131347656,
"logps/rejected": -340.35845947265625,
"loss": 0.6264,
"rewards/accuracies": 0.7437499761581421,
"rewards/chosen": -0.2236436903476715,
"rewards/margins": 0.2502152621746063,
"rewards/rejected": -0.4738590121269226,
"step": 470
},
{
"epoch": 0.5,
"learning_rate": 2.934120444167326e-07,
"logits/chosen": -1.6958894729614258,
"logits/rejected": -1.634203553199768,
"logps/chosen": -303.4564208984375,
"logps/rejected": -303.09417724609375,
"loss": 0.6552,
"rewards/accuracies": 0.612500011920929,
"rewards/chosen": -0.2998581528663635,
"rewards/margins": 0.08801662921905518,
"rewards/rejected": -0.3878747820854187,
"step": 480
},
{
"epoch": 0.51,
"learning_rate": 2.8433838945460205e-07,
"logits/chosen": -1.7905118465423584,
"logits/rejected": -1.6190725564956665,
"logps/chosen": -353.28582763671875,
"logps/rejected": -327.52947998046875,
"loss": 0.6305,
"rewards/accuracies": 0.643750011920929,
"rewards/chosen": -0.24126093089580536,
"rewards/margins": 0.1726113259792328,
"rewards/rejected": -0.41387230157852173,
"step": 490
},
{
"epoch": 0.52,
"learning_rate": 2.752183792800671e-07,
"logits/chosen": -1.780226469039917,
"logits/rejected": -1.6524947881698608,
"logps/chosen": -338.72564697265625,
"logps/rejected": -325.1302185058594,
"loss": 0.6348,
"rewards/accuracies": 0.65625,
"rewards/chosen": -0.2332509458065033,
"rewards/margins": 0.17690421640872955,
"rewards/rejected": -0.41015520691871643,
"step": 500
},
{
"epoch": 0.52,
"eval_logits/chosen": -1.8207694292068481,
"eval_logits/rejected": -1.6884433031082153,
"eval_logps/chosen": -369.601318359375,
"eval_logps/rejected": -346.1808166503906,
"eval_loss": 0.6377259492874146,
"eval_rewards/accuracies": 0.6646825671195984,
"eval_rewards/chosen": -0.26963263750076294,
"eval_rewards/margins": 0.18536561727523804,
"eval_rewards/rejected": -0.454998254776001,
"eval_runtime": 336.8999,
"eval_samples_per_second": 5.936,
"eval_steps_per_second": 0.187,
"step": 500
},
{
"epoch": 0.53,
"learning_rate": 2.6606432547836753e-07,
"logits/chosen": -1.7960855960845947,
"logits/rejected": -1.5604727268218994,
"logps/chosen": -385.92523193359375,
"logps/rejected": -332.73236083984375,
"loss": 0.6292,
"rewards/accuracies": 0.643750011920929,
"rewards/chosen": -0.2777213156223297,
"rewards/margins": 0.21707573533058167,
"rewards/rejected": -0.4947970509529114,
"step": 510
},
{
"epoch": 0.54,
"learning_rate": 2.5688858559204053e-07,
"logits/chosen": -1.7386023998260498,
"logits/rejected": -1.650543212890625,
"logps/chosen": -324.80194091796875,
"logps/rejected": -326.7395935058594,
"loss": 0.6328,
"rewards/accuracies": 0.7124999761581421,
"rewards/chosen": -0.22357122600078583,
"rewards/margins": 0.23022878170013428,
"rewards/rejected": -0.4538000226020813,
"step": 520
},
{
"epoch": 0.55,
"learning_rate": 2.477035464388184e-07,
"logits/chosen": -1.6688206195831299,
"logits/rejected": -1.5695548057556152,
"logps/chosen": -361.03753662109375,
"logps/rejected": -339.63226318359375,
"loss": 0.6309,
"rewards/accuracies": 0.625,
"rewards/chosen": -0.25370585918426514,
"rewards/margins": 0.17806772887706757,
"rewards/rejected": -0.4317736029624939,
"step": 530
},
{
"epoch": 0.57,
"learning_rate": 2.3852160739000706e-07,
"logits/chosen": -1.827457070350647,
"logits/rejected": -1.7111104726791382,
"logps/chosen": -378.85089111328125,
"logps/rejected": -386.2629699707031,
"loss": 0.6217,
"rewards/accuracies": 0.65625,
"rewards/chosen": -0.24295127391815186,
"rewards/margins": 0.199497789144516,
"rewards/rejected": -0.44244909286499023,
"step": 540
},
{
"epoch": 0.58,
"learning_rate": 2.2935516363191693e-07,
"logits/chosen": -1.7694709300994873,
"logits/rejected": -1.6132080554962158,
"logps/chosen": -350.4745788574219,
"logps/rejected": -334.8966369628906,
"loss": 0.6415,
"rewards/accuracies": 0.6875,
"rewards/chosen": -0.30643701553344727,
"rewards/margins": 0.2043788731098175,
"rewards/rejected": -0.5108158588409424,
"step": 550
},
{
"epoch": 0.59,
"learning_rate": 2.2021658943294407e-07,
"logits/chosen": -1.7891242504119873,
"logits/rejected": -1.6618837118148804,
"logps/chosen": -348.5799865722656,
"logps/rejected": -320.85675048828125,
"loss": 0.6252,
"rewards/accuracies": 0.675000011920929,
"rewards/chosen": -0.30380532145500183,
"rewards/margins": 0.2296813428401947,
"rewards/rejected": -0.5334866642951965,
"step": 560
},
{
"epoch": 0.6,
"learning_rate": 2.1111822143888928e-07,
"logits/chosen": -1.8157535791397095,
"logits/rejected": -1.6647037267684937,
"logps/chosen": -365.7635192871094,
"logps/rejected": -348.455810546875,
"loss": 0.6396,
"rewards/accuracies": 0.675000011920929,
"rewards/chosen": -0.3315108120441437,
"rewards/margins": 0.16533556580543518,
"rewards/rejected": -0.49684637784957886,
"step": 570
},
{
"epoch": 0.61,
"learning_rate": 2.0207234201906545e-07,
"logits/chosen": -1.6991872787475586,
"logits/rejected": -1.5607801675796509,
"logps/chosen": -355.08758544921875,
"logps/rejected": -318.2015075683594,
"loss": 0.6263,
"rewards/accuracies": 0.6499999761581421,
"rewards/chosen": -0.32228100299835205,
"rewards/margins": 0.16686378419399261,
"rewards/rejected": -0.48914486169815063,
"step": 580
},
{
"epoch": 0.62,
"learning_rate": 1.9309116268567671e-07,
"logits/chosen": -1.8247711658477783,
"logits/rejected": -1.7237234115600586,
"logps/chosen": -346.73529052734375,
"logps/rejected": -332.7265625,
"loss": 0.631,
"rewards/accuracies": 0.6000000238418579,
"rewards/chosen": -0.2523882985115051,
"rewards/margins": 0.11806211620569229,
"rewards/rejected": -0.37045034766197205,
"step": 590
},
{
"epoch": 0.63,
"learning_rate": 1.8418680760885024e-07,
"logits/chosen": -1.803122878074646,
"logits/rejected": -1.6771843433380127,
"logps/chosen": -412.44671630859375,
"logps/rejected": -402.94281005859375,
"loss": 0.6308,
"rewards/accuracies": 0.6937500238418579,
"rewards/chosen": -0.22780685126781464,
"rewards/margins": 0.22766557335853577,
"rewards/rejected": -0.455472469329834,
"step": 600
},
{
"epoch": 0.63,
"eval_logits/chosen": -1.8269299268722534,
"eval_logits/rejected": -1.6965351104736328,
"eval_logps/chosen": -370.46728515625,
"eval_logps/rejected": -348.8290710449219,
"eval_loss": 0.6333169937133789,
"eval_rewards/accuracies": 0.6726190447807312,
"eval_rewards/chosen": -0.27829232811927795,
"eval_rewards/margins": 0.2031887322664261,
"eval_rewards/rejected": -0.48148107528686523,
"eval_runtime": 333.0266,
"eval_samples_per_second": 6.006,
"eval_steps_per_second": 0.189,
"step": 600
},
{
"epoch": 0.64,
"learning_rate": 1.753712972495764e-07,
"logits/chosen": -1.858438491821289,
"logits/rejected": -1.6571956872940063,
"logps/chosen": -356.0987854003906,
"logps/rejected": -304.0067138671875,
"loss": 0.6257,
"rewards/accuracies": 0.699999988079071,
"rewards/chosen": -0.263310968875885,
"rewards/margins": 0.19186006486415863,
"rewards/rejected": -0.45517101883888245,
"step": 610
},
{
"epoch": 0.65,
"learning_rate": 1.666565321326512e-07,
"logits/chosen": -1.8659627437591553,
"logits/rejected": -1.709458589553833,
"logps/chosen": -364.3775634765625,
"logps/rejected": -322.25396728515625,
"loss": 0.6239,
"rewards/accuracies": 0.675000011920929,
"rewards/chosen": -0.3321610391139984,
"rewards/margins": 0.1703757643699646,
"rewards/rejected": -0.5025367736816406,
"step": 620
},
{
"epoch": 0.66,
"learning_rate": 1.5805427678152674e-07,
"logits/chosen": -1.641649603843689,
"logits/rejected": -1.5464293956756592,
"logps/chosen": -341.80950927734375,
"logps/rejected": -339.9558410644531,
"loss": 0.6251,
"rewards/accuracies": 0.6499999761581421,
"rewards/chosen": -0.2863956093788147,
"rewards/margins": 0.20320038497447968,
"rewards/rejected": -0.4895959794521332,
"step": 630
},
{
"epoch": 0.67,
"learning_rate": 1.4957614383675767e-07,
"logits/chosen": -1.7454828023910522,
"logits/rejected": -1.5985407829284668,
"logps/chosen": -347.81488037109375,
"logps/rejected": -337.49676513671875,
"loss": 0.6222,
"rewards/accuracies": 0.7124999761581421,
"rewards/chosen": -0.264575332403183,
"rewards/margins": 0.2227667272090912,
"rewards/rejected": -0.48734205961227417,
"step": 640
},
{
"epoch": 0.68,
"learning_rate": 1.4123357837948176e-07,
"logits/chosen": -1.9199473857879639,
"logits/rejected": -1.715175986289978,
"logps/chosen": -401.645263671875,
"logps/rejected": -365.5584411621094,
"loss": 0.6101,
"rewards/accuracies": 0.65625,
"rewards/chosen": -0.28458407521247864,
"rewards/margins": 0.24697256088256836,
"rewards/rejected": -0.5315566062927246,
"step": 650
},
{
"epoch": 0.69,
"learning_rate": 1.3303784248109808e-07,
"logits/chosen": -1.8878719806671143,
"logits/rejected": -1.8138688802719116,
"logps/chosen": -379.57672119140625,
"logps/rejected": -363.7694091796875,
"loss": 0.6202,
"rewards/accuracies": 0.6812499761581421,
"rewards/chosen": -0.2839195132255554,
"rewards/margins": 0.23723697662353516,
"rewards/rejected": -0.5211564898490906,
"step": 660
},
{
"epoch": 0.7,
"learning_rate": 1.2500000000000005e-07,
"logits/chosen": -1.7933881282806396,
"logits/rejected": -1.708486557006836,
"logps/chosen": -361.3472595214844,
"logps/rejected": -356.447509765625,
"loss": 0.6204,
"rewards/accuracies": 0.637499988079071,
"rewards/chosen": -0.2589954733848572,
"rewards/margins": 0.21957698464393616,
"rewards/rejected": -0.47857245802879333,
"step": 670
},
{
"epoch": 0.71,
"learning_rate": 1.1713090164588606e-07,
"logits/chosen": -1.675574541091919,
"logits/rejected": -1.6282098293304443,
"logps/chosen": -336.2772521972656,
"logps/rejected": -314.0538330078125,
"loss": 0.6279,
"rewards/accuracies": 0.5874999761581421,
"rewards/chosen": -0.24504590034484863,
"rewards/margins": 0.09308433532714844,
"rewards/rejected": -0.3381302058696747,
"step": 680
},
{
"epoch": 0.72,
"learning_rate": 1.094411703318115e-07,
"logits/chosen": -1.7943480014801025,
"logits/rejected": -1.650317907333374,
"logps/chosen": -339.38836669921875,
"logps/rejected": -322.80755615234375,
"loss": 0.6253,
"rewards/accuracies": 0.675000011920929,
"rewards/chosen": -0.22786399722099304,
"rewards/margins": 0.1742563247680664,
"rewards/rejected": -0.40212029218673706,
"step": 690
},
{
"epoch": 0.73,
"learning_rate": 1.0194118683375502e-07,
"logits/chosen": -1.752105951309204,
"logits/rejected": -1.5573476552963257,
"logps/chosen": -324.02362060546875,
"logps/rejected": -319.4602355957031,
"loss": 0.62,
"rewards/accuracies": 0.643750011920929,
"rewards/chosen": -0.25912636518478394,
"rewards/margins": 0.1973043978214264,
"rewards/rejected": -0.4564308226108551,
"step": 700
},
{
"epoch": 0.73,
"eval_logits/chosen": -1.8149123191833496,
"eval_logits/rejected": -1.6841386556625366,
"eval_logps/chosen": -365.8656005859375,
"eval_logps/rejected": -345.7306213378906,
"eval_loss": 0.63123619556427,
"eval_rewards/accuracies": 0.6805555820465088,
"eval_rewards/chosen": -0.2322753369808197,
"eval_rewards/margins": 0.21822109818458557,
"eval_rewards/rejected": -0.45049646496772766,
"eval_runtime": 334.8941,
"eval_samples_per_second": 5.972,
"eval_steps_per_second": 0.188,
"step": 700
},
{
"epoch": 0.74,
"learning_rate": 9.464107577705886e-08,
"logits/chosen": -1.8157602548599243,
"logits/rejected": -1.607846975326538,
"logps/chosen": -384.46343994140625,
"logps/rejected": -319.81805419921875,
"loss": 0.6142,
"rewards/accuracies": 0.6812499761581421,
"rewards/chosen": -0.23041348159313202,
"rewards/margins": 0.2244310826063156,
"rewards/rejected": -0.45484456419944763,
"step": 710
},
{
"epoch": 0.75,
"learning_rate": 8.755069196866013e-08,
"logits/chosen": -1.7827469110488892,
"logits/rejected": -1.6819953918457031,
"logps/chosen": -368.9613952636719,
"logps/rejected": -345.7911071777344,
"loss": 0.6334,
"rewards/accuracies": 0.6499999761581421,
"rewards/chosen": -0.24999408423900604,
"rewards/margins": 0.18186303973197937,
"rewards/rejected": -0.431857168674469,
"step": 720
},
{
"epoch": 0.76,
"learning_rate": 8.067960709356478e-08,
"logits/chosen": -1.8376483917236328,
"logits/rejected": -1.677075982093811,
"logps/chosen": -381.3594665527344,
"logps/rejected": -341.1945495605469,
"loss": 0.6275,
"rewards/accuracies": 0.6812499761581421,
"rewards/chosen": -0.25991183519363403,
"rewards/margins": 0.2162000834941864,
"rewards/rejected": -0.47611188888549805,
"step": 730
},
{
"epoch": 0.77,
"learning_rate": 7.403709679352216e-08,
"logits/chosen": -1.7825815677642822,
"logits/rejected": -1.5635161399841309,
"logps/chosen": -405.16839599609375,
"logps/rejected": -342.57476806640625,
"loss": 0.6172,
"rewards/accuracies": 0.731249988079071,
"rewards/chosen": -0.2256811410188675,
"rewards/margins": 0.23789973556995392,
"rewards/rejected": -0.4635809063911438,
"step": 740
},
{
"epoch": 0.78,
"learning_rate": 6.763212814534483e-08,
"logits/chosen": -1.8319247961044312,
"logits/rejected": -1.7559360265731812,
"logps/chosen": -367.5751647949219,
"logps/rejected": -367.4506530761719,
"loss": 0.6208,
"rewards/accuracies": 0.65625,
"rewards/chosen": -0.29129502177238464,
"rewards/margins": 0.18480955064296722,
"rewards/rejected": -0.4761045575141907,
"step": 750
},
{
"epoch": 0.8,
"learning_rate": 6.147334755577596e-08,
"logits/chosen": -1.729148268699646,
"logits/rejected": -1.6354280710220337,
"logps/chosen": -353.6579895019531,
"logps/rejected": -341.3516845703125,
"loss": 0.6157,
"rewards/accuracies": 0.65625,
"rewards/chosen": -0.3305814862251282,
"rewards/margins": 0.20686399936676025,
"rewards/rejected": -0.5374454259872437,
"step": 760
},
{
"epoch": 0.81,
"learning_rate": 5.556906908924655e-08,
"logits/chosen": -1.774147629737854,
"logits/rejected": -1.7218784093856812,
"logps/chosen": -382.23968505859375,
"logps/rejected": -393.50579833984375,
"loss": 0.621,
"rewards/accuracies": 0.6499999761581421,
"rewards/chosen": -0.30208078026771545,
"rewards/margins": 0.19531475007534027,
"rewards/rejected": -0.49739551544189453,
"step": 770
},
{
"epoch": 0.82,
"learning_rate": 4.992726324427901e-08,
"logits/chosen": -1.6901906728744507,
"logits/rejected": -1.58046555519104,
"logps/chosen": -335.16583251953125,
"logps/rejected": -316.52978515625,
"loss": 0.6264,
"rewards/accuracies": 0.581250011920929,
"rewards/chosen": -0.2825678884983063,
"rewards/margins": 0.153324693441391,
"rewards/rejected": -0.4358925223350525,
"step": 780
},
{
"epoch": 0.83,
"learning_rate": 4.4555546193688734e-08,
"logits/chosen": -1.795194387435913,
"logits/rejected": -1.722459077835083,
"logps/chosen": -390.23846435546875,
"logps/rejected": -379.72332763671875,
"loss": 0.635,
"rewards/accuracies": 0.65625,
"rewards/chosen": -0.29042816162109375,
"rewards/margins": 0.2163534164428711,
"rewards/rejected": -0.5067815184593201,
"step": 790
},
{
"epoch": 0.84,
"learning_rate": 3.94611695031086e-08,
"logits/chosen": -1.743678331375122,
"logits/rejected": -1.5813570022583008,
"logps/chosen": -426.080078125,
"logps/rejected": -366.8641662597656,
"loss": 0.6055,
"rewards/accuracies": 0.699999988079071,
"rewards/chosen": -0.26492840051651,
"rewards/margins": 0.23690399527549744,
"rewards/rejected": -0.5018323659896851,
"step": 800
},
{
"epoch": 0.84,
"eval_logits/chosen": -1.809856653213501,
"eval_logits/rejected": -1.6792672872543335,
"eval_logps/chosen": -371.4098815917969,
"eval_logps/rejected": -352.3697204589844,
"eval_loss": 0.6287124156951904,
"eval_rewards/accuracies": 0.6865079402923584,
"eval_rewards/chosen": -0.2877180278301239,
"eval_rewards/margins": 0.2291695475578308,
"eval_rewards/rejected": -0.5168876051902771,
"eval_runtime": 335.4301,
"eval_samples_per_second": 5.962,
"eval_steps_per_second": 0.188,
"step": 800
},
{
"epoch": 0.85,
"learning_rate": 3.465101034171603e-08,
"logits/chosen": -1.8098558187484741,
"logits/rejected": -1.5995447635650635,
"logps/chosen": -368.4771423339844,
"logps/rejected": -318.16156005859375,
"loss": 0.6273,
"rewards/accuracies": 0.612500011920929,
"rewards/chosen": -0.309670627117157,
"rewards/margins": 0.13772638142108917,
"rewards/rejected": -0.44739705324172974,
"step": 810
},
{
"epoch": 0.86,
"learning_rate": 3.013156219837776e-08,
"logits/chosen": -1.638256311416626,
"logits/rejected": -1.5319178104400635,
"logps/chosen": -319.73828125,
"logps/rejected": -316.17022705078125,
"loss": 0.6382,
"rewards/accuracies": 0.6937500238418579,
"rewards/chosen": -0.28010720014572144,
"rewards/margins": 0.2056259661912918,
"rewards/rejected": -0.48573318123817444,
"step": 820
},
{
"epoch": 0.87,
"learning_rate": 2.5908926115744994e-08,
"logits/chosen": -1.6816116571426392,
"logits/rejected": -1.6694707870483398,
"logps/chosen": -323.09210205078125,
"logps/rejected": -325.642822265625,
"loss": 0.6157,
"rewards/accuracies": 0.737500011920929,
"rewards/chosen": -0.2693836987018585,
"rewards/margins": 0.23751957714557648,
"rewards/rejected": -0.5069032907485962,
"step": 830
},
{
"epoch": 0.88,
"learning_rate": 2.19888024541324e-08,
"logits/chosen": -1.7210891246795654,
"logits/rejected": -1.6208484172821045,
"logps/chosen": -391.98187255859375,
"logps/rejected": -361.2024841308594,
"loss": 0.6279,
"rewards/accuracies": 0.574999988079071,
"rewards/chosen": -0.33281436562538147,
"rewards/margins": 0.1134597659111023,
"rewards/rejected": -0.4462741017341614,
"step": 840
},
{
"epoch": 0.89,
"learning_rate": 1.8376483196299558e-08,
"logits/chosen": -1.8490521907806396,
"logits/rejected": -1.7024778127670288,
"logps/chosen": -389.59765625,
"logps/rejected": -336.8246154785156,
"loss": 0.6253,
"rewards/accuracies": 0.675000011920929,
"rewards/chosen": -0.25476616621017456,
"rewards/margins": 0.24750569462776184,
"rewards/rejected": -0.502271831035614,
"step": 850
},
{
"epoch": 0.9,
"learning_rate": 1.507684480352292e-08,
"logits/chosen": -1.7873809337615967,
"logits/rejected": -1.711810827255249,
"logps/chosen": -362.58319091796875,
"logps/rejected": -348.2912292480469,
"loss": 0.6287,
"rewards/accuracies": 0.6812499761581421,
"rewards/chosen": -0.2954009175300598,
"rewards/margins": 0.25710874795913696,
"rewards/rejected": -0.5525097846984863,
"step": 860
},
{
"epoch": 0.91,
"learning_rate": 1.2094341632602062e-08,
"logits/chosen": -1.7503105401992798,
"logits/rejected": -1.698735237121582,
"logps/chosen": -318.9074401855469,
"logps/rejected": -333.00909423828125,
"loss": 0.6184,
"rewards/accuracies": 0.6937500238418579,
"rewards/chosen": -0.25144487619400024,
"rewards/margins": 0.24724820256233215,
"rewards/rejected": -0.49869304895401,
"step": 870
},
{
"epoch": 0.92,
"learning_rate": 9.432999922687396e-09,
"logits/chosen": -1.721398115158081,
"logits/rejected": -1.566706895828247,
"logps/chosen": -368.14556884765625,
"logps/rejected": -340.03277587890625,
"loss": 0.6259,
"rewards/accuracies": 0.6499999761581421,
"rewards/chosen": -0.3243555426597595,
"rewards/margins": 0.1919844150543213,
"rewards/rejected": -0.516339898109436,
"step": 880
},
{
"epoch": 0.93,
"learning_rate": 7.096412360046544e-09,
"logits/chosen": -1.7436186075210571,
"logits/rejected": -1.6783809661865234,
"logps/chosen": -356.3089294433594,
"logps/rejected": -356.8370666503906,
"loss": 0.6375,
"rewards/accuracies": 0.6625000238418579,
"rewards/chosen": -0.3216833472251892,
"rewards/margins": 0.15957646071910858,
"rewards/rejected": -0.4812597632408142,
"step": 890
},
{
"epoch": 0.94,
"learning_rate": 5.087733228106517e-09,
"logits/chosen": -1.7303647994995117,
"logits/rejected": -1.5875599384307861,
"logps/chosen": -385.02294921875,
"logps/rejected": -370.8409729003906,
"loss": 0.6357,
"rewards/accuracies": 0.675000011920929,
"rewards/chosen": -0.29238641262054443,
"rewards/margins": 0.15909823775291443,
"rewards/rejected": -0.451484739780426,
"step": 900
},
{
"epoch": 0.94,
"eval_logits/chosen": -1.8041099309921265,
"eval_logits/rejected": -1.6730901002883911,
"eval_logps/chosen": -373.1398620605469,
"eval_logps/rejected": -354.2070617675781,
"eval_loss": 0.6285176873207092,
"eval_rewards/accuracies": 0.6805555820465088,
"eval_rewards/chosen": -0.30501797795295715,
"eval_rewards/margins": 0.23024281859397888,
"eval_rewards/rejected": -0.535260796546936,
"eval_runtime": 333.5177,
"eval_samples_per_second": 5.997,
"eval_steps_per_second": 0.189,
"step": 900
},
{
"epoch": 0.95,
"learning_rate": 3.4096741493194193e-09,
"logits/chosen": -1.7266037464141846,
"logits/rejected": -1.596879482269287,
"logps/chosen": -330.7945556640625,
"logps/rejected": -328.31414794921875,
"loss": 0.6192,
"rewards/accuracies": 0.675000011920929,
"rewards/chosen": -0.3427024781703949,
"rewards/margins": 0.17462757229804993,
"rewards/rejected": -0.5173300504684448,
"step": 910
},
{
"epoch": 0.96,
"learning_rate": 2.064500424599436e-09,
"logits/chosen": -1.789232611656189,
"logits/rejected": -1.6246925592422485,
"logps/chosen": -370.4477233886719,
"logps/rejected": -351.1963806152344,
"loss": 0.6148,
"rewards/accuracies": 0.731249988079071,
"rewards/chosen": -0.2979881167411804,
"rewards/margins": 0.2658146917819977,
"rewards/rejected": -0.5638028383255005,
"step": 920
},
{
"epoch": 0.97,
"learning_rate": 1.0540279752731252e-09,
"logits/chosen": -1.856554388999939,
"logits/rejected": -1.6683080196380615,
"logps/chosen": -354.5975036621094,
"logps/rejected": -335.6180725097656,
"loss": 0.6047,
"rewards/accuracies": 0.7562500238418579,
"rewards/chosen": -0.24988925457000732,
"rewards/margins": 0.27003243565559387,
"rewards/rejected": -0.5199216604232788,
"step": 930
},
{
"epoch": 0.98,
"learning_rate": 3.7962089167095645e-10,
"logits/chosen": -1.79756760597229,
"logits/rejected": -1.610603928565979,
"logps/chosen": -409.74462890625,
"logps/rejected": -392.2386169433594,
"loss": 0.6098,
"rewards/accuracies": 0.7437499761581421,
"rewards/chosen": -0.29289019107818604,
"rewards/margins": 0.28694167733192444,
"rewards/rejected": -0.5798318386077881,
"step": 940
},
{
"epoch": 0.99,
"learning_rate": 4.2189591669322674e-11,
"logits/chosen": -1.8283344507217407,
"logits/rejected": -1.6751607656478882,
"logps/chosen": -369.1002197265625,
"logps/rejected": -350.08013916015625,
"loss": 0.621,
"rewards/accuracies": 0.71875,
"rewards/chosen": -0.29409345984458923,
"rewards/margins": 0.25061991810798645,
"rewards/rejected": -0.5447134375572205,
"step": 950
},
{
"epoch": 1.0,
"step": 955,
"total_flos": 0.0,
"train_loss": 0.6456358959537526,
"train_runtime": 17140.5232,
"train_samples_per_second": 3.567,
"train_steps_per_second": 0.056
}
],
"logging_steps": 10,
"max_steps": 955,
"num_input_tokens_seen": 0,
"num_train_epochs": 1,
"save_steps": 100,
"total_flos": 0.0,
"train_batch_size": 8,
"trial_name": null,
"trial_params": null
}