zephyr-7b-gpo-iter2 / trainer_state.json
lole25's picture
Model save
6c1d38b verified
{
"best_metric": null,
"best_model_checkpoint": null,
"epoch": 2.0,
"eval_steps": 100,
"global_step": 658,
"is_hyper_param_search": false,
"is_local_process_zero": true,
"is_world_process_zero": true,
"log_history": [
{
"epoch": 0.0,
"learning_rate": 7.575757575757576e-08,
"logits/chosen": -3.0401854515075684,
"logits/rejected": -2.9935803413391113,
"logps/chosen": -33.6235466003418,
"logps/rejected": -31.46235466003418,
"loss": 0.0011,
"rewards/accuracies": 0.0,
"rewards/chosen": 0.0,
"rewards/margins": 0.0,
"rewards/rejected": 0.0,
"step": 1
},
{
"epoch": 0.03,
"learning_rate": 7.575757575757576e-07,
"logits/chosen": -3.0845224857330322,
"logits/rejected": -2.939974784851074,
"logps/chosen": -187.66502380371094,
"logps/rejected": -172.21742248535156,
"loss": 0.0011,
"rewards/accuracies": 0.4444444477558136,
"rewards/chosen": 4.157695730100386e-05,
"rewards/margins": -0.0002048778405878693,
"rewards/rejected": 0.0002464548160787672,
"step": 10
},
{
"epoch": 0.06,
"learning_rate": 1.5151515151515152e-06,
"logits/chosen": -2.975489616394043,
"logits/rejected": -2.8098347187042236,
"logps/chosen": -101.97224426269531,
"logps/rejected": -79.41976928710938,
"loss": 0.001,
"rewards/accuracies": 0.6000000238418579,
"rewards/chosen": 0.0006187028484418988,
"rewards/margins": 0.0002881823165807873,
"rewards/rejected": 0.0003305206191726029,
"step": 20
},
{
"epoch": 0.09,
"learning_rate": 2.2727272727272728e-06,
"logits/chosen": -2.9616599082946777,
"logits/rejected": -2.853991746902466,
"logps/chosen": -93.24244689941406,
"logps/rejected": -93.37409973144531,
"loss": 0.0011,
"rewards/accuracies": 0.5,
"rewards/chosen": 0.0018816631054505706,
"rewards/margins": 0.000262443587416783,
"rewards/rejected": 0.001619219547137618,
"step": 30
},
{
"epoch": 0.12,
"learning_rate": 3.0303030303030305e-06,
"logits/chosen": -2.957383871078491,
"logits/rejected": -2.8873894214630127,
"logps/chosen": -98.40808868408203,
"logps/rejected": -90.6903305053711,
"loss": 0.0011,
"rewards/accuracies": 0.6000000238418579,
"rewards/chosen": 0.0038822232745587826,
"rewards/margins": 0.0006505203782580793,
"rewards/rejected": 0.0032317028380930424,
"step": 40
},
{
"epoch": 0.15,
"learning_rate": 3.7878787878787882e-06,
"logits/chosen": -2.793536424636841,
"logits/rejected": -2.701014995574951,
"logps/chosen": -76.89764404296875,
"logps/rejected": -76.88601684570312,
"loss": 0.0009,
"rewards/accuracies": 0.44999998807907104,
"rewards/chosen": 0.0003440352447796613,
"rewards/margins": 0.003660330083221197,
"rewards/rejected": -0.003316295798867941,
"step": 50
},
{
"epoch": 0.18,
"learning_rate": 4.5454545454545455e-06,
"logits/chosen": -2.8517611026763916,
"logits/rejected": -2.701003313064575,
"logps/chosen": -111.06632232666016,
"logps/rejected": -97.74620056152344,
"loss": 0.001,
"rewards/accuracies": 0.4000000059604645,
"rewards/chosen": 0.0011594895040616393,
"rewards/margins": 0.00348883168771863,
"rewards/rejected": -0.0023293420672416687,
"step": 60
},
{
"epoch": 0.21,
"learning_rate": 4.999436790436924e-06,
"logits/chosen": -3.074284791946411,
"logits/rejected": -2.859107494354248,
"logps/chosen": -175.5392608642578,
"logps/rejected": -165.30654907226562,
"loss": 0.0011,
"rewards/accuracies": 0.5,
"rewards/chosen": 0.0042682006023824215,
"rewards/margins": 0.004156609531491995,
"rewards/rejected": 0.00011159134737681597,
"step": 70
},
{
"epoch": 0.24,
"learning_rate": 4.993103596812269e-06,
"logits/chosen": -2.7597286701202393,
"logits/rejected": -2.6338610649108887,
"logps/chosen": -117.37264251708984,
"logps/rejected": -96.65496826171875,
"loss": 0.0014,
"rewards/accuracies": 0.30000001192092896,
"rewards/chosen": -0.004695770796388388,
"rewards/margins": -0.0023340280167758465,
"rewards/rejected": -0.002361743012443185,
"step": 80
},
{
"epoch": 0.27,
"learning_rate": 4.979751088147192e-06,
"logits/chosen": -2.7671074867248535,
"logits/rejected": -2.5734329223632812,
"logps/chosen": -124.0124740600586,
"logps/rejected": -117.49992370605469,
"loss": 0.0013,
"rewards/accuracies": 0.550000011920929,
"rewards/chosen": -0.0014447696739807725,
"rewards/margins": 0.006112386472523212,
"rewards/rejected": -0.0075571550987660885,
"step": 90
},
{
"epoch": 0.3,
"learning_rate": 4.95941685833271e-06,
"logits/chosen": -2.841676712036133,
"logits/rejected": -2.6840147972106934,
"logps/chosen": -136.9166259765625,
"logps/rejected": -142.25997924804688,
"loss": 0.0012,
"rewards/accuracies": 0.550000011920929,
"rewards/chosen": -0.005834830459207296,
"rewards/margins": 0.0021175735164433718,
"rewards/rejected": -0.007952402345836163,
"step": 100
},
{
"epoch": 0.3,
"eval_logits/chosen": -2.3582723140716553,
"eval_logits/rejected": -2.1644442081451416,
"eval_logps/chosen": -281.1500244140625,
"eval_logps/rejected": -259.7554626464844,
"eval_loss": 0.001627120072953403,
"eval_rewards/accuracies": 0.5034999847412109,
"eval_rewards/chosen": -0.016425278037786484,
"eval_rewards/margins": -0.00047451091813854873,
"eval_rewards/rejected": -0.0159507654607296,
"eval_runtime": 1420.6531,
"eval_samples_per_second": 1.408,
"eval_steps_per_second": 0.704,
"step": 100
},
{
"epoch": 0.33,
"learning_rate": 4.9321581582449365e-06,
"logits/chosen": -2.8993427753448486,
"logits/rejected": -2.786620616912842,
"logps/chosen": -118.65157318115234,
"logps/rejected": -127.34912109375,
"loss": 0.0011,
"rewards/accuracies": 0.44999998807907104,
"rewards/chosen": -0.0042579686269164085,
"rewards/margins": 0.004396714735776186,
"rewards/rejected": -0.008654682897031307,
"step": 110
},
{
"epoch": 0.36,
"learning_rate": 4.898051734555676e-06,
"logits/chosen": -3.0458617210388184,
"logits/rejected": -2.99739670753479,
"logps/chosen": -95.90953063964844,
"logps/rejected": -102.84895324707031,
"loss": 0.0013,
"rewards/accuracies": 0.550000011920929,
"rewards/chosen": -0.004254769999533892,
"rewards/margins": -0.0005313962465152144,
"rewards/rejected": -0.0037233736366033554,
"step": 120
},
{
"epoch": 0.4,
"learning_rate": 4.857193613652711e-06,
"logits/chosen": -3.0125811100006104,
"logits/rejected": -2.8767762184143066,
"logps/chosen": -66.88941955566406,
"logps/rejected": -71.67733001708984,
"loss": 0.0012,
"rewards/accuracies": 0.5,
"rewards/chosen": 0.004949026275426149,
"rewards/margins": 0.0032058332581073046,
"rewards/rejected": 0.001743193482980132,
"step": 130
},
{
"epoch": 0.43,
"learning_rate": 4.809698831278217e-06,
"logits/chosen": -2.9552953243255615,
"logits/rejected": -2.7150585651397705,
"logps/chosen": -75.46265411376953,
"logps/rejected": -72.64187622070312,
"loss": 0.001,
"rewards/accuracies": 0.800000011920929,
"rewards/chosen": 0.00027392804622650146,
"rewards/margins": 0.016122380271553993,
"rewards/rejected": -0.01584845408797264,
"step": 140
},
{
"epoch": 0.46,
"learning_rate": 4.7557011086464634e-06,
"logits/chosen": -2.9132320880889893,
"logits/rejected": -2.64630126953125,
"logps/chosen": -76.73291778564453,
"logps/rejected": -97.9271011352539,
"loss": 0.0013,
"rewards/accuracies": 0.6000000238418579,
"rewards/chosen": -0.00335231376811862,
"rewards/margins": 0.01173357479274273,
"rewards/rejected": -0.015085889026522636,
"step": 150
},
{
"epoch": 0.49,
"learning_rate": 4.695352475952706e-06,
"logits/chosen": -3.013190746307373,
"logits/rejected": -2.926192045211792,
"logps/chosen": -73.78190612792969,
"logps/rejected": -84.00215148925781,
"loss": 0.0016,
"rewards/accuracies": 0.699999988079071,
"rewards/chosen": -0.006495082285255194,
"rewards/margins": 0.01742926612496376,
"rewards/rejected": -0.023924347013235092,
"step": 160
},
{
"epoch": 0.52,
"learning_rate": 4.6288228443332786e-06,
"logits/chosen": -2.892408847808838,
"logits/rejected": -2.785109758377075,
"logps/chosen": -88.50574493408203,
"logps/rejected": -63.979164123535156,
"loss": 0.0014,
"rewards/accuracies": 0.44999998807907104,
"rewards/chosen": -0.015971587970852852,
"rewards/margins": 0.0037319730035960674,
"rewards/rejected": -0.019703561440110207,
"step": 170
},
{
"epoch": 0.55,
"learning_rate": 4.556299527482029e-06,
"logits/chosen": -3.024970293045044,
"logits/rejected": -2.9614624977111816,
"logps/chosen": -95.4378433227539,
"logps/rejected": -86.34746551513672,
"loss": 0.0009,
"rewards/accuracies": 0.75,
"rewards/chosen": 0.0011760194320231676,
"rewards/margins": 0.010892638936638832,
"rewards/rejected": -0.009716618806123734,
"step": 180
},
{
"epoch": 0.58,
"learning_rate": 4.4779867142699715e-06,
"logits/chosen": -3.0479438304901123,
"logits/rejected": -2.903231620788574,
"logps/chosen": -138.85531616210938,
"logps/rejected": -146.4200439453125,
"loss": 0.0017,
"rewards/accuracies": 0.44999998807907104,
"rewards/chosen": 0.013675776310265064,
"rewards/margins": 0.003341741394251585,
"rewards/rejected": 0.010334033519029617,
"step": 190
},
{
"epoch": 0.61,
"learning_rate": 4.394104893853007e-06,
"logits/chosen": -2.9416115283966064,
"logits/rejected": -2.8143601417541504,
"logps/chosen": -85.66619110107422,
"logps/rejected": -84.1985092163086,
"loss": 0.0011,
"rewards/accuracies": 0.6499999761581421,
"rewards/chosen": 0.003594033420085907,
"rewards/margins": 0.005169789306819439,
"rewards/rejected": -0.0015757561195641756,
"step": 200
},
{
"epoch": 0.61,
"eval_logits/chosen": -2.378091812133789,
"eval_logits/rejected": -2.183748960494995,
"eval_logps/chosen": -280.3858337402344,
"eval_logps/rejected": -258.9316711425781,
"eval_loss": 0.0018177788006141782,
"eval_rewards/accuracies": 0.4814999997615814,
"eval_rewards/chosen": -0.008783474564552307,
"eval_rewards/margins": -0.0010705279419198632,
"eval_rewards/rejected": -0.007712946273386478,
"eval_runtime": 1420.2588,
"eval_samples_per_second": 1.408,
"eval_steps_per_second": 0.704,
"step": 200
},
{
"epoch": 0.64,
"learning_rate": 4.3048902348863116e-06,
"logits/chosen": -2.9989707469940186,
"logits/rejected": -2.8650078773498535,
"logps/chosen": -102.37027740478516,
"logps/rejected": -118.4636001586914,
"loss": 0.0012,
"rewards/accuracies": 0.6000000238418579,
"rewards/chosen": 0.001182993990369141,
"rewards/margins": 0.004763666074723005,
"rewards/rejected": -0.003580672200769186,
"step": 210
},
{
"epoch": 0.67,
"learning_rate": 4.210593920593201e-06,
"logits/chosen": -2.9398374557495117,
"logits/rejected": -2.8116965293884277,
"logps/chosen": -70.69538879394531,
"logps/rejected": -75.53007507324219,
"loss": 0.0013,
"rewards/accuracies": 0.6000000238418579,
"rewards/chosen": 0.005323179066181183,
"rewards/margins": 0.004396263975650072,
"rewards/rejected": 0.0009269150905311108,
"step": 220
},
{
"epoch": 0.7,
"learning_rate": 4.111481441560598e-06,
"logits/chosen": -2.972592830657959,
"logits/rejected": -2.852461099624634,
"logps/chosen": -160.5238037109375,
"logps/rejected": -129.62954711914062,
"loss": 0.0013,
"rewards/accuracies": 0.44999998807907104,
"rewards/chosen": 0.00035965832648798823,
"rewards/margins": 0.005134374834597111,
"rewards/rejected": -0.004774716217070818,
"step": 230
},
{
"epoch": 0.73,
"learning_rate": 4.007831848252212e-06,
"logits/chosen": -3.1746320724487305,
"logits/rejected": -2.9887349605560303,
"logps/chosen": -124.45001220703125,
"logps/rejected": -101.69696807861328,
"loss": 0.0009,
"rewards/accuracies": 0.6499999761581421,
"rewards/chosen": 0.019443057477474213,
"rewards/margins": 0.012666161172091961,
"rewards/rejected": 0.0067768944427371025,
"step": 240
},
{
"epoch": 0.76,
"learning_rate": 3.899936965343989e-06,
"logits/chosen": -3.0935516357421875,
"logits/rejected": -3.028534412384033,
"logps/chosen": -131.24337768554688,
"logps/rejected": -116.88172912597656,
"loss": 0.0016,
"rewards/accuracies": 0.44999998807907104,
"rewards/chosen": 0.005977076943963766,
"rewards/margins": 0.005966751836240292,
"rewards/rejected": 1.0324455615773331e-05,
"step": 250
},
{
"epoch": 0.79,
"learning_rate": 3.7881005700938635e-06,
"logits/chosen": -2.7336764335632324,
"logits/rejected": -2.7675654888153076,
"logps/chosen": -95.82691955566406,
"logps/rejected": -99.57430267333984,
"loss": 0.0008,
"rewards/accuracies": 0.8500000238418579,
"rewards/chosen": 0.012143732979893684,
"rewards/margins": 0.008911015465855598,
"rewards/rejected": 0.003232717514038086,
"step": 260
},
{
"epoch": 0.82,
"learning_rate": 3.6726375370590927e-06,
"logits/chosen": -2.959117889404297,
"logits/rejected": -2.841716766357422,
"logps/chosen": -132.79884338378906,
"logps/rejected": -122.66941833496094,
"loss": 0.0015,
"rewards/accuracies": 0.550000011920929,
"rewards/chosen": 0.003037696471437812,
"rewards/margins": 0.00202806293964386,
"rewards/rejected": 0.0010096338810399175,
"step": 270
},
{
"epoch": 0.85,
"learning_rate": 3.553872951569236e-06,
"logits/chosen": -3.0903332233428955,
"logits/rejected": -3.009970188140869,
"logps/chosen": -90.32659912109375,
"logps/rejected": -92.47020721435547,
"loss": 0.0012,
"rewards/accuracies": 0.5,
"rewards/chosen": 0.005868277512490749,
"rewards/margins": 0.011209800839424133,
"rewards/rejected": -0.005341522395610809,
"step": 280
},
{
"epoch": 0.88,
"learning_rate": 3.432141194450772e-06,
"logits/chosen": -2.920701503753662,
"logits/rejected": -2.721304416656494,
"logps/chosen": -111.44612121582031,
"logps/rejected": -111.91583251953125,
"loss": 0.0007,
"rewards/accuracies": 0.699999988079071,
"rewards/chosen": -0.004392694681882858,
"rewards/margins": 0.016653168946504593,
"rewards/rejected": -0.02104586362838745,
"step": 290
},
{
"epoch": 0.91,
"learning_rate": 3.307785000580313e-06,
"logits/chosen": -2.757887363433838,
"logits/rejected": -2.5684075355529785,
"logps/chosen": -128.5594024658203,
"logps/rejected": -115.97843933105469,
"loss": 0.0015,
"rewards/accuracies": 0.4000000059604645,
"rewards/chosen": -0.006182619370520115,
"rewards/margins": 0.0031226237770169973,
"rewards/rejected": -0.009305243380367756,
"step": 300
},
{
"epoch": 0.91,
"eval_logits/chosen": -2.374006748199463,
"eval_logits/rejected": -2.1796298027038574,
"eval_logps/chosen": -281.17401123046875,
"eval_logps/rejected": -259.6520690917969,
"eval_loss": 0.0019075373420491815,
"eval_rewards/accuracies": 0.4805000126361847,
"eval_rewards/chosen": -0.016665350645780563,
"eval_rewards/margins": -0.0017483622068539262,
"eval_rewards/rejected": -0.014916987158358097,
"eval_runtime": 1421.116,
"eval_samples_per_second": 1.407,
"eval_steps_per_second": 0.704,
"step": 300
},
{
"epoch": 0.94,
"learning_rate": 3.1811544939170573e-06,
"logits/chosen": -2.7975335121154785,
"logits/rejected": -2.667598009109497,
"logps/chosen": -161.1660919189453,
"logps/rejected": -133.93838500976562,
"loss": 0.0012,
"rewards/accuracies": 0.44999998807907104,
"rewards/chosen": -0.010928490199148655,
"rewards/margins": 0.00381992245092988,
"rewards/rejected": -0.014748412184417248,
"step": 310
},
{
"epoch": 0.97,
"learning_rate": 3.052606201731325e-06,
"logits/chosen": -2.93320894241333,
"logits/rejected": -2.7140159606933594,
"logps/chosen": -79.52749633789062,
"logps/rejected": -84.45579528808594,
"loss": 0.0008,
"rewards/accuracies": 0.699999988079071,
"rewards/chosen": -0.0026647502090781927,
"rewards/margins": 0.009594769217073917,
"rewards/rejected": -0.01225951872766018,
"step": 320
},
{
"epoch": 1.0,
"learning_rate": 2.9225020508046233e-06,
"logits/chosen": -2.901916742324829,
"logits/rejected": -2.736692190170288,
"logps/chosen": -115.4826431274414,
"logps/rejected": -130.37356567382812,
"loss": 0.0313,
"rewards/accuracies": 0.699999988079071,
"rewards/chosen": 0.017747070640325546,
"rewards/margins": 0.05266844481229782,
"rewards/rejected": -0.034921370446681976,
"step": 330
},
{
"epoch": 1.03,
"learning_rate": 2.7912083484274266e-06,
"logits/chosen": -2.7787222862243652,
"logits/rejected": -2.63339900970459,
"logps/chosen": -137.5430450439453,
"logps/rejected": -127.07392883300781,
"loss": 0.0436,
"rewards/accuracies": 0.800000011920929,
"rewards/chosen": 0.0036686602979898453,
"rewards/margins": 0.05118006467819214,
"rewards/rejected": -0.04751140996813774,
"step": 340
},
{
"epoch": 1.06,
"learning_rate": 2.659094751063666e-06,
"logits/chosen": -2.9705729484558105,
"logits/rejected": -2.832822322845459,
"logps/chosen": -143.50698852539062,
"logps/rejected": -140.02352905273438,
"loss": 0.0133,
"rewards/accuracies": 0.75,
"rewards/chosen": 0.008256057277321815,
"rewards/margins": 0.05859958380460739,
"rewards/rejected": -0.05034352466464043,
"step": 350
},
{
"epoch": 1.09,
"learning_rate": 2.526533223585641e-06,
"logits/chosen": -2.8163113594055176,
"logits/rejected": -2.697537660598755,
"logps/chosen": -85.76821899414062,
"logps/rejected": -81.71627807617188,
"loss": 0.0073,
"rewards/accuracies": 0.6499999761581421,
"rewards/chosen": 0.010457667522132397,
"rewards/margins": 0.053769804537296295,
"rewards/rejected": -0.043312136083841324,
"step": 360
},
{
"epoch": 1.12,
"learning_rate": 2.39389699200963e-06,
"logits/chosen": -3.000455141067505,
"logits/rejected": -2.8073432445526123,
"logps/chosen": -107.18000793457031,
"logps/rejected": -118.2176284790039,
"loss": 0.019,
"rewards/accuracies": 0.800000011920929,
"rewards/chosen": 0.008137053810060024,
"rewards/margins": 0.07659684866666794,
"rewards/rejected": -0.06845979392528534,
"step": 370
},
{
"epoch": 1.16,
"learning_rate": 2.2615594926807554e-06,
"logits/chosen": -2.9433727264404297,
"logits/rejected": -2.817605495452881,
"logps/chosen": -84.96995544433594,
"logps/rejected": -95.87684631347656,
"loss": 0.0277,
"rewards/accuracies": 0.8999999761581421,
"rewards/chosen": 0.01651051640510559,
"rewards/margins": 0.06000148132443428,
"rewards/rejected": -0.04349096864461899,
"step": 380
},
{
"epoch": 1.19,
"learning_rate": 2.129893320865672e-06,
"logits/chosen": -3.0510168075561523,
"logits/rejected": -2.938931465148926,
"logps/chosen": -76.3388442993164,
"logps/rejected": -62.002540588378906,
"loss": 0.0302,
"rewards/accuracies": 0.8500000238418579,
"rewards/chosen": 0.04483187943696976,
"rewards/margins": 0.10971853882074356,
"rewards/rejected": -0.0648866593837738,
"step": 390
},
{
"epoch": 1.22,
"learning_rate": 1.9992691817133025e-06,
"logits/chosen": -3.05735445022583,
"logits/rejected": -2.815657138824463,
"logps/chosen": -251.65219116210938,
"logps/rejected": -255.02658081054688,
"loss": 0.0397,
"rewards/accuracies": 0.75,
"rewards/chosen": -0.03865816444158554,
"rewards/margins": 0.04983866214752197,
"rewards/rejected": -0.08849682658910751,
"step": 400
},
{
"epoch": 1.22,
"eval_logits/chosen": -2.356828451156616,
"eval_logits/rejected": -2.1631951332092285,
"eval_logps/chosen": -287.29345703125,
"eval_logps/rejected": -264.43231201171875,
"eval_loss": 0.007440177723765373,
"eval_rewards/accuracies": 0.41600000858306885,
"eval_rewards/chosen": -0.0778598040342331,
"eval_rewards/margins": -0.015140472911298275,
"eval_rewards/rejected": -0.06271932274103165,
"eval_runtime": 1421.1528,
"eval_samples_per_second": 1.407,
"eval_steps_per_second": 0.704,
"step": 400
},
{
"epoch": 1.25,
"learning_rate": 1.8700548465371877e-06,
"logits/chosen": -3.085784435272217,
"logits/rejected": -2.8534493446350098,
"logps/chosen": -150.0517120361328,
"logps/rejected": -130.98390197753906,
"loss": 0.0304,
"rewards/accuracies": 0.75,
"rewards/chosen": -0.026314150542020798,
"rewards/margins": 0.05200232192873955,
"rewards/rejected": -0.07831647992134094,
"step": 410
},
{
"epoch": 1.28,
"learning_rate": 1.742614117358029e-06,
"logits/chosen": -2.8179147243499756,
"logits/rejected": -2.72475528717041,
"logps/chosen": -85.7872543334961,
"logps/rejected": -98.0990219116211,
"loss": 0.0839,
"rewards/accuracies": 0.6499999761581421,
"rewards/chosen": -0.052498847246170044,
"rewards/margins": 0.009759088046848774,
"rewards/rejected": -0.06225793436169624,
"step": 420
},
{
"epoch": 1.31,
"learning_rate": 1.617305802621748e-06,
"logits/chosen": -2.978959083557129,
"logits/rejected": -2.9247641563415527,
"logps/chosen": -94.34339904785156,
"logps/rejected": -103.36552429199219,
"loss": 0.0221,
"rewards/accuracies": 0.800000011920929,
"rewards/chosen": 0.004420995246618986,
"rewards/margins": 0.09026098251342773,
"rewards/rejected": -0.08583998680114746,
"step": 430
},
{
"epoch": 1.34,
"learning_rate": 1.4944827069769125e-06,
"logits/chosen": -2.94343900680542,
"logits/rejected": -2.8201680183410645,
"logps/chosen": -164.69044494628906,
"logps/rejected": -141.93528747558594,
"loss": 0.004,
"rewards/accuracies": 0.550000011920929,
"rewards/chosen": -0.01349672395735979,
"rewards/margins": 0.0265080276876688,
"rewards/rejected": -0.040004752576351166,
"step": 440
},
{
"epoch": 1.37,
"learning_rate": 1.3744906379558165e-06,
"logits/chosen": -2.9248745441436768,
"logits/rejected": -2.9322562217712402,
"logps/chosen": -103.63645935058594,
"logps/rejected": -123.60746765136719,
"loss": 0.0383,
"rewards/accuracies": 0.8500000238418579,
"rewards/chosen": 0.014714512042701244,
"rewards/margins": 0.11696286499500275,
"rewards/rejected": -0.10224835574626923,
"step": 450
},
{
"epoch": 1.4,
"learning_rate": 1.257667432355893e-06,
"logits/chosen": -2.974544048309326,
"logits/rejected": -2.780351161956787,
"logps/chosen": -76.25318145751953,
"logps/rejected": -95.62451934814453,
"loss": 0.0157,
"rewards/accuracies": 0.8500000238418579,
"rewards/chosen": 0.0021516154520213604,
"rewards/margins": 0.10200606286525726,
"rewards/rejected": -0.09985444694757462,
"step": 460
},
{
"epoch": 1.43,
"learning_rate": 1.1443420050626624e-06,
"logits/chosen": -3.061354875564575,
"logits/rejected": -2.9399447441101074,
"logps/chosen": -71.61457824707031,
"logps/rejected": -85.53517150878906,
"loss": 0.047,
"rewards/accuracies": 0.800000011920929,
"rewards/chosen": -0.00521849375218153,
"rewards/margins": 0.06701686233282089,
"rewards/rejected": -0.0722353607416153,
"step": 470
},
{
"epoch": 1.46,
"learning_rate": 1.0348334229922677e-06,
"logits/chosen": -3.0284695625305176,
"logits/rejected": -2.7861952781677246,
"logps/chosen": -84.50325012207031,
"logps/rejected": -80.56702423095703,
"loss": 0.0331,
"rewards/accuracies": 0.75,
"rewards/chosen": 0.010606551542878151,
"rewards/margins": 0.08576520532369614,
"rewards/rejected": -0.07515865564346313,
"step": 480
},
{
"epoch": 1.49,
"learning_rate": 9.294500067608941e-07,
"logits/chosen": -2.929229974746704,
"logits/rejected": -2.727160692214966,
"logps/chosen": -106.28855895996094,
"logps/rejected": -96.00597381591797,
"loss": 0.0183,
"rewards/accuracies": 0.6000000238418579,
"rewards/chosen": -0.032337434589862823,
"rewards/margins": 0.024760348722338676,
"rewards/rejected": -0.05709778517484665,
"step": 490
},
{
"epoch": 1.52,
"learning_rate": 8.284884626103165e-07,
"logits/chosen": -2.9482462406158447,
"logits/rejected": -2.7884809970855713,
"logps/chosen": -77.4437255859375,
"logps/rejected": -83.18536376953125,
"loss": 0.0305,
"rewards/accuracies": 0.8999999761581421,
"rewards/chosen": 0.025916963815689087,
"rewards/margins": 0.11217900365591049,
"rewards/rejected": -0.086262047290802,
"step": 500
},
{
"epoch": 1.52,
"eval_logits/chosen": -2.3874802589416504,
"eval_logits/rejected": -2.1901655197143555,
"eval_logps/chosen": -288.4842224121094,
"eval_logps/rejected": -264.83880615234375,
"eval_loss": 0.011713836342096329,
"eval_rewards/accuracies": 0.3944999873638153,
"eval_rewards/chosen": -0.08976726979017258,
"eval_rewards/margins": -0.022982925176620483,
"eval_rewards/rejected": -0.06678435206413269,
"eval_runtime": 1421.6286,
"eval_samples_per_second": 1.407,
"eval_steps_per_second": 0.703,
"step": 500
},
{
"epoch": 1.55,
"learning_rate": 7.322330470336314e-07,
"logits/chosen": -3.0825541019439697,
"logits/rejected": -2.9925718307495117,
"logps/chosen": -95.14485931396484,
"logps/rejected": -104.95149993896484,
"loss": 0.0318,
"rewards/accuracies": 0.8500000238418579,
"rewards/chosen": 0.004621284082531929,
"rewards/margins": 0.12566979229450226,
"rewards/rejected": -0.12104851007461548,
"step": 510
},
{
"epoch": 1.58,
"learning_rate": 6.409547664531734e-07,
"logits/chosen": -2.9336235523223877,
"logits/rejected": -2.8529655933380127,
"logps/chosen": -130.2211151123047,
"logps/rejected": -122.72911071777344,
"loss": 0.0063,
"rewards/accuracies": 0.800000011920929,
"rewards/chosen": 0.002978086471557617,
"rewards/margins": 0.05790657550096512,
"rewards/rejected": -0.0549284890294075,
"step": 520
},
{
"epoch": 1.61,
"learning_rate": 5.549106142039018e-07,
"logits/chosen": -3.001368522644043,
"logits/rejected": -2.9931020736694336,
"logps/chosen": -122.59478759765625,
"logps/rejected": -139.2528839111328,
"loss": 0.0343,
"rewards/accuracies": 0.6499999761581421,
"rewards/chosen": -0.03542733192443848,
"rewards/margins": 0.028016597032546997,
"rewards/rejected": -0.06344393640756607,
"step": 530
},
{
"epoch": 1.64,
"learning_rate": 4.743428469705336e-07,
"logits/chosen": -2.895446300506592,
"logits/rejected": -2.805732011795044,
"logps/chosen": -77.88665771484375,
"logps/rejected": -86.38732147216797,
"loss": 0.0152,
"rewards/accuracies": 0.8500000238418579,
"rewards/chosen": 0.012998471967875957,
"rewards/margins": 0.09006098657846451,
"rewards/rejected": -0.07706250250339508,
"step": 540
},
{
"epoch": 1.67,
"learning_rate": 3.994783027156143e-07,
"logits/chosen": -2.8439111709594727,
"logits/rejected": -2.765376567840576,
"logps/chosen": -124.60064697265625,
"logps/rejected": -115.8946533203125,
"loss": 0.0116,
"rewards/accuracies": 0.800000011920929,
"rewards/chosen": -0.0016067728865891695,
"rewards/margins": 0.08273078501224518,
"rewards/rejected": -0.08433755487203598,
"step": 550
},
{
"epoch": 1.7,
"learning_rate": 3.3052776201888266e-07,
"logits/chosen": -2.8251261711120605,
"logits/rejected": -2.69478178024292,
"logps/chosen": -85.72721862792969,
"logps/rejected": -84.4536361694336,
"loss": 0.0146,
"rewards/accuracies": 0.75,
"rewards/chosen": 0.007551318965852261,
"rewards/margins": 0.08319269120693207,
"rewards/rejected": -0.07564137130975723,
"step": 560
},
{
"epoch": 1.73,
"learning_rate": 2.676853546260791e-07,
"logits/chosen": -2.8717586994171143,
"logits/rejected": -2.805624485015869,
"logps/chosen": -102.53792572021484,
"logps/rejected": -89.75617980957031,
"loss": 0.0086,
"rewards/accuracies": 0.8999999761581421,
"rewards/chosen": -0.008890565484762192,
"rewards/margins": 0.04497462511062622,
"rewards/rejected": -0.05386519432067871,
"step": 570
},
{
"epoch": 1.76,
"learning_rate": 2.111280128780638e-07,
"logits/chosen": -3.0598039627075195,
"logits/rejected": -2.9559364318847656,
"logps/chosen": -84.91885375976562,
"logps/rejected": -75.99688720703125,
"loss": 0.0162,
"rewards/accuracies": 0.75,
"rewards/chosen": -0.025450268760323524,
"rewards/margins": 0.03124385140836239,
"rewards/rejected": -0.056694112718105316,
"step": 580
},
{
"epoch": 1.79,
"learning_rate": 1.610149735590949e-07,
"logits/chosen": -2.929579496383667,
"logits/rejected": -2.8060431480407715,
"logps/chosen": -60.950294494628906,
"logps/rejected": -59.30121994018555,
"loss": 0.0299,
"rewards/accuracies": 0.8500000238418579,
"rewards/chosen": -0.03121623769402504,
"rewards/margins": 0.015263142995536327,
"rewards/rejected": -0.04647938907146454,
"step": 590
},
{
"epoch": 1.82,
"learning_rate": 1.1748732956682023e-07,
"logits/chosen": -3.107717275619507,
"logits/rejected": -2.9001567363739014,
"logps/chosen": -90.45118713378906,
"logps/rejected": -84.7816390991211,
"loss": 0.0366,
"rewards/accuracies": 0.75,
"rewards/chosen": -0.021300766617059708,
"rewards/margins": 0.0555335208773613,
"rewards/rejected": -0.0768342837691307,
"step": 600
},
{
"epoch": 1.82,
"eval_logits/chosen": -2.3873283863067627,
"eval_logits/rejected": -2.1899969577789307,
"eval_logps/chosen": -288.27227783203125,
"eval_logps/rejected": -264.630126953125,
"eval_loss": 0.011489564552903175,
"eval_rewards/accuracies": 0.4000000059604645,
"eval_rewards/chosen": -0.08764798194169998,
"eval_rewards/margins": -0.02295052260160446,
"eval_rewards/rejected": -0.06469745188951492,
"eval_runtime": 1422.4891,
"eval_samples_per_second": 1.406,
"eval_steps_per_second": 0.703,
"step": 600
},
{
"epoch": 1.85,
"learning_rate": 8.066763266625283e-08,
"logits/chosen": -2.9506607055664062,
"logits/rejected": -2.825108051300049,
"logps/chosen": -100.47017669677734,
"logps/rejected": -127.78892517089844,
"loss": 0.0349,
"rewards/accuracies": 0.6499999761581421,
"rewards/chosen": 0.004581383429467678,
"rewards/margins": 0.1253204494714737,
"rewards/rejected": -0.12073905766010284,
"step": 610
},
{
"epoch": 1.88,
"learning_rate": 5.065954844616722e-08,
"logits/chosen": -2.9829344749450684,
"logits/rejected": -2.7227559089660645,
"logps/chosen": -101.5233154296875,
"logps/rejected": -99.42388916015625,
"loss": 0.027,
"rewards/accuracies": 0.699999988079071,
"rewards/chosen": -0.0389946773648262,
"rewards/margins": 0.016298165544867516,
"rewards/rejected": -0.05529283359646797,
"step": 620
},
{
"epoch": 1.91,
"learning_rate": 2.7547564449386666e-08,
"logits/chosen": -2.7666523456573486,
"logits/rejected": -2.719158887863159,
"logps/chosen": -215.42599487304688,
"logps/rejected": -217.2696075439453,
"loss": 0.0308,
"rewards/accuracies": 0.699999988079071,
"rewards/chosen": -0.015156986191868782,
"rewards/margins": 0.07782644778490067,
"rewards/rejected": -0.0929834321141243,
"step": 630
},
{
"epoch": 1.95,
"learning_rate": 1.1396752298723501e-08,
"logits/chosen": -2.9985458850860596,
"logits/rejected": -2.927014112472534,
"logps/chosen": -109.7343521118164,
"logps/rejected": -111.255859375,
"loss": 0.0344,
"rewards/accuracies": 0.800000011920929,
"rewards/chosen": 0.020465310662984848,
"rewards/margins": 0.10245206207036972,
"rewards/rejected": -0.08198676258325577,
"step": 640
},
{
"epoch": 1.98,
"learning_rate": 2.252584488296461e-09,
"logits/chosen": -2.9206175804138184,
"logits/rejected": -2.8202061653137207,
"logps/chosen": -157.34161376953125,
"logps/rejected": -171.29029846191406,
"loss": 0.0403,
"rewards/accuracies": 0.8999999761581421,
"rewards/chosen": -0.005368991754949093,
"rewards/margins": 0.11630947887897491,
"rewards/rejected": -0.12167844921350479,
"step": 650
},
{
"epoch": 2.0,
"step": 658,
"total_flos": 0.0,
"train_loss": 0.014390302914878392,
"train_runtime": 11750.1681,
"train_samples_per_second": 0.112,
"train_steps_per_second": 0.056
}
],
"logging_steps": 10,
"max_steps": 658,
"num_input_tokens_seen": 0,
"num_train_epochs": 2,
"save_steps": 100,
"total_flos": 0.0,
"train_batch_size": 1,
"trial_name": null,
"trial_params": null
}