|
{ |
|
"best_metric": null, |
|
"best_model_checkpoint": null, |
|
"epoch": 35.55555555555556, |
|
"eval_steps": 40, |
|
"global_step": 560, |
|
"is_hyper_param_search": false, |
|
"is_local_process_zero": true, |
|
"is_world_process_zero": true, |
|
"log_history": [ |
|
{ |
|
"epoch": 0.25, |
|
"grad_norm": 7.928347587585449, |
|
"learning_rate": 7.5e-08, |
|
"logits/chosen": -2.7277705669403076, |
|
"logits/rejected": -2.7679762840270996, |
|
"logps/chosen": -128.4798583984375, |
|
"logps/rejected": -98.36178588867188, |
|
"loss": 0.6931, |
|
"rewards/accuracies": 0.109375, |
|
"rewards/chosen": -7.17043731128797e-05, |
|
"rewards/margins": 5.325676465872675e-05, |
|
"rewards/rejected": -0.0001249611668754369, |
|
"step": 4 |
|
}, |
|
{ |
|
"epoch": 0.51, |
|
"grad_norm": 9.242063522338867, |
|
"learning_rate": 1.75e-07, |
|
"logits/chosen": -2.6805009841918945, |
|
"logits/rejected": -2.712625026702881, |
|
"logps/chosen": -138.11959838867188, |
|
"logps/rejected": -96.05882263183594, |
|
"loss": 0.693, |
|
"rewards/accuracies": 0.546875, |
|
"rewards/chosen": 0.0008271098486147821, |
|
"rewards/margins": 0.00034989125560969114, |
|
"rewards/rejected": 0.00047721865121275187, |
|
"step": 8 |
|
}, |
|
{ |
|
"epoch": 0.76, |
|
"grad_norm": 8.156970977783203, |
|
"learning_rate": 2.75e-07, |
|
"logits/chosen": -2.7695400714874268, |
|
"logits/rejected": -2.7420883178710938, |
|
"logps/chosen": -133.04307556152344, |
|
"logps/rejected": -104.19400024414062, |
|
"loss": 0.6919, |
|
"rewards/accuracies": 0.609375, |
|
"rewards/chosen": 0.0034509659744799137, |
|
"rewards/margins": 0.0025641382671892643, |
|
"rewards/rejected": 0.0008868275908753276, |
|
"step": 12 |
|
}, |
|
{ |
|
"epoch": 1.02, |
|
"grad_norm": 8.723258018493652, |
|
"learning_rate": 3.75e-07, |
|
"logits/chosen": -2.6869003772735596, |
|
"logits/rejected": -2.7081785202026367, |
|
"logps/chosen": -126.2979736328125, |
|
"logps/rejected": -98.4257583618164, |
|
"loss": 0.6889, |
|
"rewards/accuracies": 0.828125, |
|
"rewards/chosen": 0.011365305632352829, |
|
"rewards/margins": 0.008443659171462059, |
|
"rewards/rejected": 0.0029216469265520573, |
|
"step": 16 |
|
}, |
|
{ |
|
"epoch": 1.27, |
|
"grad_norm": 10.34566593170166, |
|
"learning_rate": 4.7499999999999995e-07, |
|
"logits/chosen": -2.704402208328247, |
|
"logits/rejected": -2.764737367630005, |
|
"logps/chosen": -125.91941833496094, |
|
"logps/rejected": -99.02344512939453, |
|
"loss": 0.6832, |
|
"rewards/accuracies": 0.9375, |
|
"rewards/chosen": 0.02058127522468567, |
|
"rewards/margins": 0.020110297948122025, |
|
"rewards/rejected": 0.00047097797505557537, |
|
"step": 20 |
|
}, |
|
{ |
|
"epoch": 1.52, |
|
"grad_norm": 9.291991233825684, |
|
"learning_rate": 4.999985633126757e-07, |
|
"logits/chosen": -2.716165781021118, |
|
"logits/rejected": -2.6845083236694336, |
|
"logps/chosen": -137.2718505859375, |
|
"logps/rejected": -97.50873565673828, |
|
"loss": 0.6785, |
|
"rewards/accuracies": 0.90625, |
|
"rewards/chosen": 0.03298673778772354, |
|
"rewards/margins": 0.029706846922636032, |
|
"rewards/rejected": 0.0032798887696117163, |
|
"step": 24 |
|
}, |
|
{ |
|
"epoch": 1.78, |
|
"grad_norm": 9.161444664001465, |
|
"learning_rate": 4.999921780689761e-07, |
|
"logits/chosen": -2.735781192779541, |
|
"logits/rejected": -2.7485365867614746, |
|
"logps/chosen": -128.5098876953125, |
|
"logps/rejected": -90.1419906616211, |
|
"loss": 0.6693, |
|
"rewards/accuracies": 0.953125, |
|
"rewards/chosen": 0.04758748784661293, |
|
"rewards/margins": 0.04847026616334915, |
|
"rewards/rejected": -0.0008827749406918883, |
|
"step": 28 |
|
}, |
|
{ |
|
"epoch": 2.03, |
|
"grad_norm": 8.495955467224121, |
|
"learning_rate": 4.99980684767309e-07, |
|
"logits/chosen": -2.7206833362579346, |
|
"logits/rejected": -2.750042676925659, |
|
"logps/chosen": -133.54827880859375, |
|
"logps/rejected": -108.23342895507812, |
|
"loss": 0.6669, |
|
"rewards/accuracies": 0.953125, |
|
"rewards/chosen": 0.0631466805934906, |
|
"rewards/margins": 0.05378420650959015, |
|
"rewards/rejected": 0.009362474083900452, |
|
"step": 32 |
|
}, |
|
{ |
|
"epoch": 2.29, |
|
"grad_norm": 8.951277732849121, |
|
"learning_rate": 4.999640836425158e-07, |
|
"logits/chosen": -2.6706807613372803, |
|
"logits/rejected": -2.6151680946350098, |
|
"logps/chosen": -136.55226135253906, |
|
"logps/rejected": -97.76315307617188, |
|
"loss": 0.6578, |
|
"rewards/accuracies": 0.953125, |
|
"rewards/chosen": 0.08149851858615875, |
|
"rewards/margins": 0.07302143424749374, |
|
"rewards/rejected": 0.008477086201310158, |
|
"step": 36 |
|
}, |
|
{ |
|
"epoch": 2.54, |
|
"grad_norm": 8.608317375183105, |
|
"learning_rate": 4.99942375033805e-07, |
|
"logits/chosen": -2.7676639556884766, |
|
"logits/rejected": -2.833590030670166, |
|
"logps/chosen": -133.7703857421875, |
|
"logps/rejected": -106.12245178222656, |
|
"loss": 0.6488, |
|
"rewards/accuracies": 0.9375, |
|
"rewards/chosen": 0.10222765803337097, |
|
"rewards/margins": 0.09170522540807724, |
|
"rewards/rejected": 0.010522443801164627, |
|
"step": 40 |
|
}, |
|
{ |
|
"epoch": 2.54, |
|
"eval_logits/chosen": -2.806352138519287, |
|
"eval_logits/rejected": -2.8898861408233643, |
|
"eval_logps/chosen": -133.14093017578125, |
|
"eval_logps/rejected": -97.8843002319336, |
|
"eval_loss": 0.6592782139778137, |
|
"eval_rewards/accuracies": 0.7857142686843872, |
|
"eval_rewards/chosen": 0.09283973276615143, |
|
"eval_rewards/margins": 0.07038124650716782, |
|
"eval_rewards/rejected": 0.022458484396338463, |
|
"eval_runtime": 19.3683, |
|
"eval_samples_per_second": 1.446, |
|
"eval_steps_per_second": 1.446, |
|
"step": 40 |
|
}, |
|
{ |
|
"epoch": 2.79, |
|
"grad_norm": 9.09485912322998, |
|
"learning_rate": 4.999155593847457e-07, |
|
"logits/chosen": -2.7024385929107666, |
|
"logits/rejected": -2.7564737796783447, |
|
"logps/chosen": -123.412353515625, |
|
"logps/rejected": -91.91966247558594, |
|
"loss": 0.6347, |
|
"rewards/accuracies": 0.96875, |
|
"rewards/chosen": 0.114817313849926, |
|
"rewards/margins": 0.12200291454792023, |
|
"rewards/rejected": -0.007185595575720072, |
|
"step": 44 |
|
}, |
|
{ |
|
"epoch": 3.05, |
|
"grad_norm": 9.453149795532227, |
|
"learning_rate": 4.998836372432589e-07, |
|
"logits/chosen": -2.736665725708008, |
|
"logits/rejected": -2.757841110229492, |
|
"logps/chosen": -127.0240478515625, |
|
"logps/rejected": -103.66780853271484, |
|
"loss": 0.636, |
|
"rewards/accuracies": 0.96875, |
|
"rewards/chosen": 0.13144327700138092, |
|
"rewards/margins": 0.11972987651824951, |
|
"rewards/rejected": 0.011713397689163685, |
|
"step": 48 |
|
}, |
|
{ |
|
"epoch": 3.3, |
|
"grad_norm": 8.674492835998535, |
|
"learning_rate": 4.998466092616052e-07, |
|
"logits/chosen": -2.710110664367676, |
|
"logits/rejected": -2.7205581665039062, |
|
"logps/chosen": -131.0464324951172, |
|
"logps/rejected": -99.71279907226562, |
|
"loss": 0.6166, |
|
"rewards/accuracies": 0.96875, |
|
"rewards/chosen": 0.15897569060325623, |
|
"rewards/margins": 0.16229701042175293, |
|
"rewards/rejected": -0.003321333322674036, |
|
"step": 52 |
|
}, |
|
{ |
|
"epoch": 3.56, |
|
"grad_norm": 9.434328079223633, |
|
"learning_rate": 4.99804476196373e-07, |
|
"logits/chosen": -2.685814380645752, |
|
"logits/rejected": -2.736269474029541, |
|
"logps/chosen": -125.750732421875, |
|
"logps/rejected": -93.69428253173828, |
|
"loss": 0.6131, |
|
"rewards/accuracies": 0.96875, |
|
"rewards/chosen": 0.16781488060951233, |
|
"rewards/margins": 0.16998659074306488, |
|
"rewards/rejected": -0.002171688713133335, |
|
"step": 56 |
|
}, |
|
{ |
|
"epoch": 3.81, |
|
"grad_norm": 8.643455505371094, |
|
"learning_rate": 4.997572389084615e-07, |
|
"logits/chosen": -2.7429587841033936, |
|
"logits/rejected": -2.75490140914917, |
|
"logps/chosen": -131.4405517578125, |
|
"logps/rejected": -99.93754577636719, |
|
"loss": 0.614, |
|
"rewards/accuracies": 0.921875, |
|
"rewards/chosen": 0.18873561918735504, |
|
"rewards/margins": 0.17132429778575897, |
|
"rewards/rejected": 0.01741132140159607, |
|
"step": 60 |
|
}, |
|
{ |
|
"epoch": 4.06, |
|
"grad_norm": 8.412778854370117, |
|
"learning_rate": 4.997048983630643e-07, |
|
"logits/chosen": -2.6886143684387207, |
|
"logits/rejected": -2.710732936859131, |
|
"logps/chosen": -123.77167510986328, |
|
"logps/rejected": -97.30846405029297, |
|
"loss": 0.6099, |
|
"rewards/accuracies": 0.9375, |
|
"rewards/chosen": 0.19313779473304749, |
|
"rewards/margins": 0.1793561577796936, |
|
"rewards/rejected": 0.013781649991869926, |
|
"step": 64 |
|
}, |
|
{ |
|
"epoch": 4.32, |
|
"grad_norm": 7.851288795471191, |
|
"learning_rate": 4.99647455629649e-07, |
|
"logits/chosen": -2.744528293609619, |
|
"logits/rejected": -2.7117881774902344, |
|
"logps/chosen": -136.25863647460938, |
|
"logps/rejected": -95.11305236816406, |
|
"loss": 0.5968, |
|
"rewards/accuracies": 0.96875, |
|
"rewards/chosen": 0.21463464200496674, |
|
"rewards/margins": 0.20793242752552032, |
|
"rewards/rejected": 0.0067022331058979034, |
|
"step": 68 |
|
}, |
|
{ |
|
"epoch": 4.57, |
|
"grad_norm": 9.278959274291992, |
|
"learning_rate": 4.995849118819353e-07, |
|
"logits/chosen": -2.7611894607543945, |
|
"logits/rejected": -2.7447633743286133, |
|
"logps/chosen": -134.3462677001953, |
|
"logps/rejected": -103.75216674804688, |
|
"loss": 0.5828, |
|
"rewards/accuracies": 0.984375, |
|
"rewards/chosen": 0.25433310866355896, |
|
"rewards/margins": 0.24298511445522308, |
|
"rewards/rejected": 0.011347985826432705, |
|
"step": 72 |
|
}, |
|
{ |
|
"epoch": 4.83, |
|
"grad_norm": 7.795536994934082, |
|
"learning_rate": 4.995172683978719e-07, |
|
"logits/chosen": -2.698429584503174, |
|
"logits/rejected": -2.7458348274230957, |
|
"logps/chosen": -127.65770721435547, |
|
"logps/rejected": -104.23495483398438, |
|
"loss": 0.5796, |
|
"rewards/accuracies": 0.9375, |
|
"rewards/chosen": 0.25434932112693787, |
|
"rewards/margins": 0.25114190578460693, |
|
"rewards/rejected": 0.0032073920592665672, |
|
"step": 76 |
|
}, |
|
{ |
|
"epoch": 5.08, |
|
"grad_norm": 7.788575649261475, |
|
"learning_rate": 4.994445265596091e-07, |
|
"logits/chosen": -2.6734955310821533, |
|
"logits/rejected": -2.7292165756225586, |
|
"logps/chosen": -123.53501892089844, |
|
"logps/rejected": -95.23420715332031, |
|
"loss": 0.5676, |
|
"rewards/accuracies": 0.96875, |
|
"rewards/chosen": 0.2950035631656647, |
|
"rewards/margins": 0.2805778682231903, |
|
"rewards/rejected": 0.014425676316022873, |
|
"step": 80 |
|
}, |
|
{ |
|
"epoch": 5.08, |
|
"eval_logits/chosen": -2.8074097633361816, |
|
"eval_logits/rejected": -2.890533685684204, |
|
"eval_logps/chosen": -131.55104064941406, |
|
"eval_logps/rejected": -97.56663513183594, |
|
"eval_loss": 0.6045485734939575, |
|
"eval_rewards/accuracies": 0.7857142686843872, |
|
"eval_rewards/chosen": 0.2518289387226105, |
|
"eval_rewards/margins": 0.1976027488708496, |
|
"eval_rewards/rejected": 0.05422618240118027, |
|
"eval_runtime": 19.3586, |
|
"eval_samples_per_second": 1.446, |
|
"eval_steps_per_second": 1.446, |
|
"step": 80 |
|
}, |
|
{ |
|
"epoch": 5.33, |
|
"grad_norm": 6.709670543670654, |
|
"learning_rate": 4.993666878534718e-07, |
|
"logits/chosen": -2.736870527267456, |
|
"logits/rejected": -2.778140068054199, |
|
"logps/chosen": -128.86972045898438, |
|
"logps/rejected": -104.70977020263672, |
|
"loss": 0.5637, |
|
"rewards/accuracies": 0.953125, |
|
"rewards/chosen": 0.31255263090133667, |
|
"rewards/margins": 0.2908375561237335, |
|
"rewards/rejected": 0.021715058013796806, |
|
"step": 84 |
|
}, |
|
{ |
|
"epoch": 5.59, |
|
"grad_norm": 8.178019523620605, |
|
"learning_rate": 4.99283753869928e-07, |
|
"logits/chosen": -2.686039447784424, |
|
"logits/rejected": -2.681716203689575, |
|
"logps/chosen": -129.52798461914062, |
|
"logps/rejected": -97.80433654785156, |
|
"loss": 0.5475, |
|
"rewards/accuracies": 1.0, |
|
"rewards/chosen": 0.32084640860557556, |
|
"rewards/margins": 0.33009591698646545, |
|
"rewards/rejected": -0.00924946740269661, |
|
"step": 88 |
|
}, |
|
{ |
|
"epoch": 5.84, |
|
"grad_norm": 7.515178680419922, |
|
"learning_rate": 4.991957263035573e-07, |
|
"logits/chosen": -2.763765811920166, |
|
"logits/rejected": -2.7660324573516846, |
|
"logps/chosen": -128.16851806640625, |
|
"logps/rejected": -95.72579956054688, |
|
"loss": 0.5493, |
|
"rewards/accuracies": 0.96875, |
|
"rewards/chosen": 0.3080848455429077, |
|
"rewards/margins": 0.32350555062294006, |
|
"rewards/rejected": -0.015420722775161266, |
|
"step": 92 |
|
}, |
|
{ |
|
"epoch": 6.1, |
|
"grad_norm": 8.599517822265625, |
|
"learning_rate": 4.991026069530156e-07, |
|
"logits/chosen": -2.7107338905334473, |
|
"logits/rejected": -2.745981216430664, |
|
"logps/chosen": -131.5907745361328, |
|
"logps/rejected": -102.65729522705078, |
|
"loss": 0.5568, |
|
"rewards/accuracies": 0.9375, |
|
"rewards/chosen": 0.36698901653289795, |
|
"rewards/margins": 0.3141450881958008, |
|
"rewards/rejected": 0.05284389108419418, |
|
"step": 96 |
|
}, |
|
{ |
|
"epoch": 6.35, |
|
"grad_norm": 7.087887287139893, |
|
"learning_rate": 4.990043977209984e-07, |
|
"logits/chosen": -2.75876522064209, |
|
"logits/rejected": -2.800281047821045, |
|
"logps/chosen": -124.87574005126953, |
|
"logps/rejected": -99.60267639160156, |
|
"loss": 0.5332, |
|
"rewards/accuracies": 0.96875, |
|
"rewards/chosen": 0.3760470151901245, |
|
"rewards/margins": 0.37043848633766174, |
|
"rewards/rejected": 0.00560858054086566, |
|
"step": 100 |
|
}, |
|
{ |
|
"epoch": 6.6, |
|
"grad_norm": 6.458583831787109, |
|
"learning_rate": 4.989011006142024e-07, |
|
"logits/chosen": -2.70744252204895, |
|
"logits/rejected": -2.7285239696502686, |
|
"logps/chosen": -129.41262817382812, |
|
"logps/rejected": -98.60986328125, |
|
"loss": 0.5286, |
|
"rewards/accuracies": 0.984375, |
|
"rewards/chosen": 0.3909401297569275, |
|
"rewards/margins": 0.3845981955528259, |
|
"rewards/rejected": 0.006341893225908279, |
|
"step": 104 |
|
}, |
|
{ |
|
"epoch": 6.86, |
|
"grad_norm": 8.084393501281738, |
|
"learning_rate": 4.987927177432842e-07, |
|
"logits/chosen": -2.731127977371216, |
|
"logits/rejected": -2.7352988719940186, |
|
"logps/chosen": -123.85149383544922, |
|
"logps/rejected": -97.99423217773438, |
|
"loss": 0.544, |
|
"rewards/accuracies": 0.890625, |
|
"rewards/chosen": 0.38825392723083496, |
|
"rewards/margins": 0.34452566504478455, |
|
"rewards/rejected": 0.043728262186050415, |
|
"step": 108 |
|
}, |
|
{ |
|
"epoch": 7.11, |
|
"grad_norm": 7.936505317687988, |
|
"learning_rate": 4.987080943856886e-07, |
|
"logits/chosen": -2.6931982040405273, |
|
"logits/rejected": -2.683288812637329, |
|
"logps/chosen": -130.97227478027344, |
|
"logps/rejected": -93.71176147460938, |
|
"loss": 0.4903, |
|
"rewards/accuracies": 0.953125, |
|
"rewards/chosen": 0.45238468050956726, |
|
"rewards/margins": 0.48204928636550903, |
|
"rewards/rejected": -0.029664600268006325, |
|
"step": 112 |
|
}, |
|
{ |
|
"epoch": 7.37, |
|
"grad_norm": 6.827272415161133, |
|
"learning_rate": 4.985908168188602e-07, |
|
"logits/chosen": -2.697653293609619, |
|
"logits/rejected": -2.7467117309570312, |
|
"logps/chosen": -119.16637420654297, |
|
"logps/rejected": -95.0647201538086, |
|
"loss": 0.5088, |
|
"rewards/accuracies": 0.953125, |
|
"rewards/chosen": 0.4183903634548187, |
|
"rewards/margins": 0.43436184525489807, |
|
"rewards/rejected": -0.015971507877111435, |
|
"step": 116 |
|
}, |
|
{ |
|
"epoch": 7.62, |
|
"grad_norm": 6.599144458770752, |
|
"learning_rate": 4.984684598278982e-07, |
|
"logits/chosen": -2.7136449813842773, |
|
"logits/rejected": -2.7430026531219482, |
|
"logps/chosen": -126.032958984375, |
|
"logps/rejected": -102.69947814941406, |
|
"loss": 0.5231, |
|
"rewards/accuracies": 0.9375, |
|
"rewards/chosen": 0.4575308859348297, |
|
"rewards/margins": 0.40544256567955017, |
|
"rewards/rejected": 0.052088312804698944, |
|
"step": 120 |
|
}, |
|
{ |
|
"epoch": 7.62, |
|
"eval_logits/chosen": -2.807668447494507, |
|
"eval_logits/rejected": -2.8901402950286865, |
|
"eval_logps/chosen": -129.97122192382812, |
|
"eval_logps/rejected": -97.28166961669922, |
|
"eval_loss": 0.5570675134658813, |
|
"eval_rewards/accuracies": 0.7857142686843872, |
|
"eval_rewards/chosen": 0.4098115563392639, |
|
"eval_rewards/margins": 0.3270883560180664, |
|
"eval_rewards/rejected": 0.0827232226729393, |
|
"eval_runtime": 19.4265, |
|
"eval_samples_per_second": 1.441, |
|
"eval_steps_per_second": 1.441, |
|
"step": 120 |
|
}, |
|
{ |
|
"epoch": 7.87, |
|
"grad_norm": 7.769857883453369, |
|
"learning_rate": 4.983410259129073e-07, |
|
"logits/chosen": -2.7467093467712402, |
|
"logits/rejected": -2.7454986572265625, |
|
"logps/chosen": -134.60739135742188, |
|
"logps/rejected": -103.20536804199219, |
|
"loss": 0.4995, |
|
"rewards/accuracies": 0.984375, |
|
"rewards/chosen": 0.4985056221485138, |
|
"rewards/margins": 0.4699307978153229, |
|
"rewards/rejected": 0.028574790805578232, |
|
"step": 124 |
|
}, |
|
{ |
|
"epoch": 8.13, |
|
"grad_norm": 7.224113941192627, |
|
"learning_rate": 4.982085176777285e-07, |
|
"logits/chosen": -2.725543975830078, |
|
"logits/rejected": -2.7536404132843018, |
|
"logps/chosen": -121.35929870605469, |
|
"logps/rejected": -97.49042510986328, |
|
"loss": 0.4988, |
|
"rewards/accuracies": 0.921875, |
|
"rewards/chosen": 0.4724448025226593, |
|
"rewards/margins": 0.4719679057598114, |
|
"rewards/rejected": 0.0004768962971866131, |
|
"step": 128 |
|
}, |
|
{ |
|
"epoch": 8.38, |
|
"grad_norm": 5.916596412658691, |
|
"learning_rate": 4.980709378298851e-07, |
|
"logits/chosen": -2.7414817810058594, |
|
"logits/rejected": -2.7378463745117188, |
|
"logps/chosen": -119.76469421386719, |
|
"logps/rejected": -92.40216827392578, |
|
"loss": 0.4946, |
|
"rewards/accuracies": 0.984375, |
|
"rewards/chosen": 0.48038381338119507, |
|
"rewards/margins": 0.4809207320213318, |
|
"rewards/rejected": -0.0005369335412979126, |
|
"step": 132 |
|
}, |
|
{ |
|
"epoch": 8.63, |
|
"grad_norm": 6.414724826812744, |
|
"learning_rate": 4.979282891805286e-07, |
|
"logits/chosen": -2.7029452323913574, |
|
"logits/rejected": -2.737753391265869, |
|
"logps/chosen": -130.59829711914062, |
|
"logps/rejected": -99.5873794555664, |
|
"loss": 0.4564, |
|
"rewards/accuracies": 0.96875, |
|
"rewards/chosen": 0.5609064698219299, |
|
"rewards/margins": 0.5927521586418152, |
|
"rewards/rejected": -0.03184571862220764, |
|
"step": 136 |
|
}, |
|
{ |
|
"epoch": 8.89, |
|
"grad_norm": 7.006284236907959, |
|
"learning_rate": 4.977805746443806e-07, |
|
"logits/chosen": -2.719618320465088, |
|
"logits/rejected": -2.7250468730926514, |
|
"logps/chosen": -128.04678344726562, |
|
"logps/rejected": -101.81006622314453, |
|
"loss": 0.4761, |
|
"rewards/accuracies": 0.96875, |
|
"rewards/chosen": 0.5477940440177917, |
|
"rewards/margins": 0.5303957462310791, |
|
"rewards/rejected": 0.017398254945874214, |
|
"step": 140 |
|
}, |
|
{ |
|
"epoch": 9.14, |
|
"grad_norm": 7.040762901306152, |
|
"learning_rate": 4.97627797239673e-07, |
|
"logits/chosen": -2.6895217895507812, |
|
"logits/rejected": -2.7162435054779053, |
|
"logps/chosen": -133.3381805419922, |
|
"logps/rejected": -102.25975799560547, |
|
"loss": 0.4403, |
|
"rewards/accuracies": 0.953125, |
|
"rewards/chosen": 0.6185128688812256, |
|
"rewards/margins": 0.6498669981956482, |
|
"rewards/rejected": -0.03135409206151962, |
|
"step": 144 |
|
}, |
|
{ |
|
"epoch": 9.4, |
|
"grad_norm": 6.93245267868042, |
|
"learning_rate": 4.974699600880869e-07, |
|
"logits/chosen": -2.6911263465881348, |
|
"logits/rejected": -2.7075552940368652, |
|
"logps/chosen": -126.95785522460938, |
|
"logps/rejected": -102.03314971923828, |
|
"loss": 0.4587, |
|
"rewards/accuracies": 0.96875, |
|
"rewards/chosen": 0.573100745677948, |
|
"rewards/margins": 0.5804403424263, |
|
"rewards/rejected": -0.0073395660147070885, |
|
"step": 148 |
|
}, |
|
{ |
|
"epoch": 9.65, |
|
"grad_norm": 7.084275245666504, |
|
"learning_rate": 4.973070664146885e-07, |
|
"logits/chosen": -2.7053210735321045, |
|
"logits/rejected": -2.7612156867980957, |
|
"logps/chosen": -119.92796325683594, |
|
"logps/rejected": -97.49836730957031, |
|
"loss": 0.483, |
|
"rewards/accuracies": 0.96875, |
|
"rewards/chosen": 0.5761281251907349, |
|
"rewards/margins": 0.5190825462341309, |
|
"rewards/rejected": 0.057045597583055496, |
|
"step": 152 |
|
}, |
|
{ |
|
"epoch": 9.9, |
|
"grad_norm": 5.3888258934021, |
|
"learning_rate": 4.971391195478631e-07, |
|
"logits/chosen": -2.750107765197754, |
|
"logits/rejected": -2.7442452907562256, |
|
"logps/chosen": -128.85630798339844, |
|
"logps/rejected": -96.89845275878906, |
|
"loss": 0.4517, |
|
"rewards/accuracies": 0.9375, |
|
"rewards/chosen": 0.6420415043830872, |
|
"rewards/margins": 0.6197397708892822, |
|
"rewards/rejected": 0.022301698103547096, |
|
"step": 156 |
|
}, |
|
{ |
|
"epoch": 10.16, |
|
"grad_norm": 6.74313497543335, |
|
"learning_rate": 4.969661229192477e-07, |
|
"logits/chosen": -2.7252540588378906, |
|
"logits/rejected": -2.709249258041382, |
|
"logps/chosen": -126.21170043945312, |
|
"logps/rejected": -105.25879669189453, |
|
"loss": 0.4607, |
|
"rewards/accuracies": 0.984375, |
|
"rewards/chosen": 0.6133719682693481, |
|
"rewards/margins": 0.6076129674911499, |
|
"rewards/rejected": 0.005758981220424175, |
|
"step": 160 |
|
}, |
|
{ |
|
"epoch": 10.16, |
|
"eval_logits/chosen": -2.807128429412842, |
|
"eval_logits/rejected": -2.8889076709747314, |
|
"eval_logps/chosen": -128.48606872558594, |
|
"eval_logps/rejected": -97.06517028808594, |
|
"eval_loss": 0.5175719857215881, |
|
"eval_rewards/accuracies": 0.8214285969734192, |
|
"eval_rewards/chosen": 0.5583271384239197, |
|
"eval_rewards/margins": 0.4539553225040436, |
|
"eval_rewards/rejected": 0.10437185317277908, |
|
"eval_runtime": 19.3848, |
|
"eval_samples_per_second": 1.444, |
|
"eval_steps_per_second": 1.444, |
|
"step": 160 |
|
}, |
|
{ |
|
"epoch": 10.41, |
|
"grad_norm": 5.443469524383545, |
|
"learning_rate": 4.967880800636598e-07, |
|
"logits/chosen": -2.7496747970581055, |
|
"logits/rejected": -2.768813133239746, |
|
"logps/chosen": -122.11544799804688, |
|
"logps/rejected": -94.54540252685547, |
|
"loss": 0.436, |
|
"rewards/accuracies": 0.96875, |
|
"rewards/chosen": 0.6616504788398743, |
|
"rewards/margins": 0.67127525806427, |
|
"rewards/rejected": -0.009624744765460491, |
|
"step": 164 |
|
}, |
|
{ |
|
"epoch": 10.67, |
|
"grad_norm": 5.551913738250732, |
|
"learning_rate": 4.966049946190263e-07, |
|
"logits/chosen": -2.7052814960479736, |
|
"logits/rejected": -2.7216036319732666, |
|
"logps/chosen": -131.8173065185547, |
|
"logps/rejected": -102.79641723632812, |
|
"loss": 0.4326, |
|
"rewards/accuracies": 0.953125, |
|
"rewards/chosen": 0.7013098001480103, |
|
"rewards/margins": 0.6718720197677612, |
|
"rewards/rejected": 0.02943783439695835, |
|
"step": 168 |
|
}, |
|
{ |
|
"epoch": 10.92, |
|
"grad_norm": 5.826549530029297, |
|
"learning_rate": 4.964168703263086e-07, |
|
"logits/chosen": -2.701007604598999, |
|
"logits/rejected": -2.741464614868164, |
|
"logps/chosen": -116.08711242675781, |
|
"logps/rejected": -94.73246765136719, |
|
"loss": 0.446, |
|
"rewards/accuracies": 0.953125, |
|
"rewards/chosen": 0.6390998363494873, |
|
"rewards/margins": 0.6307864785194397, |
|
"rewards/rejected": 0.008313396014273167, |
|
"step": 172 |
|
}, |
|
{ |
|
"epoch": 11.17, |
|
"grad_norm": 6.066051483154297, |
|
"learning_rate": 4.962237110294259e-07, |
|
"logits/chosen": -2.7041149139404297, |
|
"logits/rejected": -2.731684684753418, |
|
"logps/chosen": -124.84115600585938, |
|
"logps/rejected": -103.8166732788086, |
|
"loss": 0.4294, |
|
"rewards/accuracies": 0.953125, |
|
"rewards/chosen": 0.6835889220237732, |
|
"rewards/margins": 0.6921644806861877, |
|
"rewards/rejected": -0.00857558287680149, |
|
"step": 176 |
|
}, |
|
{ |
|
"epoch": 11.43, |
|
"grad_norm": 5.8145012855529785, |
|
"learning_rate": 4.960255206751773e-07, |
|
"logits/chosen": -2.7352046966552734, |
|
"logits/rejected": -2.7856600284576416, |
|
"logps/chosen": -123.47644805908203, |
|
"logps/rejected": -93.96688079833984, |
|
"loss": 0.4056, |
|
"rewards/accuracies": 0.96875, |
|
"rewards/chosen": 0.7594637274742126, |
|
"rewards/margins": 0.7790957093238831, |
|
"rewards/rejected": -0.019631966948509216, |
|
"step": 180 |
|
}, |
|
{ |
|
"epoch": 11.68, |
|
"grad_norm": 5.962335109710693, |
|
"learning_rate": 4.958223033131609e-07, |
|
"logits/chosen": -2.740914821624756, |
|
"logits/rejected": -2.7585391998291016, |
|
"logps/chosen": -124.35005950927734, |
|
"logps/rejected": -105.05398559570312, |
|
"loss": 0.4293, |
|
"rewards/accuracies": 0.984375, |
|
"rewards/chosen": 0.6812430620193481, |
|
"rewards/margins": 0.6962765455245972, |
|
"rewards/rejected": -0.015033497475087643, |
|
"step": 184 |
|
}, |
|
{ |
|
"epoch": 11.94, |
|
"grad_norm": 5.529379367828369, |
|
"learning_rate": 4.956140630956907e-07, |
|
"logits/chosen": -2.7024734020233154, |
|
"logits/rejected": -2.6746203899383545, |
|
"logps/chosen": -125.29593658447266, |
|
"logps/rejected": -94.42231750488281, |
|
"loss": 0.3963, |
|
"rewards/accuracies": 0.984375, |
|
"rewards/chosen": 0.7892162799835205, |
|
"rewards/margins": 0.8050926327705383, |
|
"rewards/rejected": -0.01587628945708275, |
|
"step": 188 |
|
}, |
|
{ |
|
"epoch": 12.19, |
|
"grad_norm": 5.774294376373291, |
|
"learning_rate": 4.954008042777125e-07, |
|
"logits/chosen": -2.7662813663482666, |
|
"logits/rejected": -2.8167452812194824, |
|
"logps/chosen": -124.10614776611328, |
|
"logps/rejected": -94.9889907836914, |
|
"loss": 0.3728, |
|
"rewards/accuracies": 0.96875, |
|
"rewards/chosen": 0.8182777166366577, |
|
"rewards/margins": 0.8937400579452515, |
|
"rewards/rejected": -0.07546230405569077, |
|
"step": 192 |
|
}, |
|
{ |
|
"epoch": 12.44, |
|
"grad_norm": 4.990906715393066, |
|
"learning_rate": 4.951825312167159e-07, |
|
"logits/chosen": -2.7352771759033203, |
|
"logits/rejected": -2.7206149101257324, |
|
"logps/chosen": -116.98539733886719, |
|
"logps/rejected": -96.68843078613281, |
|
"loss": 0.4435, |
|
"rewards/accuracies": 0.9375, |
|
"rewards/chosen": 0.7208267450332642, |
|
"rewards/margins": 0.6662319302558899, |
|
"rewards/rejected": 0.05459484085440636, |
|
"step": 196 |
|
}, |
|
{ |
|
"epoch": 12.7, |
|
"grad_norm": 7.139024257659912, |
|
"learning_rate": 4.949592483726464e-07, |
|
"logits/chosen": -2.728516101837158, |
|
"logits/rejected": -2.732177257537842, |
|
"logps/chosen": -125.51838684082031, |
|
"logps/rejected": -103.7790298461914, |
|
"loss": 0.391, |
|
"rewards/accuracies": 0.984375, |
|
"rewards/chosen": 0.7955451011657715, |
|
"rewards/margins": 0.8273122310638428, |
|
"rewards/rejected": -0.03176717832684517, |
|
"step": 200 |
|
}, |
|
{ |
|
"epoch": 12.7, |
|
"eval_logits/chosen": -2.8053383827209473, |
|
"eval_logits/rejected": -2.8861937522888184, |
|
"eval_logps/chosen": -127.08057403564453, |
|
"eval_logps/rejected": -96.8802261352539, |
|
"eval_loss": 0.4858916103839874, |
|
"eval_rewards/accuracies": 0.8214285969734192, |
|
"eval_rewards/chosen": 0.6988765597343445, |
|
"eval_rewards/margins": 0.5760089755058289, |
|
"eval_rewards/rejected": 0.12286762148141861, |
|
"eval_runtime": 19.4173, |
|
"eval_samples_per_second": 1.442, |
|
"eval_steps_per_second": 1.442, |
|
"step": 200 |
|
}, |
|
{ |
|
"epoch": 12.95, |
|
"grad_norm": 5.450782299041748, |
|
"learning_rate": 4.947309603078138e-07, |
|
"logits/chosen": -2.6719353199005127, |
|
"logits/rejected": -2.7134647369384766, |
|
"logps/chosen": -124.22660827636719, |
|
"logps/rejected": -99.73461151123047, |
|
"loss": 0.4062, |
|
"rewards/accuracies": 0.96875, |
|
"rewards/chosen": 0.7935754060745239, |
|
"rewards/margins": 0.783437192440033, |
|
"rewards/rejected": 0.010138224810361862, |
|
"step": 204 |
|
}, |
|
{ |
|
"epoch": 13.21, |
|
"grad_norm": 5.752992153167725, |
|
"learning_rate": 4.944976716867984e-07, |
|
"logits/chosen": -2.7026755809783936, |
|
"logits/rejected": -2.698920726776123, |
|
"logps/chosen": -123.63475036621094, |
|
"logps/rejected": -107.06332397460938, |
|
"loss": 0.3952, |
|
"rewards/accuracies": 0.96875, |
|
"rewards/chosen": 0.8116875886917114, |
|
"rewards/margins": 0.8335397839546204, |
|
"rewards/rejected": -0.02185220457613468, |
|
"step": 208 |
|
}, |
|
{ |
|
"epoch": 13.46, |
|
"grad_norm": 5.456747531890869, |
|
"learning_rate": 4.942593872763566e-07, |
|
"logits/chosen": -2.701968193054199, |
|
"logits/rejected": -2.786785840988159, |
|
"logps/chosen": -124.91243743896484, |
|
"logps/rejected": -101.95451354980469, |
|
"loss": 0.3673, |
|
"rewards/accuracies": 0.96875, |
|
"rewards/chosen": 0.8831655979156494, |
|
"rewards/margins": 0.9259040951728821, |
|
"rewards/rejected": -0.04273851588368416, |
|
"step": 212 |
|
}, |
|
{ |
|
"epoch": 13.71, |
|
"grad_norm": 4.753533363342285, |
|
"learning_rate": 4.940161119453231e-07, |
|
"logits/chosen": -2.7109193801879883, |
|
"logits/rejected": -2.6961448192596436, |
|
"logps/chosen": -123.21333312988281, |
|
"logps/rejected": -92.74653625488281, |
|
"loss": 0.3832, |
|
"rewards/accuracies": 0.984375, |
|
"rewards/chosen": 0.8407853245735168, |
|
"rewards/margins": 0.8698095083236694, |
|
"rewards/rejected": -0.02902432158589363, |
|
"step": 216 |
|
}, |
|
{ |
|
"epoch": 13.97, |
|
"grad_norm": 5.374657154083252, |
|
"learning_rate": 4.937678506645115e-07, |
|
"logits/chosen": -2.7288832664489746, |
|
"logits/rejected": -2.7349352836608887, |
|
"logps/chosen": -123.1498031616211, |
|
"logps/rejected": -95.77909851074219, |
|
"loss": 0.3644, |
|
"rewards/accuracies": 0.984375, |
|
"rewards/chosen": 0.8925550580024719, |
|
"rewards/margins": 0.9183064699172974, |
|
"rewards/rejected": -0.025751333683729172, |
|
"step": 220 |
|
}, |
|
{ |
|
"epoch": 14.22, |
|
"grad_norm": 5.842701435089111, |
|
"learning_rate": 4.935146085066125e-07, |
|
"logits/chosen": -2.7030577659606934, |
|
"logits/rejected": -2.7200875282287598, |
|
"logps/chosen": -120.81951904296875, |
|
"logps/rejected": -100.53494262695312, |
|
"loss": 0.3581, |
|
"rewards/accuracies": 0.96875, |
|
"rewards/chosen": 0.9156714677810669, |
|
"rewards/margins": 0.9704564809799194, |
|
"rewards/rejected": -0.05478499457240105, |
|
"step": 224 |
|
}, |
|
{ |
|
"epoch": 14.48, |
|
"grad_norm": 5.020945072174072, |
|
"learning_rate": 4.932563906460904e-07, |
|
"logits/chosen": -2.722069501876831, |
|
"logits/rejected": -2.731358051300049, |
|
"logps/chosen": -123.65387725830078, |
|
"logps/rejected": -103.83390808105469, |
|
"loss": 0.391, |
|
"rewards/accuracies": 0.96875, |
|
"rewards/chosen": 0.8787897229194641, |
|
"rewards/margins": 0.8743190765380859, |
|
"rewards/rejected": 0.004470685496926308, |
|
"step": 228 |
|
}, |
|
{ |
|
"epoch": 14.73, |
|
"grad_norm": 5.041106700897217, |
|
"learning_rate": 4.929932023590776e-07, |
|
"logits/chosen": -2.735006093978882, |
|
"logits/rejected": -2.7546191215515137, |
|
"logps/chosen": -124.65998840332031, |
|
"logps/rejected": -101.49640655517578, |
|
"loss": 0.3586, |
|
"rewards/accuracies": 0.96875, |
|
"rewards/chosen": 0.9206160306930542, |
|
"rewards/margins": 0.9452594518661499, |
|
"rewards/rejected": -0.024643439799547195, |
|
"step": 232 |
|
}, |
|
{ |
|
"epoch": 14.98, |
|
"grad_norm": 5.318466663360596, |
|
"learning_rate": 4.927250490232664e-07, |
|
"logits/chosen": -2.6883015632629395, |
|
"logits/rejected": -2.718212842941284, |
|
"logps/chosen": -120.88870239257812, |
|
"logps/rejected": -94.79769134521484, |
|
"loss": 0.3564, |
|
"rewards/accuracies": 0.96875, |
|
"rewards/chosen": 0.9339860677719116, |
|
"rewards/margins": 0.9901734590530396, |
|
"rewards/rejected": -0.05618742108345032, |
|
"step": 236 |
|
}, |
|
{ |
|
"epoch": 15.24, |
|
"grad_norm": 5.438621997833252, |
|
"learning_rate": 4.924519361177992e-07, |
|
"logits/chosen": -2.7316794395446777, |
|
"logits/rejected": -2.812800407409668, |
|
"logps/chosen": -120.06542205810547, |
|
"logps/rejected": -97.61270141601562, |
|
"loss": 0.3471, |
|
"rewards/accuracies": 0.96875, |
|
"rewards/chosen": 0.9851706624031067, |
|
"rewards/margins": 1.039989948272705, |
|
"rewards/rejected": -0.05481935292482376, |
|
"step": 240 |
|
}, |
|
{ |
|
"epoch": 15.24, |
|
"eval_logits/chosen": -2.8022568225860596, |
|
"eval_logits/rejected": -2.882326602935791, |
|
"eval_logps/chosen": -125.87542724609375, |
|
"eval_logps/rejected": -96.83832550048828, |
|
"eval_loss": 0.46029001474380493, |
|
"eval_rewards/accuracies": 0.7857142686843872, |
|
"eval_rewards/chosen": 0.8193913102149963, |
|
"eval_rewards/margins": 0.6923348307609558, |
|
"eval_rewards/rejected": 0.12705650925636292, |
|
"eval_runtime": 19.4065, |
|
"eval_samples_per_second": 1.443, |
|
"eval_steps_per_second": 1.443, |
|
"step": 240 |
|
}, |
|
{ |
|
"epoch": 15.49, |
|
"grad_norm": 5.301855087280273, |
|
"learning_rate": 4.921738692231572e-07, |
|
"logits/chosen": -2.661013126373291, |
|
"logits/rejected": -2.666978359222412, |
|
"logps/chosen": -121.69336700439453, |
|
"logps/rejected": -101.86541748046875, |
|
"loss": 0.3574, |
|
"rewards/accuracies": 0.984375, |
|
"rewards/chosen": 0.9682250022888184, |
|
"rewards/margins": 0.9734476208686829, |
|
"rewards/rejected": -0.0052225226536393166, |
|
"step": 244 |
|
}, |
|
{ |
|
"epoch": 15.75, |
|
"grad_norm": 4.478713512420654, |
|
"learning_rate": 4.918908540210451e-07, |
|
"logits/chosen": -2.702775239944458, |
|
"logits/rejected": -2.7098357677459717, |
|
"logps/chosen": -119.93941497802734, |
|
"logps/rejected": -94.76625061035156, |
|
"loss": 0.3517, |
|
"rewards/accuracies": 0.953125, |
|
"rewards/chosen": 0.8942417502403259, |
|
"rewards/margins": 0.9958410263061523, |
|
"rewards/rejected": -0.10159924626350403, |
|
"step": 248 |
|
}, |
|
{ |
|
"epoch": 16.0, |
|
"grad_norm": 4.398434162139893, |
|
"learning_rate": 4.916028962942762e-07, |
|
"logits/chosen": -2.7399990558624268, |
|
"logits/rejected": -2.7476425170898438, |
|
"logps/chosen": -124.2032470703125, |
|
"logps/rejected": -101.86943054199219, |
|
"loss": 0.3372, |
|
"rewards/accuracies": 0.984375, |
|
"rewards/chosen": 1.0369476079940796, |
|
"rewards/margins": 1.0664609670639038, |
|
"rewards/rejected": -0.029513416811823845, |
|
"step": 252 |
|
}, |
|
{ |
|
"epoch": 16.25, |
|
"grad_norm": 4.430797576904297, |
|
"learning_rate": 4.913100019266536e-07, |
|
"logits/chosen": -2.6950950622558594, |
|
"logits/rejected": -2.7181689739227295, |
|
"logps/chosen": -123.97085571289062, |
|
"logps/rejected": -97.85747528076172, |
|
"loss": 0.3226, |
|
"rewards/accuracies": 0.984375, |
|
"rewards/chosen": 1.0282835960388184, |
|
"rewards/margins": 1.1182292699813843, |
|
"rewards/rejected": -0.08994561433792114, |
|
"step": 256 |
|
}, |
|
{ |
|
"epoch": 16.51, |
|
"grad_norm": 6.383476734161377, |
|
"learning_rate": 4.910121769028503e-07, |
|
"logits/chosen": -2.7224864959716797, |
|
"logits/rejected": -2.727613925933838, |
|
"logps/chosen": -117.91517639160156, |
|
"logps/rejected": -99.52539825439453, |
|
"loss": 0.3658, |
|
"rewards/accuracies": 0.953125, |
|
"rewards/chosen": 0.9342272877693176, |
|
"rewards/margins": 0.9777376651763916, |
|
"rewards/rejected": -0.04351034015417099, |
|
"step": 260 |
|
}, |
|
{ |
|
"epoch": 16.76, |
|
"grad_norm": 4.453210353851318, |
|
"learning_rate": 4.907094273082864e-07, |
|
"logits/chosen": -2.7149100303649902, |
|
"logits/rejected": -2.710146188735962, |
|
"logps/chosen": -126.07495880126953, |
|
"logps/rejected": -100.1856689453125, |
|
"loss": 0.3162, |
|
"rewards/accuracies": 0.96875, |
|
"rewards/chosen": 1.1231064796447754, |
|
"rewards/margins": 1.175417423248291, |
|
"rewards/rejected": -0.05231098458170891, |
|
"step": 264 |
|
}, |
|
{ |
|
"epoch": 17.02, |
|
"grad_norm": 4.718712329864502, |
|
"learning_rate": 4.904017593290055e-07, |
|
"logits/chosen": -2.710063934326172, |
|
"logits/rejected": -2.7535319328308105, |
|
"logps/chosen": -117.01451873779297, |
|
"logps/rejected": -99.95758819580078, |
|
"loss": 0.322, |
|
"rewards/accuracies": 0.984375, |
|
"rewards/chosen": 1.033459186553955, |
|
"rewards/margins": 1.1137871742248535, |
|
"rewards/rejected": -0.08032786101102829, |
|
"step": 268 |
|
}, |
|
{ |
|
"epoch": 17.27, |
|
"grad_norm": 5.164140701293945, |
|
"learning_rate": 4.900891792515478e-07, |
|
"logits/chosen": -2.7342586517333984, |
|
"logits/rejected": -2.7561776638031006, |
|
"logps/chosen": -118.80335998535156, |
|
"logps/rejected": -104.33666229248047, |
|
"loss": 0.3314, |
|
"rewards/accuracies": 0.984375, |
|
"rewards/chosen": 1.078546404838562, |
|
"rewards/margins": 1.0902469158172607, |
|
"rewards/rejected": -0.011700518429279327, |
|
"step": 272 |
|
}, |
|
{ |
|
"epoch": 17.52, |
|
"grad_norm": 4.392764568328857, |
|
"learning_rate": 4.897716934628217e-07, |
|
"logits/chosen": -2.6796340942382812, |
|
"logits/rejected": -2.6666793823242188, |
|
"logps/chosen": -122.01671600341797, |
|
"logps/rejected": -97.74385070800781, |
|
"loss": 0.3047, |
|
"rewards/accuracies": 1.0, |
|
"rewards/chosen": 1.0594760179519653, |
|
"rewards/margins": 1.1904582977294922, |
|
"rewards/rejected": -0.1309822052717209, |
|
"step": 276 |
|
}, |
|
{ |
|
"epoch": 17.78, |
|
"grad_norm": 4.647542953491211, |
|
"learning_rate": 4.894493084499736e-07, |
|
"logits/chosen": -2.7203075885772705, |
|
"logits/rejected": -2.7299890518188477, |
|
"logps/chosen": -120.08210754394531, |
|
"logps/rejected": -98.8072738647461, |
|
"loss": 0.3496, |
|
"rewards/accuracies": 0.921875, |
|
"rewards/chosen": 1.0658477544784546, |
|
"rewards/margins": 1.065575122833252, |
|
"rewards/rejected": 0.0002726661041378975, |
|
"step": 280 |
|
}, |
|
{ |
|
"epoch": 17.78, |
|
"eval_logits/chosen": -2.798058271408081, |
|
"eval_logits/rejected": -2.8771989345550537, |
|
"eval_logps/chosen": -124.81001281738281, |
|
"eval_logps/rejected": -96.83968353271484, |
|
"eval_loss": 0.44096532464027405, |
|
"eval_rewards/accuracies": 0.7857142686843872, |
|
"eval_rewards/chosen": 0.9259332418441772, |
|
"eval_rewards/margins": 0.799013078212738, |
|
"eval_rewards/rejected": 0.12692023813724518, |
|
"eval_runtime": 19.3925, |
|
"eval_samples_per_second": 1.444, |
|
"eval_steps_per_second": 1.444, |
|
"step": 280 |
|
}, |
|
{ |
|
"epoch": 18.03, |
|
"grad_norm": 4.158952236175537, |
|
"learning_rate": 4.891220308002547e-07, |
|
"logits/chosen": -2.6795365810394287, |
|
"logits/rejected": -2.7270519733428955, |
|
"logps/chosen": -120.39765167236328, |
|
"logps/rejected": -100.02141571044922, |
|
"loss": 0.319, |
|
"rewards/accuracies": 0.96875, |
|
"rewards/chosen": 1.0584629774093628, |
|
"rewards/margins": 1.19419264793396, |
|
"rewards/rejected": -0.13572952151298523, |
|
"step": 284 |
|
}, |
|
{ |
|
"epoch": 18.29, |
|
"grad_norm": 4.605132579803467, |
|
"learning_rate": 4.887898672008871e-07, |
|
"logits/chosen": -2.696953296661377, |
|
"logits/rejected": -2.718316078186035, |
|
"logps/chosen": -120.32958984375, |
|
"logps/rejected": -101.76454162597656, |
|
"loss": 0.331, |
|
"rewards/accuracies": 0.953125, |
|
"rewards/chosen": 1.0935906171798706, |
|
"rewards/margins": 1.1072776317596436, |
|
"rewards/rejected": -0.013687020167708397, |
|
"step": 288 |
|
}, |
|
{ |
|
"epoch": 18.54, |
|
"grad_norm": 5.824638843536377, |
|
"learning_rate": 4.884528244389268e-07, |
|
"logits/chosen": -2.6889617443084717, |
|
"logits/rejected": -2.690692663192749, |
|
"logps/chosen": -120.37040710449219, |
|
"logps/rejected": -98.27632141113281, |
|
"loss": 0.3204, |
|
"rewards/accuracies": 0.953125, |
|
"rewards/chosen": 1.1049582958221436, |
|
"rewards/margins": 1.175376057624817, |
|
"rewards/rejected": -0.07041767239570618, |
|
"step": 292 |
|
}, |
|
{ |
|
"epoch": 18.79, |
|
"grad_norm": 5.431331634521484, |
|
"learning_rate": 4.881109094011253e-07, |
|
"logits/chosen": -2.740906238555908, |
|
"logits/rejected": -2.725914478302002, |
|
"logps/chosen": -124.35327911376953, |
|
"logps/rejected": -100.71537780761719, |
|
"loss": 0.2884, |
|
"rewards/accuracies": 1.0, |
|
"rewards/chosen": 1.2057325839996338, |
|
"rewards/margins": 1.3113868236541748, |
|
"rewards/rejected": -0.10565409809350967, |
|
"step": 296 |
|
}, |
|
{ |
|
"epoch": 19.05, |
|
"grad_norm": 4.33558988571167, |
|
"learning_rate": 4.877641290737883e-07, |
|
"logits/chosen": -2.700420379638672, |
|
"logits/rejected": -2.7527971267700195, |
|
"logps/chosen": -117.01264953613281, |
|
"logps/rejected": -101.50834655761719, |
|
"loss": 0.2757, |
|
"rewards/accuracies": 1.0, |
|
"rewards/chosen": 1.150115728378296, |
|
"rewards/margins": 1.34402596950531, |
|
"rewards/rejected": -0.19391010701656342, |
|
"step": 300 |
|
}, |
|
{ |
|
"epoch": 19.3, |
|
"grad_norm": 4.987298488616943, |
|
"learning_rate": 4.874124905426338e-07, |
|
"logits/chosen": -2.715477705001831, |
|
"logits/rejected": -2.760146141052246, |
|
"logps/chosen": -117.20336151123047, |
|
"logps/rejected": -94.89369201660156, |
|
"loss": 0.3244, |
|
"rewards/accuracies": 0.953125, |
|
"rewards/chosen": 1.107708215713501, |
|
"rewards/margins": 1.1569105386734009, |
|
"rewards/rejected": -0.04920227825641632, |
|
"step": 304 |
|
}, |
|
{ |
|
"epoch": 19.56, |
|
"grad_norm": 4.090747833251953, |
|
"learning_rate": 4.870560009926464e-07, |
|
"logits/chosen": -2.6785225868225098, |
|
"logits/rejected": -2.6984167098999023, |
|
"logps/chosen": -112.27914428710938, |
|
"logps/rejected": -94.59768676757812, |
|
"loss": 0.2833, |
|
"rewards/accuracies": 0.984375, |
|
"rewards/chosen": 1.2028326988220215, |
|
"rewards/margins": 1.334948182106018, |
|
"rewards/rejected": -0.13211557269096375, |
|
"step": 308 |
|
}, |
|
{ |
|
"epoch": 19.81, |
|
"grad_norm": 5.0767083168029785, |
|
"learning_rate": 4.866946677079314e-07, |
|
"logits/chosen": -2.712416410446167, |
|
"logits/rejected": -2.7296035289764404, |
|
"logps/chosen": -121.65861511230469, |
|
"logps/rejected": -106.11611938476562, |
|
"loss": 0.2893, |
|
"rewards/accuracies": 0.984375, |
|
"rewards/chosen": 1.1842504739761353, |
|
"rewards/margins": 1.3191744089126587, |
|
"rewards/rejected": -0.13492396473884583, |
|
"step": 312 |
|
}, |
|
{ |
|
"epoch": 20.06, |
|
"grad_norm": 4.1403326988220215, |
|
"learning_rate": 4.863284980715648e-07, |
|
"logits/chosen": -2.658827781677246, |
|
"logits/rejected": -2.631199836730957, |
|
"logps/chosen": -129.3588409423828, |
|
"logps/rejected": -103.25465393066406, |
|
"loss": 0.2678, |
|
"rewards/accuracies": 0.984375, |
|
"rewards/chosen": 1.291969895362854, |
|
"rewards/margins": 1.4036282300949097, |
|
"rewards/rejected": -0.11165820807218552, |
|
"step": 316 |
|
}, |
|
{ |
|
"epoch": 20.32, |
|
"grad_norm": 4.350497245788574, |
|
"learning_rate": 4.859574995654441e-07, |
|
"logits/chosen": -2.761063814163208, |
|
"logits/rejected": -2.7751922607421875, |
|
"logps/chosen": -114.76087951660156, |
|
"logps/rejected": -100.04124450683594, |
|
"loss": 0.298, |
|
"rewards/accuracies": 0.984375, |
|
"rewards/chosen": 1.1642746925354004, |
|
"rewards/margins": 1.3020979166030884, |
|
"rewards/rejected": -0.13782335817813873, |
|
"step": 320 |
|
}, |
|
{ |
|
"epoch": 20.32, |
|
"eval_logits/chosen": -2.792503595352173, |
|
"eval_logits/rejected": -2.870570421218872, |
|
"eval_logps/chosen": -123.9360122680664, |
|
"eval_logps/rejected": -96.95860290527344, |
|
"eval_loss": 0.4256473481655121, |
|
"eval_rewards/accuracies": 0.8214285969734192, |
|
"eval_rewards/chosen": 1.0133329629898071, |
|
"eval_rewards/margins": 0.8983038663864136, |
|
"eval_rewards/rejected": 0.11502902954816818, |
|
"eval_runtime": 19.3701, |
|
"eval_samples_per_second": 1.446, |
|
"eval_steps_per_second": 1.446, |
|
"step": 320 |
|
}, |
|
{ |
|
"epoch": 20.57, |
|
"grad_norm": 4.334249973297119, |
|
"learning_rate": 4.855816797701336e-07, |
|
"logits/chosen": -2.690117359161377, |
|
"logits/rejected": -2.7377476692199707, |
|
"logps/chosen": -116.01565551757812, |
|
"logps/rejected": -102.1231689453125, |
|
"loss": 0.2872, |
|
"rewards/accuracies": 0.953125, |
|
"rewards/chosen": 1.2394335269927979, |
|
"rewards/margins": 1.3586478233337402, |
|
"rewards/rejected": -0.11921422928571701, |
|
"step": 324 |
|
}, |
|
{ |
|
"epoch": 20.83, |
|
"grad_norm": 4.597221851348877, |
|
"learning_rate": 4.85201046364711e-07, |
|
"logits/chosen": -2.741600751876831, |
|
"logits/rejected": -2.712954044342041, |
|
"logps/chosen": -124.32752990722656, |
|
"logps/rejected": -101.2073974609375, |
|
"loss": 0.2676, |
|
"rewards/accuracies": 0.984375, |
|
"rewards/chosen": 1.22231924533844, |
|
"rewards/margins": 1.4293614625930786, |
|
"rewards/rejected": -0.20704227685928345, |
|
"step": 328 |
|
}, |
|
{ |
|
"epoch": 21.08, |
|
"grad_norm": 3.682258367538452, |
|
"learning_rate": 4.848156071266095e-07, |
|
"logits/chosen": -2.6585566997528076, |
|
"logits/rejected": -2.704587459564209, |
|
"logps/chosen": -120.91087341308594, |
|
"logps/rejected": -98.83493041992188, |
|
"loss": 0.2705, |
|
"rewards/accuracies": 1.0, |
|
"rewards/chosen": 1.2277172803878784, |
|
"rewards/margins": 1.3790152072906494, |
|
"rewards/rejected": -0.15129785239696503, |
|
"step": 332 |
|
}, |
|
{ |
|
"epoch": 21.33, |
|
"grad_norm": 3.7775039672851562, |
|
"learning_rate": 4.844253699314596e-07, |
|
"logits/chosen": -2.7104837894439697, |
|
"logits/rejected": -2.7378451824188232, |
|
"logps/chosen": -117.22598266601562, |
|
"logps/rejected": -97.59563446044922, |
|
"loss": 0.2755, |
|
"rewards/accuracies": 1.0, |
|
"rewards/chosen": 1.2548604011535645, |
|
"rewards/margins": 1.3694878816604614, |
|
"rewards/rejected": -0.11462761461734772, |
|
"step": 336 |
|
}, |
|
{ |
|
"epoch": 21.59, |
|
"grad_norm": 4.84611701965332, |
|
"learning_rate": 4.840303427529273e-07, |
|
"logits/chosen": -2.6837968826293945, |
|
"logits/rejected": -2.7071595191955566, |
|
"logps/chosen": -121.88581085205078, |
|
"logps/rejected": -102.45454406738281, |
|
"loss": 0.286, |
|
"rewards/accuracies": 0.96875, |
|
"rewards/chosen": 1.2159500122070312, |
|
"rewards/margins": 1.3315218687057495, |
|
"rewards/rejected": -0.11557181179523468, |
|
"step": 340 |
|
}, |
|
{ |
|
"epoch": 21.84, |
|
"grad_norm": 3.760068655014038, |
|
"learning_rate": 4.836305336625523e-07, |
|
"logits/chosen": -2.709066390991211, |
|
"logits/rejected": -2.73175048828125, |
|
"logps/chosen": -117.1901626586914, |
|
"logps/rejected": -105.62377166748047, |
|
"loss": 0.2661, |
|
"rewards/accuracies": 0.984375, |
|
"rewards/chosen": 1.3225661516189575, |
|
"rewards/margins": 1.445369005203247, |
|
"rewards/rejected": -0.12280277907848358, |
|
"step": 344 |
|
}, |
|
{ |
|
"epoch": 22.1, |
|
"grad_norm": 3.8488616943359375, |
|
"learning_rate": 4.832259508295822e-07, |
|
"logits/chosen": -2.7598538398742676, |
|
"logits/rejected": -2.776313066482544, |
|
"logps/chosen": -117.15644836425781, |
|
"logps/rejected": -93.95286560058594, |
|
"loss": 0.2711, |
|
"rewards/accuracies": 0.953125, |
|
"rewards/chosen": 1.3003535270690918, |
|
"rewards/margins": 1.5021978616714478, |
|
"rewards/rejected": -0.20184442400932312, |
|
"step": 348 |
|
}, |
|
{ |
|
"epoch": 22.35, |
|
"grad_norm": 3.152104377746582, |
|
"learning_rate": 4.828166025208058e-07, |
|
"logits/chosen": -2.6503114700317383, |
|
"logits/rejected": -2.6512064933776855, |
|
"logps/chosen": -118.29179382324219, |
|
"logps/rejected": -97.83879852294922, |
|
"loss": 0.2394, |
|
"rewards/accuracies": 0.984375, |
|
"rewards/chosen": 1.2750773429870605, |
|
"rewards/margins": 1.585729718208313, |
|
"rewards/rejected": -0.31065240502357483, |
|
"step": 352 |
|
}, |
|
{ |
|
"epoch": 22.6, |
|
"grad_norm": 4.1737165451049805, |
|
"learning_rate": 4.824024971003845e-07, |
|
"logits/chosen": -2.708979845046997, |
|
"logits/rejected": -2.718264579772949, |
|
"logps/chosen": -113.75922393798828, |
|
"logps/rejected": -100.5313720703125, |
|
"loss": 0.2626, |
|
"rewards/accuracies": 1.0, |
|
"rewards/chosen": 1.3227206468582153, |
|
"rewards/margins": 1.4637939929962158, |
|
"rewards/rejected": -0.1410733461380005, |
|
"step": 356 |
|
}, |
|
{ |
|
"epoch": 22.86, |
|
"grad_norm": 4.2186737060546875, |
|
"learning_rate": 4.819836430296808e-07, |
|
"logits/chosen": -2.688643217086792, |
|
"logits/rejected": -2.7126376628875732, |
|
"logps/chosen": -119.3924560546875, |
|
"logps/rejected": -102.88687896728516, |
|
"loss": 0.272, |
|
"rewards/accuracies": 0.984375, |
|
"rewards/chosen": 1.3319997787475586, |
|
"rewards/margins": 1.3775808811187744, |
|
"rewards/rejected": -0.04558102786540985, |
|
"step": 360 |
|
}, |
|
{ |
|
"epoch": 22.86, |
|
"eval_logits/chosen": -2.7863781452178955, |
|
"eval_logits/rejected": -2.8632652759552, |
|
"eval_logps/chosen": -123.22322082519531, |
|
"eval_logps/rejected": -97.17105865478516, |
|
"eval_loss": 0.4147447645664215, |
|
"eval_rewards/accuracies": 0.8214285969734192, |
|
"eval_rewards/chosen": 1.0846121311187744, |
|
"eval_rewards/margins": 0.9908290505409241, |
|
"eval_rewards/rejected": 0.09378316253423691, |
|
"eval_runtime": 19.3953, |
|
"eval_samples_per_second": 1.444, |
|
"eval_steps_per_second": 1.444, |
|
"step": 360 |
|
}, |
|
{ |
|
"epoch": 23.11, |
|
"grad_norm": 3.8801398277282715, |
|
"learning_rate": 4.815600488670862e-07, |
|
"logits/chosen": -2.661937713623047, |
|
"logits/rejected": -2.69406795501709, |
|
"logps/chosen": -120.42941284179688, |
|
"logps/rejected": -107.56249237060547, |
|
"loss": 0.2463, |
|
"rewards/accuracies": 1.0, |
|
"rewards/chosen": 1.303568959236145, |
|
"rewards/margins": 1.5796161890029907, |
|
"rewards/rejected": -0.2760472595691681, |
|
"step": 364 |
|
}, |
|
{ |
|
"epoch": 23.37, |
|
"grad_norm": 3.6953530311584473, |
|
"learning_rate": 4.811317232678456e-07, |
|
"logits/chosen": -2.696389675140381, |
|
"logits/rejected": -2.7162017822265625, |
|
"logps/chosen": -115.95695495605469, |
|
"logps/rejected": -95.26593017578125, |
|
"loss": 0.2557, |
|
"rewards/accuracies": 0.984375, |
|
"rewards/chosen": 1.3928656578063965, |
|
"rewards/margins": 1.5144150257110596, |
|
"rewards/rejected": -0.12154923379421234, |
|
"step": 368 |
|
}, |
|
{ |
|
"epoch": 23.62, |
|
"grad_norm": 4.2311553955078125, |
|
"learning_rate": 4.806986749838806e-07, |
|
"logits/chosen": -2.735870599746704, |
|
"logits/rejected": -2.7359132766723633, |
|
"logps/chosen": -120.84663391113281, |
|
"logps/rejected": -107.17831420898438, |
|
"loss": 0.2922, |
|
"rewards/accuracies": 0.984375, |
|
"rewards/chosen": 1.2136850357055664, |
|
"rewards/margins": 1.3251327276229858, |
|
"rewards/rejected": -0.11144763976335526, |
|
"step": 372 |
|
}, |
|
{ |
|
"epoch": 23.87, |
|
"grad_norm": 3.250326633453369, |
|
"learning_rate": 4.802609128636112e-07, |
|
"logits/chosen": -2.6809797286987305, |
|
"logits/rejected": -2.700901508331299, |
|
"logps/chosen": -121.92070007324219, |
|
"logps/rejected": -99.91931915283203, |
|
"loss": 0.2062, |
|
"rewards/accuracies": 0.984375, |
|
"rewards/chosen": 1.4374094009399414, |
|
"rewards/margins": 1.8276853561401367, |
|
"rewards/rejected": -0.3902759850025177, |
|
"step": 376 |
|
}, |
|
{ |
|
"epoch": 24.13, |
|
"grad_norm": 4.59958553314209, |
|
"learning_rate": 4.798184458517744e-07, |
|
"logits/chosen": -2.7332396507263184, |
|
"logits/rejected": -2.768554210662842, |
|
"logps/chosen": -114.1657943725586, |
|
"logps/rejected": -100.7848892211914, |
|
"loss": 0.2353, |
|
"rewards/accuracies": 1.0, |
|
"rewards/chosen": 1.4016469717025757, |
|
"rewards/margins": 1.5988764762878418, |
|
"rewards/rejected": -0.19722968339920044, |
|
"step": 380 |
|
}, |
|
{ |
|
"epoch": 24.38, |
|
"grad_norm": 3.6598093509674072, |
|
"learning_rate": 4.793712829892414e-07, |
|
"logits/chosen": -2.7024924755096436, |
|
"logits/rejected": -2.7085440158843994, |
|
"logps/chosen": -114.50930786132812, |
|
"logps/rejected": -104.57618713378906, |
|
"loss": 0.2538, |
|
"rewards/accuracies": 1.0, |
|
"rewards/chosen": 1.4238942861557007, |
|
"rewards/margins": 1.5259473323822021, |
|
"rewards/rejected": -0.10205309092998505, |
|
"step": 384 |
|
}, |
|
{ |
|
"epoch": 24.63, |
|
"grad_norm": 3.3857998847961426, |
|
"learning_rate": 4.789194334128338e-07, |
|
"logits/chosen": -2.7139151096343994, |
|
"logits/rejected": -2.7087197303771973, |
|
"logps/chosen": -121.57815551757812, |
|
"logps/rejected": -102.66044616699219, |
|
"loss": 0.2324, |
|
"rewards/accuracies": 1.0, |
|
"rewards/chosen": 1.3619202375411987, |
|
"rewards/margins": 1.667473316192627, |
|
"rewards/rejected": -0.305553138256073, |
|
"step": 388 |
|
}, |
|
{ |
|
"epoch": 24.89, |
|
"grad_norm": 4.038039207458496, |
|
"learning_rate": 4.784629063551354e-07, |
|
"logits/chosen": -2.651282548904419, |
|
"logits/rejected": -2.679689884185791, |
|
"logps/chosen": -118.9875717163086, |
|
"logps/rejected": -97.62678527832031, |
|
"loss": 0.2304, |
|
"rewards/accuracies": 0.96875, |
|
"rewards/chosen": 1.3316906690597534, |
|
"rewards/margins": 1.7062404155731201, |
|
"rewards/rejected": -0.3745496869087219, |
|
"step": 392 |
|
}, |
|
{ |
|
"epoch": 25.14, |
|
"grad_norm": 3.0911149978637695, |
|
"learning_rate": 4.780017111443047e-07, |
|
"logits/chosen": -2.718074083328247, |
|
"logits/rejected": -2.7447762489318848, |
|
"logps/chosen": -121.20558166503906, |
|
"logps/rejected": -105.63926696777344, |
|
"loss": 0.2411, |
|
"rewards/accuracies": 1.0, |
|
"rewards/chosen": 1.4981815814971924, |
|
"rewards/margins": 1.644660472869873, |
|
"rewards/rejected": -0.14647886157035828, |
|
"step": 396 |
|
}, |
|
{ |
|
"epoch": 25.4, |
|
"grad_norm": 3.236288070678711, |
|
"learning_rate": 4.775358572038845e-07, |
|
"logits/chosen": -2.693341016769409, |
|
"logits/rejected": -2.681511878967285, |
|
"logps/chosen": -114.28794860839844, |
|
"logps/rejected": -91.69850158691406, |
|
"loss": 0.259, |
|
"rewards/accuracies": 0.984375, |
|
"rewards/chosen": 1.3284053802490234, |
|
"rewards/margins": 1.5037521123886108, |
|
"rewards/rejected": -0.17534679174423218, |
|
"step": 400 |
|
}, |
|
{ |
|
"epoch": 25.4, |
|
"eval_logits/chosen": -2.7796428203582764, |
|
"eval_logits/rejected": -2.8552372455596924, |
|
"eval_logps/chosen": -122.60997009277344, |
|
"eval_logps/rejected": -97.46685028076172, |
|
"eval_loss": 0.4054030478000641, |
|
"eval_rewards/accuracies": 0.7857142686843872, |
|
"eval_rewards/chosen": 1.1459368467330933, |
|
"eval_rewards/margins": 1.0817322731018066, |
|
"eval_rewards/rejected": 0.06420455127954483, |
|
"eval_runtime": 19.3983, |
|
"eval_samples_per_second": 1.443, |
|
"eval_steps_per_second": 1.443, |
|
"step": 400 |
|
}, |
|
{ |
|
"epoch": 25.65, |
|
"grad_norm": 3.2894155979156494, |
|
"learning_rate": 4.770653540526078e-07, |
|
"logits/chosen": -2.652012825012207, |
|
"logits/rejected": -2.690704345703125, |
|
"logps/chosen": -118.52864837646484, |
|
"logps/rejected": -102.0433578491211, |
|
"loss": 0.1918, |
|
"rewards/accuracies": 1.0, |
|
"rewards/chosen": 1.5184407234191895, |
|
"rewards/margins": 1.8730953931808472, |
|
"rewards/rejected": -0.3546547293663025, |
|
"step": 404 |
|
}, |
|
{ |
|
"epoch": 25.9, |
|
"grad_norm": 4.958095550537109, |
|
"learning_rate": 4.765902113042053e-07, |
|
"logits/chosen": -2.7191715240478516, |
|
"logits/rejected": -2.701702833175659, |
|
"logps/chosen": -115.93360900878906, |
|
"logps/rejected": -100.49573516845703, |
|
"loss": 0.2536, |
|
"rewards/accuracies": 1.0, |
|
"rewards/chosen": 1.357507348060608, |
|
"rewards/margins": 1.542129635810852, |
|
"rewards/rejected": -0.18462230265140533, |
|
"step": 408 |
|
}, |
|
{ |
|
"epoch": 26.16, |
|
"grad_norm": 3.1129214763641357, |
|
"learning_rate": 4.7611043866720737e-07, |
|
"logits/chosen": -2.6934893131256104, |
|
"logits/rejected": -2.7433109283447266, |
|
"logps/chosen": -117.78741455078125, |
|
"logps/rejected": -102.46746063232422, |
|
"loss": 0.2137, |
|
"rewards/accuracies": 0.96875, |
|
"rewards/chosen": 1.484704613685608, |
|
"rewards/margins": 1.8401193618774414, |
|
"rewards/rejected": -0.3554147481918335, |
|
"step": 412 |
|
}, |
|
{ |
|
"epoch": 26.41, |
|
"grad_norm": 3.2823047637939453, |
|
"learning_rate": 4.7562604594474643e-07, |
|
"logits/chosen": -2.6928577423095703, |
|
"logits/rejected": -2.708937644958496, |
|
"logps/chosen": -124.94667053222656, |
|
"logps/rejected": -106.61329650878906, |
|
"loss": 0.2137, |
|
"rewards/accuracies": 1.0, |
|
"rewards/chosen": 1.471519112586975, |
|
"rewards/margins": 1.7852158546447754, |
|
"rewards/rejected": -0.31369656324386597, |
|
"step": 416 |
|
}, |
|
{ |
|
"epoch": 26.67, |
|
"grad_norm": 3.7071120738983154, |
|
"learning_rate": 4.7513704303435674e-07, |
|
"logits/chosen": -2.670248031616211, |
|
"logits/rejected": -2.6764914989471436, |
|
"logps/chosen": -110.71532440185547, |
|
"logps/rejected": -101.84510803222656, |
|
"loss": 0.229, |
|
"rewards/accuracies": 0.984375, |
|
"rewards/chosen": 1.4525586366653442, |
|
"rewards/margins": 1.6690635681152344, |
|
"rewards/rejected": -0.21650481224060059, |
|
"step": 420 |
|
}, |
|
{ |
|
"epoch": 26.92, |
|
"grad_norm": 3.449319362640381, |
|
"learning_rate": 4.7464343992777165e-07, |
|
"logits/chosen": -2.688829183578491, |
|
"logits/rejected": -2.7286489009857178, |
|
"logps/chosen": -115.69140625, |
|
"logps/rejected": -103.97221374511719, |
|
"loss": 0.2231, |
|
"rewards/accuracies": 1.0, |
|
"rewards/chosen": 1.4910831451416016, |
|
"rewards/margins": 1.7472798824310303, |
|
"rewards/rejected": -0.2561967670917511, |
|
"step": 424 |
|
}, |
|
{ |
|
"epoch": 27.17, |
|
"grad_norm": 2.4958574771881104, |
|
"learning_rate": 4.741452467107199e-07, |
|
"logits/chosen": -2.6915664672851562, |
|
"logits/rejected": -2.7021422386169434, |
|
"logps/chosen": -110.91499328613281, |
|
"logps/rejected": -98.70269775390625, |
|
"loss": 0.233, |
|
"rewards/accuracies": 0.984375, |
|
"rewards/chosen": 1.4354866743087769, |
|
"rewards/margins": 1.7168078422546387, |
|
"rewards/rejected": -0.28132107853889465, |
|
"step": 428 |
|
}, |
|
{ |
|
"epoch": 27.43, |
|
"grad_norm": 3.7154319286346436, |
|
"learning_rate": 4.7364247356271927e-07, |
|
"logits/chosen": -2.6695199012756348, |
|
"logits/rejected": -2.6698789596557617, |
|
"logps/chosen": -118.46913146972656, |
|
"logps/rejected": -102.86263275146484, |
|
"loss": 0.2337, |
|
"rewards/accuracies": 1.0, |
|
"rewards/chosen": 1.4410640001296997, |
|
"rewards/margins": 1.6770535707473755, |
|
"rewards/rejected": -0.23598948121070862, |
|
"step": 432 |
|
}, |
|
{ |
|
"epoch": 27.68, |
|
"grad_norm": 3.101527690887451, |
|
"learning_rate": 4.731351307568687e-07, |
|
"logits/chosen": -2.6910083293914795, |
|
"logits/rejected": -2.6801414489746094, |
|
"logps/chosen": -115.44275665283203, |
|
"logps/rejected": -97.7877197265625, |
|
"loss": 0.2, |
|
"rewards/accuracies": 1.0, |
|
"rewards/chosen": 1.5193870067596436, |
|
"rewards/margins": 1.8750090599060059, |
|
"rewards/rejected": -0.35562214255332947, |
|
"step": 436 |
|
}, |
|
{ |
|
"epoch": 27.94, |
|
"grad_norm": 3.9017672538757324, |
|
"learning_rate": 4.7262322865963846e-07, |
|
"logits/chosen": -2.7122743129730225, |
|
"logits/rejected": -2.723294258117676, |
|
"logps/chosen": -116.90724182128906, |
|
"logps/rejected": -104.45236206054688, |
|
"loss": 0.1959, |
|
"rewards/accuracies": 1.0, |
|
"rewards/chosen": 1.5187357664108276, |
|
"rewards/margins": 1.918578863143921, |
|
"rewards/rejected": -0.39984312653541565, |
|
"step": 440 |
|
}, |
|
{ |
|
"epoch": 27.94, |
|
"eval_logits/chosen": -2.7724335193634033, |
|
"eval_logits/rejected": -2.8468399047851562, |
|
"eval_logps/chosen": -122.11509704589844, |
|
"eval_logps/rejected": -97.87715148925781, |
|
"eval_loss": 0.39711207151412964, |
|
"eval_rewards/accuracies": 0.7857142686843872, |
|
"eval_rewards/chosen": 1.1954243183135986, |
|
"eval_rewards/margins": 1.1722500324249268, |
|
"eval_rewards/rejected": 0.023174213245511055, |
|
"eval_runtime": 19.4446, |
|
"eval_samples_per_second": 1.44, |
|
"eval_steps_per_second": 1.44, |
|
"step": 440 |
|
}, |
|
{ |
|
"epoch": 28.19, |
|
"grad_norm": 4.0338592529296875, |
|
"learning_rate": 4.721067777306581e-07, |
|
"logits/chosen": -2.6722593307495117, |
|
"logits/rejected": -2.682727575302124, |
|
"logps/chosen": -119.6982650756836, |
|
"logps/rejected": -105.52352905273438, |
|
"loss": 0.2108, |
|
"rewards/accuracies": 0.984375, |
|
"rewards/chosen": 1.5048611164093018, |
|
"rewards/margins": 1.8753631114959717, |
|
"rewards/rejected": -0.3705020546913147, |
|
"step": 444 |
|
}, |
|
{ |
|
"epoch": 28.44, |
|
"grad_norm": 3.167048692703247, |
|
"learning_rate": 4.71585788522503e-07, |
|
"logits/chosen": -2.652644157409668, |
|
"logits/rejected": -2.6818065643310547, |
|
"logps/chosen": -113.42478942871094, |
|
"logps/rejected": -104.37007141113281, |
|
"loss": 0.2076, |
|
"rewards/accuracies": 1.0, |
|
"rewards/chosen": 1.474198341369629, |
|
"rewards/margins": 1.7973999977111816, |
|
"rewards/rejected": -0.32320165634155273, |
|
"step": 448 |
|
}, |
|
{ |
|
"epoch": 28.7, |
|
"grad_norm": 3.4248368740081787, |
|
"learning_rate": 4.7106027168047833e-07, |
|
"logits/chosen": -2.7117884159088135, |
|
"logits/rejected": -2.77234148979187, |
|
"logps/chosen": -116.26769256591797, |
|
"logps/rejected": -101.22008514404297, |
|
"loss": 0.1858, |
|
"rewards/accuracies": 1.0, |
|
"rewards/chosen": 1.5492609739303589, |
|
"rewards/margins": 1.9960644245147705, |
|
"rewards/rejected": -0.4468035101890564, |
|
"step": 452 |
|
}, |
|
{ |
|
"epoch": 28.95, |
|
"grad_norm": 2.855328321456909, |
|
"learning_rate": 4.705302379424022e-07, |
|
"logits/chosen": -2.6717069149017334, |
|
"logits/rejected": -2.672755718231201, |
|
"logps/chosen": -116.75204467773438, |
|
"logps/rejected": -106.22608184814453, |
|
"loss": 0.2065, |
|
"rewards/accuracies": 1.0, |
|
"rewards/chosen": 1.5143406391143799, |
|
"rewards/margins": 1.8520642518997192, |
|
"rewards/rejected": -0.33772367238998413, |
|
"step": 456 |
|
}, |
|
{ |
|
"epoch": 29.21, |
|
"grad_norm": 2.8747594356536865, |
|
"learning_rate": 4.699956981383857e-07, |
|
"logits/chosen": -2.693591356277466, |
|
"logits/rejected": -2.662400960922241, |
|
"logps/chosen": -120.1160888671875, |
|
"logps/rejected": -91.72486877441406, |
|
"loss": 0.1874, |
|
"rewards/accuracies": 1.0, |
|
"rewards/chosen": 1.6478428840637207, |
|
"rewards/margins": 2.0628483295440674, |
|
"rewards/rejected": -0.4150054454803467, |
|
"step": 460 |
|
}, |
|
{ |
|
"epoch": 29.46, |
|
"grad_norm": 2.8170723915100098, |
|
"learning_rate": 4.694566631906116e-07, |
|
"logits/chosen": -2.6773173809051514, |
|
"logits/rejected": -2.672825574874878, |
|
"logps/chosen": -113.59075927734375, |
|
"logps/rejected": -103.25709533691406, |
|
"loss": 0.1984, |
|
"rewards/accuracies": 1.0, |
|
"rewards/chosen": 1.4863029718399048, |
|
"rewards/margins": 1.892244577407837, |
|
"rewards/rejected": -0.405941367149353, |
|
"step": 464 |
|
}, |
|
{ |
|
"epoch": 29.71, |
|
"grad_norm": 3.132080078125, |
|
"learning_rate": 4.689131441131118e-07, |
|
"logits/chosen": -2.6520447731018066, |
|
"logits/rejected": -2.6872832775115967, |
|
"logps/chosen": -118.07131958007812, |
|
"logps/rejected": -107.88346862792969, |
|
"loss": 0.2029, |
|
"rewards/accuracies": 0.984375, |
|
"rewards/chosen": 1.6008590459823608, |
|
"rewards/margins": 1.909461259841919, |
|
"rewards/rejected": -0.30860215425491333, |
|
"step": 468 |
|
}, |
|
{ |
|
"epoch": 29.97, |
|
"grad_norm": 3.000974416732788, |
|
"learning_rate": 4.6836515201154135e-07, |
|
"logits/chosen": -2.709559202194214, |
|
"logits/rejected": -2.723712205886841, |
|
"logps/chosen": -112.09906768798828, |
|
"logps/rejected": -106.09660339355469, |
|
"loss": 0.2037, |
|
"rewards/accuracies": 1.0, |
|
"rewards/chosen": 1.5191339254379272, |
|
"rewards/margins": 1.8524271249771118, |
|
"rewards/rejected": -0.3332933187484741, |
|
"step": 472 |
|
}, |
|
{ |
|
"epoch": 30.22, |
|
"grad_norm": 2.544147491455078, |
|
"learning_rate": 4.678126980829524e-07, |
|
"logits/chosen": -2.678325653076172, |
|
"logits/rejected": -2.699833631515503, |
|
"logps/chosen": -115.4670639038086, |
|
"logps/rejected": -101.5027847290039, |
|
"loss": 0.1813, |
|
"rewards/accuracies": 1.0, |
|
"rewards/chosen": 1.4956222772598267, |
|
"rewards/margins": 1.9856685400009155, |
|
"rewards/rejected": -0.4900462329387665, |
|
"step": 476 |
|
}, |
|
{ |
|
"epoch": 30.48, |
|
"grad_norm": 2.9770796298980713, |
|
"learning_rate": 4.672557936155649e-07, |
|
"logits/chosen": -2.708721160888672, |
|
"logits/rejected": -2.728306770324707, |
|
"logps/chosen": -109.71602630615234, |
|
"logps/rejected": -98.1273422241211, |
|
"loss": 0.1946, |
|
"rewards/accuracies": 0.984375, |
|
"rewards/chosen": 1.5404164791107178, |
|
"rewards/margins": 1.9511140584945679, |
|
"rewards/rejected": -0.4106977581977844, |
|
"step": 480 |
|
}, |
|
{ |
|
"epoch": 30.48, |
|
"eval_logits/chosen": -2.764019012451172, |
|
"eval_logits/rejected": -2.8370797634124756, |
|
"eval_logps/chosen": -121.74720001220703, |
|
"eval_logps/rejected": -98.30638122558594, |
|
"eval_loss": 0.393496572971344, |
|
"eval_rewards/accuracies": 0.7857142686843872, |
|
"eval_rewards/chosen": 1.232213020324707, |
|
"eval_rewards/margins": 1.2519627809524536, |
|
"eval_rewards/rejected": -0.019749613478779793, |
|
"eval_runtime": 19.4183, |
|
"eval_samples_per_second": 1.442, |
|
"eval_steps_per_second": 1.442, |
|
"step": 480 |
|
}, |
|
{ |
|
"epoch": 30.73, |
|
"grad_norm": 3.826716661453247, |
|
"learning_rate": 4.6669444998853605e-07, |
|
"logits/chosen": -2.6732892990112305, |
|
"logits/rejected": -2.700220823287964, |
|
"logps/chosen": -117.65553283691406, |
|
"logps/rejected": -111.19263458251953, |
|
"loss": 0.1876, |
|
"rewards/accuracies": 1.0, |
|
"rewards/chosen": 1.6948100328445435, |
|
"rewards/margins": 2.0582737922668457, |
|
"rewards/rejected": -0.3634639382362366, |
|
"step": 484 |
|
}, |
|
{ |
|
"epoch": 30.98, |
|
"grad_norm": 3.326704740524292, |
|
"learning_rate": 4.661286786717278e-07, |
|
"logits/chosen": -2.6515111923217773, |
|
"logits/rejected": -2.6504664421081543, |
|
"logps/chosen": -120.06623840332031, |
|
"logps/rejected": -101.47470092773438, |
|
"loss": 0.1828, |
|
"rewards/accuracies": 1.0, |
|
"rewards/chosen": 1.5970648527145386, |
|
"rewards/margins": 2.0543787479400635, |
|
"rewards/rejected": -0.45731377601623535, |
|
"step": 488 |
|
}, |
|
{ |
|
"epoch": 31.24, |
|
"grad_norm": 2.700209856033325, |
|
"learning_rate": 4.6555849122547263e-07, |
|
"logits/chosen": -2.6469173431396484, |
|
"logits/rejected": -2.666424036026001, |
|
"logps/chosen": -111.76505279541016, |
|
"logps/rejected": -103.40081024169922, |
|
"loss": 0.185, |
|
"rewards/accuracies": 1.0, |
|
"rewards/chosen": 1.524863839149475, |
|
"rewards/margins": 1.9786373376846313, |
|
"rewards/rejected": -0.4537736773490906, |
|
"step": 492 |
|
}, |
|
{ |
|
"epoch": 31.49, |
|
"grad_norm": 2.592606544494629, |
|
"learning_rate": 4.649838993003372e-07, |
|
"logits/chosen": -2.7192447185516357, |
|
"logits/rejected": -2.726841926574707, |
|
"logps/chosen": -112.72746276855469, |
|
"logps/rejected": -100.51376342773438, |
|
"loss": 0.1678, |
|
"rewards/accuracies": 0.984375, |
|
"rewards/chosen": 1.5847443342208862, |
|
"rewards/margins": 2.0699594020843506, |
|
"rewards/rejected": -0.4852150082588196, |
|
"step": 496 |
|
}, |
|
{ |
|
"epoch": 31.75, |
|
"grad_norm": 2.768076181411743, |
|
"learning_rate": 4.644049146368843e-07, |
|
"logits/chosen": -2.6624693870544434, |
|
"logits/rejected": -2.677476406097412, |
|
"logps/chosen": -119.13687896728516, |
|
"logps/rejected": -103.49778747558594, |
|
"loss": 0.1873, |
|
"rewards/accuracies": 1.0, |
|
"rewards/chosen": 1.7677032947540283, |
|
"rewards/margins": 2.089017868041992, |
|
"rewards/rejected": -0.32131490111351013, |
|
"step": 500 |
|
}, |
|
{ |
|
"epoch": 32.0, |
|
"grad_norm": 2.4647607803344727, |
|
"learning_rate": 4.638215490654329e-07, |
|
"logits/chosen": -2.6946330070495605, |
|
"logits/rejected": -2.6913187503814697, |
|
"logps/chosen": -118.64775085449219, |
|
"logps/rejected": -107.40900421142578, |
|
"loss": 0.1847, |
|
"rewards/accuracies": 1.0, |
|
"rewards/chosen": 1.5691322088241577, |
|
"rewards/margins": 2.105942964553833, |
|
"rewards/rejected": -0.5368106365203857, |
|
"step": 504 |
|
}, |
|
{ |
|
"epoch": 32.25, |
|
"grad_norm": 3.723209857940674, |
|
"learning_rate": 4.6323381450581665e-07, |
|
"logits/chosen": -2.6984715461730957, |
|
"logits/rejected": -2.686072826385498, |
|
"logps/chosen": -118.19576263427734, |
|
"logps/rejected": -104.87760162353516, |
|
"loss": 0.1853, |
|
"rewards/accuracies": 1.0, |
|
"rewards/chosen": 1.504220724105835, |
|
"rewards/margins": 2.023249864578247, |
|
"rewards/rejected": -0.519028902053833, |
|
"step": 508 |
|
}, |
|
{ |
|
"epoch": 32.51, |
|
"grad_norm": 2.6810081005096436, |
|
"learning_rate": 4.6264172296714e-07, |
|
"logits/chosen": -2.6358213424682617, |
|
"logits/rejected": -2.6452157497406006, |
|
"logps/chosen": -116.47501373291016, |
|
"logps/rejected": -104.9029769897461, |
|
"loss": 0.1948, |
|
"rewards/accuracies": 0.984375, |
|
"rewards/chosen": 1.6027023792266846, |
|
"rewards/margins": 2.105417251586914, |
|
"rewards/rejected": -0.5027150511741638, |
|
"step": 512 |
|
}, |
|
{ |
|
"epoch": 32.76, |
|
"grad_norm": 2.1455435752868652, |
|
"learning_rate": 4.6204528654753304e-07, |
|
"logits/chosen": -2.692194700241089, |
|
"logits/rejected": -2.705289125442505, |
|
"logps/chosen": -113.38731384277344, |
|
"logps/rejected": -103.49237823486328, |
|
"loss": 0.1617, |
|
"rewards/accuracies": 1.0, |
|
"rewards/chosen": 1.7136329412460327, |
|
"rewards/margins": 2.1900761127471924, |
|
"rewards/rejected": -0.47644317150115967, |
|
"step": 516 |
|
}, |
|
{ |
|
"epoch": 33.02, |
|
"grad_norm": 3.499128818511963, |
|
"learning_rate": 4.614445174339045e-07, |
|
"logits/chosen": -2.661409616470337, |
|
"logits/rejected": -2.700990915298462, |
|
"logps/chosen": -113.71514892578125, |
|
"logps/rejected": -103.91769409179688, |
|
"loss": 0.1507, |
|
"rewards/accuracies": 1.0, |
|
"rewards/chosen": 1.7427312135696411, |
|
"rewards/margins": 2.1958351135253906, |
|
"rewards/rejected": -0.45310381054878235, |
|
"step": 520 |
|
}, |
|
{ |
|
"epoch": 33.02, |
|
"eval_logits/chosen": -2.755352020263672, |
|
"eval_logits/rejected": -2.826873540878296, |
|
"eval_logps/chosen": -121.47989654541016, |
|
"eval_logps/rejected": -98.8564453125, |
|
"eval_loss": 0.3899853527545929, |
|
"eval_rewards/accuracies": 0.7857142686843872, |
|
"eval_rewards/chosen": 1.2589442729949951, |
|
"eval_rewards/margins": 1.333700180053711, |
|
"eval_rewards/rejected": -0.07475597411394119, |
|
"eval_runtime": 19.3899, |
|
"eval_samples_per_second": 1.444, |
|
"eval_steps_per_second": 1.444, |
|
"step": 520 |
|
}, |
|
{ |
|
"epoch": 33.27, |
|
"grad_norm": 2.419506549835205, |
|
"learning_rate": 4.6083942790169206e-07, |
|
"logits/chosen": -2.6735386848449707, |
|
"logits/rejected": -2.696384906768799, |
|
"logps/chosen": -111.79097747802734, |
|
"logps/rejected": -98.7368392944336, |
|
"loss": 0.1581, |
|
"rewards/accuracies": 0.984375, |
|
"rewards/chosen": 1.5749679803848267, |
|
"rewards/margins": 2.265718460083008, |
|
"rewards/rejected": -0.690750777721405, |
|
"step": 524 |
|
}, |
|
{ |
|
"epoch": 33.52, |
|
"grad_norm": 2.1046078205108643, |
|
"learning_rate": 4.6023003031461227e-07, |
|
"logits/chosen": -2.6465351581573486, |
|
"logits/rejected": -2.6622884273529053, |
|
"logps/chosen": -114.73282623291016, |
|
"logps/rejected": -103.12973022460938, |
|
"loss": 0.1778, |
|
"rewards/accuracies": 1.0, |
|
"rewards/chosen": 1.6710339784622192, |
|
"rewards/margins": 2.119300365447998, |
|
"rewards/rejected": -0.4482663571834564, |
|
"step": 528 |
|
}, |
|
{ |
|
"epoch": 33.78, |
|
"grad_norm": 3.0457003116607666, |
|
"learning_rate": 4.596163371244076e-07, |
|
"logits/chosen": -2.6422927379608154, |
|
"logits/rejected": -2.6462604999542236, |
|
"logps/chosen": -114.708740234375, |
|
"logps/rejected": -105.07784271240234, |
|
"loss": 0.16, |
|
"rewards/accuracies": 1.0, |
|
"rewards/chosen": 1.7936890125274658, |
|
"rewards/margins": 2.2034826278686523, |
|
"rewards/rejected": -0.40979355573654175, |
|
"step": 532 |
|
}, |
|
{ |
|
"epoch": 34.03, |
|
"grad_norm": 2.8257133960723877, |
|
"learning_rate": 4.5899836087059174e-07, |
|
"logits/chosen": -2.6918063163757324, |
|
"logits/rejected": -2.696005344390869, |
|
"logps/chosen": -117.54481506347656, |
|
"logps/rejected": -111.74267578125, |
|
"loss": 0.1753, |
|
"rewards/accuracies": 1.0, |
|
"rewards/chosen": 1.625847339630127, |
|
"rewards/margins": 2.2450764179229736, |
|
"rewards/rejected": -0.6192291975021362, |
|
"step": 536 |
|
}, |
|
{ |
|
"epoch": 34.29, |
|
"grad_norm": 2.5381155014038086, |
|
"learning_rate": 4.58376114180194e-07, |
|
"logits/chosen": -2.684847116470337, |
|
"logits/rejected": -2.6875247955322266, |
|
"logps/chosen": -121.1004867553711, |
|
"logps/rejected": -101.69570922851562, |
|
"loss": 0.1562, |
|
"rewards/accuracies": 0.984375, |
|
"rewards/chosen": 1.6668047904968262, |
|
"rewards/margins": 2.3632285594940186, |
|
"rewards/rejected": -0.6964237689971924, |
|
"step": 540 |
|
}, |
|
{ |
|
"epoch": 34.54, |
|
"grad_norm": 2.519352912902832, |
|
"learning_rate": 4.577496097675009e-07, |
|
"logits/chosen": -2.6664843559265137, |
|
"logits/rejected": -2.681034803390503, |
|
"logps/chosen": -115.94261169433594, |
|
"logps/rejected": -108.1679458618164, |
|
"loss": 0.1516, |
|
"rewards/accuracies": 1.0, |
|
"rewards/chosen": 1.7702218294143677, |
|
"rewards/margins": 2.2215099334716797, |
|
"rewards/rejected": -0.4512881338596344, |
|
"step": 544 |
|
}, |
|
{ |
|
"epoch": 34.79, |
|
"grad_norm": 2.789398431777954, |
|
"learning_rate": 4.571188604337962e-07, |
|
"logits/chosen": -2.668377637863159, |
|
"logits/rejected": -2.688797950744629, |
|
"logps/chosen": -108.2844467163086, |
|
"logps/rejected": -104.94053649902344, |
|
"loss": 0.1838, |
|
"rewards/accuracies": 1.0, |
|
"rewards/chosen": 1.559061050415039, |
|
"rewards/margins": 2.064011812210083, |
|
"rewards/rejected": -0.504950761795044, |
|
"step": 548 |
|
}, |
|
{ |
|
"epoch": 35.05, |
|
"grad_norm": 3.570829153060913, |
|
"learning_rate": 4.5648387906709995e-07, |
|
"logits/chosen": -2.6559903621673584, |
|
"logits/rejected": -2.650508165359497, |
|
"logps/chosen": -112.94985961914062, |
|
"logps/rejected": -101.36627197265625, |
|
"loss": 0.1484, |
|
"rewards/accuracies": 1.0, |
|
"rewards/chosen": 1.709464192390442, |
|
"rewards/margins": 2.303464889526367, |
|
"rewards/rejected": -0.5940006375312805, |
|
"step": 552 |
|
}, |
|
{ |
|
"epoch": 35.3, |
|
"grad_norm": 2.487671136856079, |
|
"learning_rate": 4.558446786419045e-07, |
|
"logits/chosen": -2.6454193592071533, |
|
"logits/rejected": -2.667191982269287, |
|
"logps/chosen": -112.91888427734375, |
|
"logps/rejected": -100.34318542480469, |
|
"loss": 0.1571, |
|
"rewards/accuracies": 0.984375, |
|
"rewards/chosen": 1.6041696071624756, |
|
"rewards/margins": 2.3108744621276855, |
|
"rewards/rejected": -0.7067050337791443, |
|
"step": 556 |
|
}, |
|
{ |
|
"epoch": 35.56, |
|
"grad_norm": 2.396897315979004, |
|
"learning_rate": 4.552012722189099e-07, |
|
"logits/chosen": -2.6444239616394043, |
|
"logits/rejected": -2.6464180946350098, |
|
"logps/chosen": -115.7559814453125, |
|
"logps/rejected": -111.63291931152344, |
|
"loss": 0.1752, |
|
"rewards/accuracies": 1.0, |
|
"rewards/chosen": 1.7378344535827637, |
|
"rewards/margins": 2.1783785820007324, |
|
"rewards/rejected": -0.4405441880226135, |
|
"step": 560 |
|
}, |
|
{ |
|
"epoch": 35.56, |
|
"eval_logits/chosen": -2.746135711669922, |
|
"eval_logits/rejected": -2.8161184787750244, |
|
"eval_logps/chosen": -121.27652740478516, |
|
"eval_logps/rejected": -99.45245361328125, |
|
"eval_loss": 0.38749733567237854, |
|
"eval_rewards/accuracies": 0.7857142686843872, |
|
"eval_rewards/chosen": 1.2792812585830688, |
|
"eval_rewards/margins": 1.4136368036270142, |
|
"eval_rewards/rejected": -0.1343555599451065, |
|
"eval_runtime": 19.3831, |
|
"eval_samples_per_second": 1.445, |
|
"eval_steps_per_second": 1.445, |
|
"step": 560 |
|
} |
|
], |
|
"logging_steps": 4, |
|
"max_steps": 2800, |
|
"num_input_tokens_seen": 0, |
|
"num_train_epochs": 187, |
|
"save_steps": 40, |
|
"total_flos": 0.0, |
|
"train_batch_size": 4, |
|
"trial_name": null, |
|
"trial_params": null |
|
} |
|
|