|
{ |
|
"best_metric": null, |
|
"best_model_checkpoint": null, |
|
"epoch": 1.0, |
|
"eval_steps": 100, |
|
"global_step": 328, |
|
"is_hyper_param_search": false, |
|
"is_local_process_zero": true, |
|
"is_world_process_zero": true, |
|
"log_history": [ |
|
{ |
|
"epoch": 0.0, |
|
"grad_norm": 52.10260886950557, |
|
"learning_rate": 1.5151515151515152e-08, |
|
"logits/chosen": -2.9049293994903564, |
|
"logits/rejected": -3.137899398803711, |
|
"logps/chosen": -848.0860595703125, |
|
"logps/rejected": -1267.928466796875, |
|
"loss": 0.6931, |
|
"rewards/accuracies": 0.0, |
|
"rewards/chosen": 0.0, |
|
"rewards/margins": 0.0, |
|
"rewards/rejected": 0.0, |
|
"step": 1 |
|
}, |
|
{ |
|
"epoch": 0.03, |
|
"grad_norm": 54.32742967797558, |
|
"learning_rate": 1.5151515151515152e-07, |
|
"logits/chosen": -3.1098616123199463, |
|
"logits/rejected": -3.1124260425567627, |
|
"logps/chosen": -1029.3779296875, |
|
"logps/rejected": -1345.0582275390625, |
|
"loss": 0.6914, |
|
"rewards/accuracies": 0.4930555522441864, |
|
"rewards/chosen": 0.00401829881593585, |
|
"rewards/margins": 0.004102306440472603, |
|
"rewards/rejected": -8.400815568165854e-05, |
|
"step": 10 |
|
}, |
|
{ |
|
"epoch": 0.06, |
|
"grad_norm": 31.172998615351542, |
|
"learning_rate": 3.0303030303030305e-07, |
|
"logits/chosen": -3.117793560028076, |
|
"logits/rejected": -3.1491177082061768, |
|
"logps/chosen": -1071.2353515625, |
|
"logps/rejected": -1354.4710693359375, |
|
"loss": 0.6537, |
|
"rewards/accuracies": 0.7250000238418579, |
|
"rewards/chosen": 0.07634462416172028, |
|
"rewards/margins": 0.08821593225002289, |
|
"rewards/rejected": -0.011871310882270336, |
|
"step": 20 |
|
}, |
|
{ |
|
"epoch": 0.09, |
|
"grad_norm": 28.792134111609187, |
|
"learning_rate": 4.545454545454545e-07, |
|
"logits/chosen": -3.1649086475372314, |
|
"logits/rejected": -3.2152061462402344, |
|
"logps/chosen": -1064.19970703125, |
|
"logps/rejected": -1391.3843994140625, |
|
"loss": 0.5897, |
|
"rewards/accuracies": 0.7437499761581421, |
|
"rewards/chosen": 0.1755533069372177, |
|
"rewards/margins": 0.30122771859169006, |
|
"rewards/rejected": -0.12567442655563354, |
|
"step": 30 |
|
}, |
|
{ |
|
"epoch": 0.12, |
|
"grad_norm": 26.037250152730635, |
|
"learning_rate": 4.993056783981012e-07, |
|
"logits/chosen": -3.338702440261841, |
|
"logits/rejected": -3.3210806846618652, |
|
"logps/chosen": -1098.9052734375, |
|
"logps/rejected": -1450.2767333984375, |
|
"loss": 0.4997, |
|
"rewards/accuracies": 0.8187500238418579, |
|
"rewards/chosen": 0.13215045630931854, |
|
"rewards/margins": 0.7883480787277222, |
|
"rewards/rejected": -0.6561976075172424, |
|
"step": 40 |
|
}, |
|
{ |
|
"epoch": 0.15, |
|
"grad_norm": 27.996786342247532, |
|
"learning_rate": 4.95914200522199e-07, |
|
"logits/chosen": -3.2485861778259277, |
|
"logits/rejected": -3.324843168258667, |
|
"logps/chosen": -996.6271362304688, |
|
"logps/rejected": -1481.156982421875, |
|
"loss": 0.467, |
|
"rewards/accuracies": 0.84375, |
|
"rewards/chosen": 0.08611873537302017, |
|
"rewards/margins": 1.149458646774292, |
|
"rewards/rejected": -1.0633398294448853, |
|
"step": 50 |
|
}, |
|
{ |
|
"epoch": 0.18, |
|
"grad_norm": 25.09282538012874, |
|
"learning_rate": 4.897364164920514e-07, |
|
"logits/chosen": -3.313831329345703, |
|
"logits/rejected": -3.2785778045654297, |
|
"logps/chosen": -959.1129150390625, |
|
"logps/rejected": -1587.156982421875, |
|
"loss": 0.3729, |
|
"rewards/accuracies": 0.8812500238418579, |
|
"rewards/chosen": 0.13842354714870453, |
|
"rewards/margins": 1.846784234046936, |
|
"rewards/rejected": -1.7083606719970703, |
|
"step": 60 |
|
}, |
|
{ |
|
"epoch": 0.21, |
|
"grad_norm": 30.921162958697664, |
|
"learning_rate": 4.808423230692373e-07, |
|
"logits/chosen": -3.2474944591522217, |
|
"logits/rejected": -3.291019916534424, |
|
"logps/chosen": -1021.1806640625, |
|
"logps/rejected": -1590.413818359375, |
|
"loss": 0.3752, |
|
"rewards/accuracies": 0.893750011920929, |
|
"rewards/chosen": -0.16332893073558807, |
|
"rewards/margins": 2.0271217823028564, |
|
"rewards/rejected": -2.190450429916382, |
|
"step": 70 |
|
}, |
|
{ |
|
"epoch": 0.24, |
|
"grad_norm": 31.254868204859726, |
|
"learning_rate": 4.693326938861366e-07, |
|
"logits/chosen": -3.301940441131592, |
|
"logits/rejected": -3.3018264770507812, |
|
"logps/chosen": -1133.804443359375, |
|
"logps/rejected": -1672.321533203125, |
|
"loss": 0.3439, |
|
"rewards/accuracies": 0.887499988079071, |
|
"rewards/chosen": -0.7101578712463379, |
|
"rewards/margins": 2.033524751663208, |
|
"rewards/rejected": -2.743682861328125, |
|
"step": 80 |
|
}, |
|
{ |
|
"epoch": 0.27, |
|
"grad_norm": 34.55073034392427, |
|
"learning_rate": 4.553379376404084e-07, |
|
"logits/chosen": -3.212740421295166, |
|
"logits/rejected": -3.263364791870117, |
|
"logps/chosen": -1189.436279296875, |
|
"logps/rejected": -1603.1971435546875, |
|
"loss": 0.3203, |
|
"rewards/accuracies": 0.84375, |
|
"rewards/chosen": -0.9354013204574585, |
|
"rewards/margins": 2.177839517593384, |
|
"rewards/rejected": -3.113241195678711, |
|
"step": 90 |
|
}, |
|
{ |
|
"epoch": 0.3, |
|
"grad_norm": 44.68710091171008, |
|
"learning_rate": 4.3901662051233745e-07, |
|
"logits/chosen": -3.138298511505127, |
|
"logits/rejected": -3.148540496826172, |
|
"logps/chosen": -1065.9473876953125, |
|
"logps/rejected": -1705.128173828125, |
|
"loss": 0.2894, |
|
"rewards/accuracies": 0.9125000238418579, |
|
"rewards/chosen": -0.9562646150588989, |
|
"rewards/margins": 2.6660516262054443, |
|
"rewards/rejected": -3.622316360473633, |
|
"step": 100 |
|
}, |
|
{ |
|
"epoch": 0.3, |
|
"eval_logits/chosen": -2.4093375205993652, |
|
"eval_logits/rejected": -2.694366216659546, |
|
"eval_logps/chosen": -111.92229461669922, |
|
"eval_logps/rejected": -801.1410522460938, |
|
"eval_loss": 0.2025093287229538, |
|
"eval_rewards/accuracies": 0.875, |
|
"eval_rewards/chosen": -0.21333497762680054, |
|
"eval_rewards/margins": 1.7935402393341064, |
|
"eval_rewards/rejected": -2.0068752765655518, |
|
"eval_runtime": 5.1839, |
|
"eval_samples_per_second": 4.63, |
|
"eval_steps_per_second": 0.193, |
|
"step": 100 |
|
}, |
|
{ |
|
"epoch": 0.34, |
|
"grad_norm": 38.50657095083863, |
|
"learning_rate": 4.2055366954665237e-07, |
|
"logits/chosen": -3.1230928897857666, |
|
"logits/rejected": -3.1047117710113525, |
|
"logps/chosen": -1150.9886474609375, |
|
"logps/rejected": -1801.260009765625, |
|
"loss": 0.269, |
|
"rewards/accuracies": 0.893750011920929, |
|
"rewards/chosen": -1.3049598932266235, |
|
"rewards/margins": 2.929929494857788, |
|
"rewards/rejected": -4.234889030456543, |
|
"step": 110 |
|
}, |
|
{ |
|
"epoch": 0.37, |
|
"grad_norm": 29.424331589339463, |
|
"learning_rate": 4.0015827735521523e-07, |
|
"logits/chosen": -3.0130834579467773, |
|
"logits/rejected": -3.0237557888031006, |
|
"logps/chosen": -1210.0142822265625, |
|
"logps/rejected": -1803.6591796875, |
|
"loss": 0.2426, |
|
"rewards/accuracies": 0.90625, |
|
"rewards/chosen": -1.3954451084136963, |
|
"rewards/margins": 3.6785919666290283, |
|
"rewards/rejected": -5.074037551879883, |
|
"step": 120 |
|
}, |
|
{ |
|
"epoch": 0.4, |
|
"grad_norm": 61.571819091215445, |
|
"learning_rate": 3.780615318811402e-07, |
|
"logits/chosen": -3.051529884338379, |
|
"logits/rejected": -2.9141249656677246, |
|
"logps/chosen": -1176.2293701171875, |
|
"logps/rejected": -2026.753173828125, |
|
"loss": 0.2321, |
|
"rewards/accuracies": 0.918749988079071, |
|
"rewards/chosen": -1.6401822566986084, |
|
"rewards/margins": 4.810054779052734, |
|
"rewards/rejected": -6.450237274169922, |
|
"step": 130 |
|
}, |
|
{ |
|
"epoch": 0.43, |
|
"grad_norm": 53.20520696391761, |
|
"learning_rate": 3.545137980800601e-07, |
|
"logits/chosen": -3.0094645023345947, |
|
"logits/rejected": -2.877675771713257, |
|
"logps/chosen": -1233.012451171875, |
|
"logps/rejected": -2089.841552734375, |
|
"loss": 0.1758, |
|
"rewards/accuracies": 0.9437500238418579, |
|
"rewards/chosen": -1.5989583730697632, |
|
"rewards/margins": 5.455811500549316, |
|
"rewards/rejected": -7.054769039154053, |
|
"step": 140 |
|
}, |
|
{ |
|
"epoch": 0.46, |
|
"grad_norm": 39.12953948720105, |
|
"learning_rate": 3.2978188118513814e-07, |
|
"logits/chosen": -2.921603202819824, |
|
"logits/rejected": -2.7563977241516113, |
|
"logps/chosen": -1218.6588134765625, |
|
"logps/rejected": -2079.738037109375, |
|
"loss": 0.1843, |
|
"rewards/accuracies": 0.9375, |
|
"rewards/chosen": -1.8553876876831055, |
|
"rewards/margins": 5.326550483703613, |
|
"rewards/rejected": -7.181939601898193, |
|
"step": 150 |
|
}, |
|
{ |
|
"epoch": 0.49, |
|
"grad_norm": 51.519512868650644, |
|
"learning_rate": 3.041460036971664e-07, |
|
"logits/chosen": -2.8195321559906006, |
|
"logits/rejected": -2.6623053550720215, |
|
"logps/chosen": -1251.6900634765625, |
|
"logps/rejected": -2145.56591796875, |
|
"loss": 0.171, |
|
"rewards/accuracies": 0.925000011920929, |
|
"rewards/chosen": -2.4222846031188965, |
|
"rewards/margins": 5.688368797302246, |
|
"rewards/rejected": -8.110652923583984, |
|
"step": 160 |
|
}, |
|
{ |
|
"epoch": 0.52, |
|
"grad_norm": 34.8732410879268, |
|
"learning_rate": 2.778966303516603e-07, |
|
"logits/chosen": -2.7237300872802734, |
|
"logits/rejected": -2.5120363235473633, |
|
"logps/chosen": -1312.8253173828125, |
|
"logps/rejected": -2203.4736328125, |
|
"loss": 0.1537, |
|
"rewards/accuracies": 0.9125000238418579, |
|
"rewards/chosen": -2.7755582332611084, |
|
"rewards/margins": 5.876898288726807, |
|
"rewards/rejected": -8.652456283569336, |
|
"step": 170 |
|
}, |
|
{ |
|
"epoch": 0.55, |
|
"grad_norm": 38.69272613867567, |
|
"learning_rate": 2.513311770373421e-07, |
|
"logits/chosen": -2.8833096027374268, |
|
"logits/rejected": -2.700117349624634, |
|
"logps/chosen": -1203.2437744140625, |
|
"logps/rejected": -2128.73876953125, |
|
"loss": 0.164, |
|
"rewards/accuracies": 0.956250011920929, |
|
"rewards/chosen": -1.8221490383148193, |
|
"rewards/margins": 5.741241455078125, |
|
"rewards/rejected": -7.563390254974365, |
|
"step": 180 |
|
}, |
|
{ |
|
"epoch": 0.58, |
|
"grad_norm": 43.40638062311321, |
|
"learning_rate": 2.2475064095527946e-07, |
|
"logits/chosen": -2.8215672969818115, |
|
"logits/rejected": -2.522494316101074, |
|
"logps/chosen": -1261.7806396484375, |
|
"logps/rejected": -2357.189453125, |
|
"loss": 0.1105, |
|
"rewards/accuracies": 0.9624999761581421, |
|
"rewards/chosen": -2.5550084114074707, |
|
"rewards/margins": 7.024233341217041, |
|
"rewards/rejected": -9.579240798950195, |
|
"step": 190 |
|
}, |
|
{ |
|
"epoch": 0.61, |
|
"grad_norm": 46.47610329796461, |
|
"learning_rate": 1.984561902003255e-07, |
|
"logits/chosen": -2.600424289703369, |
|
"logits/rejected": -2.3879284858703613, |
|
"logps/chosen": -1387.474365234375, |
|
"logps/rejected": -2327.524169921875, |
|
"loss": 0.1355, |
|
"rewards/accuracies": 0.9437500238418579, |
|
"rewards/chosen": -3.2148842811584473, |
|
"rewards/margins": 6.671014308929443, |
|
"rewards/rejected": -9.885897636413574, |
|
"step": 200 |
|
}, |
|
{ |
|
"epoch": 0.61, |
|
"eval_logits/chosen": -1.9005341529846191, |
|
"eval_logits/rejected": -1.5654257535934448, |
|
"eval_logps/chosen": -174.2285614013672, |
|
"eval_logps/rejected": -1813.7420654296875, |
|
"eval_loss": 0.05062888190150261, |
|
"eval_rewards/accuracies": 0.875, |
|
"eval_rewards/chosen": -0.8363977074623108, |
|
"eval_rewards/margins": 11.296485900878906, |
|
"eval_rewards/rejected": -12.13288402557373, |
|
"eval_runtime": 4.9098, |
|
"eval_samples_per_second": 4.888, |
|
"eval_steps_per_second": 0.204, |
|
"step": 200 |
|
}, |
|
{ |
|
"epoch": 0.64, |
|
"grad_norm": 31.32678074756829, |
|
"learning_rate": 1.7274575140626315e-07, |
|
"logits/chosen": -2.630878448486328, |
|
"logits/rejected": -2.360283136367798, |
|
"logps/chosen": -1274.6114501953125, |
|
"logps/rejected": -2422.227294921875, |
|
"loss": 0.107, |
|
"rewards/accuracies": 0.949999988079071, |
|
"rewards/chosen": -2.9166343212127686, |
|
"rewards/margins": 7.893813133239746, |
|
"rewards/rejected": -10.810447692871094, |
|
"step": 210 |
|
}, |
|
{ |
|
"epoch": 0.67, |
|
"grad_norm": 48.30372175049827, |
|
"learning_rate": 1.4791063411799937e-07, |
|
"logits/chosen": -2.758852481842041, |
|
"logits/rejected": -2.4495952129364014, |
|
"logps/chosen": -1242.4476318359375, |
|
"logps/rejected": -2313.494384765625, |
|
"loss": 0.1167, |
|
"rewards/accuracies": 0.9437500238418579, |
|
"rewards/chosen": -2.645324230194092, |
|
"rewards/margins": 7.094490051269531, |
|
"rewards/rejected": -9.739813804626465, |
|
"step": 220 |
|
}, |
|
{ |
|
"epoch": 0.7, |
|
"grad_norm": 31.566932885659007, |
|
"learning_rate": 1.2423223013801945e-07, |
|
"logits/chosen": -2.7269630432128906, |
|
"logits/rejected": -2.492689609527588, |
|
"logps/chosen": -1234.9451904296875, |
|
"logps/rejected": -2318.334228515625, |
|
"loss": 0.1129, |
|
"rewards/accuracies": 0.9750000238418579, |
|
"rewards/chosen": -2.171018123626709, |
|
"rewards/margins": 7.9699273109436035, |
|
"rewards/rejected": -10.140945434570312, |
|
"step": 230 |
|
}, |
|
{ |
|
"epoch": 0.73, |
|
"grad_norm": 34.83394111035656, |
|
"learning_rate": 1.0197882524482667e-07, |
|
"logits/chosen": -2.7251534461975098, |
|
"logits/rejected": -2.4403250217437744, |
|
"logps/chosen": -1231.4935302734375, |
|
"logps/rejected": -2440.34912109375, |
|
"loss": 0.1418, |
|
"rewards/accuracies": 0.9312499761581421, |
|
"rewards/chosen": -2.6753926277160645, |
|
"rewards/margins": 7.822265625, |
|
"rewards/rejected": -10.497659683227539, |
|
"step": 240 |
|
}, |
|
{ |
|
"epoch": 0.76, |
|
"grad_norm": 24.25621896047951, |
|
"learning_rate": 8.140255940787058e-08, |
|
"logits/chosen": -2.695079803466797, |
|
"logits/rejected": -2.338381290435791, |
|
"logps/chosen": -1329.83642578125, |
|
"logps/rejected": -2476.803955078125, |
|
"loss": 0.1016, |
|
"rewards/accuracies": 0.956250011920929, |
|
"rewards/chosen": -2.7425146102905273, |
|
"rewards/margins": 8.260651588439941, |
|
"rewards/rejected": -11.003166198730469, |
|
"step": 250 |
|
}, |
|
{ |
|
"epoch": 0.79, |
|
"grad_norm": 27.532348797533775, |
|
"learning_rate": 6.273656994094231e-08, |
|
"logits/chosen": -2.669456958770752, |
|
"logits/rejected": -2.3488214015960693, |
|
"logps/chosen": -1346.739013671875, |
|
"logps/rejected": -2527.728759765625, |
|
"loss": 0.0974, |
|
"rewards/accuracies": 0.949999988079071, |
|
"rewards/chosen": -3.164280891418457, |
|
"rewards/margins": 8.543807983398438, |
|
"rewards/rejected": -11.708089828491211, |
|
"step": 260 |
|
}, |
|
{ |
|
"epoch": 0.82, |
|
"grad_norm": 32.33656052797523, |
|
"learning_rate": 4.6192349963253137e-08, |
|
"logits/chosen": -2.675199031829834, |
|
"logits/rejected": -2.319917678833008, |
|
"logps/chosen": -1294.7154541015625, |
|
"logps/rejected": -2497.270263671875, |
|
"loss": 0.0645, |
|
"rewards/accuracies": 0.9937499761581421, |
|
"rewards/chosen": -3.0818674564361572, |
|
"rewards/margins": 7.933001518249512, |
|
"rewards/rejected": -11.014870643615723, |
|
"step": 270 |
|
}, |
|
{ |
|
"epoch": 0.85, |
|
"grad_norm": 29.09098674295539, |
|
"learning_rate": 3.195735209788528e-08, |
|
"logits/chosen": -2.6060080528259277, |
|
"logits/rejected": -2.3255152702331543, |
|
"logps/chosen": -1324.2171630859375, |
|
"logps/rejected": -2495.132080078125, |
|
"loss": 0.0796, |
|
"rewards/accuracies": 0.9750000238418579, |
|
"rewards/chosen": -3.3014044761657715, |
|
"rewards/margins": 7.963587760925293, |
|
"rewards/rejected": -11.264991760253906, |
|
"step": 280 |
|
}, |
|
{ |
|
"epoch": 0.88, |
|
"grad_norm": 50.71746791104581, |
|
"learning_rate": 2.019286455866981e-08, |
|
"logits/chosen": -2.5728938579559326, |
|
"logits/rejected": -2.3195109367370605, |
|
"logps/chosen": -1388.82861328125, |
|
"logps/rejected": -2512.4306640625, |
|
"loss": 0.0931, |
|
"rewards/accuracies": 0.9624999761581421, |
|
"rewards/chosen": -3.2404701709747314, |
|
"rewards/margins": 8.0997953414917, |
|
"rewards/rejected": -11.340266227722168, |
|
"step": 290 |
|
}, |
|
{ |
|
"epoch": 0.91, |
|
"grad_norm": 52.710422459009, |
|
"learning_rate": 1.1032183690276753e-08, |
|
"logits/chosen": -2.529834270477295, |
|
"logits/rejected": -2.2143425941467285, |
|
"logps/chosen": -1260.817626953125, |
|
"logps/rejected": -2584.9921875, |
|
"loss": 0.0848, |
|
"rewards/accuracies": 0.9750000238418579, |
|
"rewards/chosen": -3.3079543113708496, |
|
"rewards/margins": 9.132894515991211, |
|
"rewards/rejected": -12.440850257873535, |
|
"step": 300 |
|
}, |
|
{ |
|
"epoch": 0.91, |
|
"eval_logits/chosen": -1.9181619882583618, |
|
"eval_logits/rejected": -1.5808876752853394, |
|
"eval_logps/chosen": -168.04290771484375, |
|
"eval_logps/rejected": -2051.701416015625, |
|
"eval_loss": 0.04682515189051628, |
|
"eval_rewards/accuracies": 0.875, |
|
"eval_rewards/chosen": -0.7745410203933716, |
|
"eval_rewards/margins": 13.737937927246094, |
|
"eval_rewards/rejected": -14.512479782104492, |
|
"eval_runtime": 4.9321, |
|
"eval_samples_per_second": 4.866, |
|
"eval_steps_per_second": 0.203, |
|
"step": 300 |
|
}, |
|
{ |
|
"epoch": 0.95, |
|
"grad_norm": 72.2830720687719, |
|
"learning_rate": 4.579103667367384e-09, |
|
"logits/chosen": -2.5878891944885254, |
|
"logits/rejected": -2.2834665775299072, |
|
"logps/chosen": -1402.7425537109375, |
|
"logps/rejected": -2531.157958984375, |
|
"loss": 0.0942, |
|
"rewards/accuracies": 0.9437500238418579, |
|
"rewards/chosen": -3.4184463024139404, |
|
"rewards/margins": 8.603601455688477, |
|
"rewards/rejected": -12.022048950195312, |
|
"step": 310 |
|
}, |
|
{ |
|
"epoch": 0.98, |
|
"grad_norm": 30.83428232415201, |
|
"learning_rate": 9.067404651211807e-10, |
|
"logits/chosen": -2.603289842605591, |
|
"logits/rejected": -2.298339366912842, |
|
"logps/chosen": -1401.251708984375, |
|
"logps/rejected": -2568.3681640625, |
|
"loss": 0.1105, |
|
"rewards/accuracies": 0.9624999761581421, |
|
"rewards/chosen": -3.375328779220581, |
|
"rewards/margins": 8.309103965759277, |
|
"rewards/rejected": -11.684433937072754, |
|
"step": 320 |
|
}, |
|
{ |
|
"epoch": 1.0, |
|
"step": 328, |
|
"total_flos": 0.0, |
|
"train_loss": 0.23545521443209996, |
|
"train_runtime": 6883.5239, |
|
"train_samples_per_second": 3.046, |
|
"train_steps_per_second": 0.048 |
|
} |
|
], |
|
"logging_steps": 10, |
|
"max_steps": 328, |
|
"num_input_tokens_seen": 0, |
|
"num_train_epochs": 1, |
|
"save_steps": 100, |
|
"total_flos": 0.0, |
|
"train_batch_size": 8, |
|
"trial_name": null, |
|
"trial_params": null |
|
} |
|
|