|
{ |
|
"best_metric": null, |
|
"best_model_checkpoint": null, |
|
"epoch": 1.0, |
|
"eval_steps": 500, |
|
"global_step": 3821, |
|
"is_hyper_param_search": false, |
|
"is_local_process_zero": true, |
|
"is_world_process_zero": true, |
|
"log_history": [ |
|
{ |
|
"epoch": 0.00026171159382360636, |
|
"grad_norm": 12.705608949082016, |
|
"learning_rate": 1.3054830287206266e-09, |
|
"logits/chosen": -2.5182626247406006, |
|
"logits/rejected": -2.2643015384674072, |
|
"logps/chosen": -388.1086120605469, |
|
"logps/rejected": -98.67791748046875, |
|
"loss": 0.6931, |
|
"rewards/accuracies": 0.0, |
|
"rewards/chosen": 0.0, |
|
"rewards/margins": 0.0, |
|
"rewards/rejected": 0.0, |
|
"step": 1 |
|
}, |
|
{ |
|
"epoch": 0.0026171159382360636, |
|
"grad_norm": 11.140808468314031, |
|
"learning_rate": 1.3054830287206264e-08, |
|
"logits/chosen": -2.58882999420166, |
|
"logits/rejected": -2.5899062156677246, |
|
"logps/chosen": -243.39637756347656, |
|
"logps/rejected": -202.82931518554688, |
|
"loss": 0.6931, |
|
"rewards/accuracies": 0.4444444477558136, |
|
"rewards/chosen": 0.019962893798947334, |
|
"rewards/margins": 0.03951311111450195, |
|
"rewards/rejected": -0.01955021731555462, |
|
"step": 10 |
|
}, |
|
{ |
|
"epoch": 0.005234231876472127, |
|
"grad_norm": 11.144652062744898, |
|
"learning_rate": 2.610966057441253e-08, |
|
"logits/chosen": -2.672178030014038, |
|
"logits/rejected": -2.5746469497680664, |
|
"logps/chosen": -230.84091186523438, |
|
"logps/rejected": -228.9805145263672, |
|
"loss": 0.6932, |
|
"rewards/accuracies": 0.4749999940395355, |
|
"rewards/chosen": 0.009132862091064453, |
|
"rewards/margins": -0.06280727684497833, |
|
"rewards/rejected": 0.07194013893604279, |
|
"step": 20 |
|
}, |
|
{ |
|
"epoch": 0.007851347814708191, |
|
"grad_norm": 12.445247991485594, |
|
"learning_rate": 3.91644908616188e-08, |
|
"logits/chosen": -2.6407968997955322, |
|
"logits/rejected": -2.627769947052002, |
|
"logps/chosen": -324.41583251953125, |
|
"logps/rejected": -290.3148193359375, |
|
"loss": 0.6932, |
|
"rewards/accuracies": 0.375, |
|
"rewards/chosen": -0.03206072002649307, |
|
"rewards/margins": -0.08101508766412735, |
|
"rewards/rejected": 0.04895436763763428, |
|
"step": 30 |
|
}, |
|
{ |
|
"epoch": 0.010468463752944255, |
|
"grad_norm": 12.09719640540224, |
|
"learning_rate": 5.221932114882506e-08, |
|
"logits/chosen": -2.681063175201416, |
|
"logits/rejected": -2.5851457118988037, |
|
"logps/chosen": -293.2417907714844, |
|
"logps/rejected": -274.8295593261719, |
|
"loss": 0.6932, |
|
"rewards/accuracies": 0.5249999761581421, |
|
"rewards/chosen": -0.017620420083403587, |
|
"rewards/margins": -0.15502414107322693, |
|
"rewards/rejected": 0.1374037265777588, |
|
"step": 40 |
|
}, |
|
{ |
|
"epoch": 0.01308557969118032, |
|
"grad_norm": 11.599145099560147, |
|
"learning_rate": 6.527415143603133e-08, |
|
"logits/chosen": -2.767548084259033, |
|
"logits/rejected": -2.6930577754974365, |
|
"logps/chosen": -265.1833190917969, |
|
"logps/rejected": -234.08816528320312, |
|
"loss": 0.6931, |
|
"rewards/accuracies": 0.6499999761581421, |
|
"rewards/chosen": 0.22780108451843262, |
|
"rewards/margins": 0.1385040581226349, |
|
"rewards/rejected": 0.08929703384637833, |
|
"step": 50 |
|
}, |
|
{ |
|
"epoch": 0.015702695629416383, |
|
"grad_norm": 11.965007728798458, |
|
"learning_rate": 7.83289817232376e-08, |
|
"logits/chosen": -2.7711918354034424, |
|
"logits/rejected": -2.669393301010132, |
|
"logps/chosen": -297.89581298828125, |
|
"logps/rejected": -215.51119995117188, |
|
"loss": 0.6929, |
|
"rewards/accuracies": 0.5249999761581421, |
|
"rewards/chosen": 0.2271616905927658, |
|
"rewards/margins": 0.12766209244728088, |
|
"rewards/rejected": 0.09949960559606552, |
|
"step": 60 |
|
}, |
|
{ |
|
"epoch": 0.018319811567652448, |
|
"grad_norm": 11.022881833624007, |
|
"learning_rate": 9.138381201044386e-08, |
|
"logits/chosen": -2.879361867904663, |
|
"logits/rejected": -2.7757599353790283, |
|
"logps/chosen": -353.2497253417969, |
|
"logps/rejected": -311.20623779296875, |
|
"loss": 0.6928, |
|
"rewards/accuracies": 0.675000011920929, |
|
"rewards/chosen": 0.38858160376548767, |
|
"rewards/margins": 0.23797640204429626, |
|
"rewards/rejected": 0.1506052017211914, |
|
"step": 70 |
|
}, |
|
{ |
|
"epoch": 0.02093692750588851, |
|
"grad_norm": 11.840625592848136, |
|
"learning_rate": 1.0443864229765012e-07, |
|
"logits/chosen": -2.6824872493743896, |
|
"logits/rejected": -2.6427717208862305, |
|
"logps/chosen": -264.01666259765625, |
|
"logps/rejected": -247.1999053955078, |
|
"loss": 0.6928, |
|
"rewards/accuracies": 0.574999988079071, |
|
"rewards/chosen": 0.4467671811580658, |
|
"rewards/margins": 0.07854221016168594, |
|
"rewards/rejected": 0.36822497844696045, |
|
"step": 80 |
|
}, |
|
{ |
|
"epoch": 0.023554043444124574, |
|
"grad_norm": 9.022222501604846, |
|
"learning_rate": 1.174934725848564e-07, |
|
"logits/chosen": -2.6744754314422607, |
|
"logits/rejected": -2.594621181488037, |
|
"logps/chosen": -293.5453796386719, |
|
"logps/rejected": -247.10824584960938, |
|
"loss": 0.6919, |
|
"rewards/accuracies": 0.6000000238418579, |
|
"rewards/chosen": 0.5915378332138062, |
|
"rewards/margins": 0.33240383863449097, |
|
"rewards/rejected": 0.2591339647769928, |
|
"step": 90 |
|
}, |
|
{ |
|
"epoch": 0.02617115938236064, |
|
"grad_norm": 9.493186007745985, |
|
"learning_rate": 1.3054830287206266e-07, |
|
"logits/chosen": -2.6995768547058105, |
|
"logits/rejected": -2.698713779449463, |
|
"logps/chosen": -251.64797973632812, |
|
"logps/rejected": -263.9851379394531, |
|
"loss": 0.6915, |
|
"rewards/accuracies": 0.625, |
|
"rewards/chosen": 1.2346092462539673, |
|
"rewards/margins": 0.4688534736633301, |
|
"rewards/rejected": 0.7657557725906372, |
|
"step": 100 |
|
}, |
|
{ |
|
"epoch": 0.028788275320596704, |
|
"grad_norm": 10.644035033683487, |
|
"learning_rate": 1.4360313315926893e-07, |
|
"logits/chosen": -2.7012295722961426, |
|
"logits/rejected": -2.6298720836639404, |
|
"logps/chosen": -323.7200012207031, |
|
"logps/rejected": -251.99365234375, |
|
"loss": 0.6907, |
|
"rewards/accuracies": 0.6499999761581421, |
|
"rewards/chosen": 1.4460163116455078, |
|
"rewards/margins": 0.7907851934432983, |
|
"rewards/rejected": 0.6552310585975647, |
|
"step": 110 |
|
}, |
|
{ |
|
"epoch": 0.031405391258832765, |
|
"grad_norm": 12.593823193229781, |
|
"learning_rate": 1.566579634464752e-07, |
|
"logits/chosen": -2.8149755001068115, |
|
"logits/rejected": -2.693021535873413, |
|
"logps/chosen": -321.4913635253906, |
|
"logps/rejected": -211.90219116210938, |
|
"loss": 0.6885, |
|
"rewards/accuracies": 0.625, |
|
"rewards/chosen": 1.6065527200698853, |
|
"rewards/margins": 2.005385160446167, |
|
"rewards/rejected": -0.3988325595855713, |
|
"step": 120 |
|
}, |
|
{ |
|
"epoch": 0.03402250719706883, |
|
"grad_norm": 10.349524391749604, |
|
"learning_rate": 1.6971279373368143e-07, |
|
"logits/chosen": -2.6855309009552, |
|
"logits/rejected": -2.6297318935394287, |
|
"logps/chosen": -277.7900390625, |
|
"logps/rejected": -257.3739929199219, |
|
"loss": 0.6881, |
|
"rewards/accuracies": 0.7250000238418579, |
|
"rewards/chosen": 1.389769434928894, |
|
"rewards/margins": 1.9867759943008423, |
|
"rewards/rejected": -0.5970064401626587, |
|
"step": 130 |
|
}, |
|
{ |
|
"epoch": 0.036639623135304895, |
|
"grad_norm": 9.759987095180128, |
|
"learning_rate": 1.8276762402088773e-07, |
|
"logits/chosen": -2.6445629596710205, |
|
"logits/rejected": -2.583130121231079, |
|
"logps/chosen": -292.6871337890625, |
|
"logps/rejected": -262.7529296875, |
|
"loss": 0.6889, |
|
"rewards/accuracies": 0.6499999761581421, |
|
"rewards/chosen": 1.12567937374115, |
|
"rewards/margins": 2.6970019340515137, |
|
"rewards/rejected": -1.5713226795196533, |
|
"step": 140 |
|
}, |
|
{ |
|
"epoch": 0.03925673907354096, |
|
"grad_norm": 12.31960701088485, |
|
"learning_rate": 1.95822454308094e-07, |
|
"logits/chosen": -2.7052559852600098, |
|
"logits/rejected": -2.662947177886963, |
|
"logps/chosen": -236.67056274414062, |
|
"logps/rejected": -259.0706787109375, |
|
"loss": 0.6863, |
|
"rewards/accuracies": 0.550000011920929, |
|
"rewards/chosen": 0.3762800097465515, |
|
"rewards/margins": 1.9162286520004272, |
|
"rewards/rejected": -1.5399487018585205, |
|
"step": 150 |
|
}, |
|
{ |
|
"epoch": 0.04187385501177702, |
|
"grad_norm": 11.677562237104198, |
|
"learning_rate": 2.0887728459530023e-07, |
|
"logits/chosen": -2.7215421199798584, |
|
"logits/rejected": -2.596179962158203, |
|
"logps/chosen": -308.93212890625, |
|
"logps/rejected": -228.4113311767578, |
|
"loss": 0.6826, |
|
"rewards/accuracies": 0.8500000238418579, |
|
"rewards/chosen": 2.3878512382507324, |
|
"rewards/margins": 6.691712379455566, |
|
"rewards/rejected": -4.303861618041992, |
|
"step": 160 |
|
}, |
|
{ |
|
"epoch": 0.04449097095001309, |
|
"grad_norm": 10.693351476475623, |
|
"learning_rate": 2.2193211488250652e-07, |
|
"logits/chosen": -2.7206406593322754, |
|
"logits/rejected": -2.7074098587036133, |
|
"logps/chosen": -292.83819580078125, |
|
"logps/rejected": -281.1519775390625, |
|
"loss": 0.6824, |
|
"rewards/accuracies": 0.6000000238418579, |
|
"rewards/chosen": -1.645533800125122, |
|
"rewards/margins": 2.851980209350586, |
|
"rewards/rejected": -4.497513771057129, |
|
"step": 170 |
|
}, |
|
{ |
|
"epoch": 0.04710808688824915, |
|
"grad_norm": 11.109776793504, |
|
"learning_rate": 2.349869451697128e-07, |
|
"logits/chosen": -2.625349521636963, |
|
"logits/rejected": -2.555849552154541, |
|
"logps/chosen": -295.85986328125, |
|
"logps/rejected": -283.39111328125, |
|
"loss": 0.6795, |
|
"rewards/accuracies": 0.625, |
|
"rewards/chosen": -4.437504768371582, |
|
"rewards/margins": 6.071015357971191, |
|
"rewards/rejected": -10.508520126342773, |
|
"step": 180 |
|
}, |
|
{ |
|
"epoch": 0.04972520282648522, |
|
"grad_norm": 11.94197957738407, |
|
"learning_rate": 2.4804177545691903e-07, |
|
"logits/chosen": -2.57228684425354, |
|
"logits/rejected": -2.4936299324035645, |
|
"logps/chosen": -262.52410888671875, |
|
"logps/rejected": -211.85018920898438, |
|
"loss": 0.6775, |
|
"rewards/accuracies": 0.625, |
|
"rewards/chosen": -3.077005386352539, |
|
"rewards/margins": 6.298064231872559, |
|
"rewards/rejected": -9.375069618225098, |
|
"step": 190 |
|
}, |
|
{ |
|
"epoch": 0.05234231876472128, |
|
"grad_norm": 12.061365139178056, |
|
"learning_rate": 2.610966057441253e-07, |
|
"logits/chosen": -2.567936658859253, |
|
"logits/rejected": -2.4781880378723145, |
|
"logps/chosen": -234.6671142578125, |
|
"logps/rejected": -221.07540893554688, |
|
"loss": 0.677, |
|
"rewards/accuracies": 0.44999998807907104, |
|
"rewards/chosen": -7.456887245178223, |
|
"rewards/margins": 0.09982886165380478, |
|
"rewards/rejected": -7.556715965270996, |
|
"step": 200 |
|
}, |
|
{ |
|
"epoch": 0.05495943470295734, |
|
"grad_norm": 16.567960601555676, |
|
"learning_rate": 2.7415143603133156e-07, |
|
"logits/chosen": -2.749500036239624, |
|
"logits/rejected": -2.6309502124786377, |
|
"logps/chosen": -242.54910278320312, |
|
"logps/rejected": -182.82608032226562, |
|
"loss": 0.6795, |
|
"rewards/accuracies": 0.625, |
|
"rewards/chosen": -11.374174118041992, |
|
"rewards/margins": 6.828550815582275, |
|
"rewards/rejected": -18.20272445678711, |
|
"step": 210 |
|
}, |
|
{ |
|
"epoch": 0.05757655064119341, |
|
"grad_norm": 19.24294269180777, |
|
"learning_rate": 2.8720626631853785e-07, |
|
"logits/chosen": -2.706005811691284, |
|
"logits/rejected": -2.698056936264038, |
|
"logps/chosen": -269.6457824707031, |
|
"logps/rejected": -219.22647094726562, |
|
"loss": 0.6726, |
|
"rewards/accuracies": 0.699999988079071, |
|
"rewards/chosen": -11.772501945495605, |
|
"rewards/margins": 10.55135726928711, |
|
"rewards/rejected": -22.32386016845703, |
|
"step": 220 |
|
}, |
|
{ |
|
"epoch": 0.06019366657942947, |
|
"grad_norm": 16.374157225946007, |
|
"learning_rate": 3.002610966057441e-07, |
|
"logits/chosen": -2.663548707962036, |
|
"logits/rejected": -2.6067066192626953, |
|
"logps/chosen": -344.29693603515625, |
|
"logps/rejected": -307.93218994140625, |
|
"loss": 0.6696, |
|
"rewards/accuracies": 0.699999988079071, |
|
"rewards/chosen": 0.5850511789321899, |
|
"rewards/margins": 10.634298324584961, |
|
"rewards/rejected": -10.049245834350586, |
|
"step": 230 |
|
}, |
|
{ |
|
"epoch": 0.06281078251766553, |
|
"grad_norm": 16.386410614457457, |
|
"learning_rate": 3.133159268929504e-07, |
|
"logits/chosen": -2.657069683074951, |
|
"logits/rejected": -2.559688091278076, |
|
"logps/chosen": -230.8714599609375, |
|
"logps/rejected": -256.7811279296875, |
|
"loss": 0.6718, |
|
"rewards/accuracies": 0.7250000238418579, |
|
"rewards/chosen": 1.7591073513031006, |
|
"rewards/margins": 12.968846321105957, |
|
"rewards/rejected": -11.209738731384277, |
|
"step": 240 |
|
}, |
|
{ |
|
"epoch": 0.06542789845590159, |
|
"grad_norm": 24.693819068780147, |
|
"learning_rate": 3.263707571801567e-07, |
|
"logits/chosen": -2.754833698272705, |
|
"logits/rejected": -2.6114320755004883, |
|
"logps/chosen": -286.67083740234375, |
|
"logps/rejected": -250.2057647705078, |
|
"loss": 0.6632, |
|
"rewards/accuracies": 0.625, |
|
"rewards/chosen": -10.266336441040039, |
|
"rewards/margins": 13.779983520507812, |
|
"rewards/rejected": -24.046321868896484, |
|
"step": 250 |
|
}, |
|
{ |
|
"epoch": 0.06804501439413765, |
|
"grad_norm": 13.195073062662031, |
|
"learning_rate": 3.3942558746736286e-07, |
|
"logits/chosen": -2.57175350189209, |
|
"logits/rejected": -2.367932081222534, |
|
"logps/chosen": -289.5040588378906, |
|
"logps/rejected": -241.59048461914062, |
|
"loss": 0.6654, |
|
"rewards/accuracies": 0.7250000238418579, |
|
"rewards/chosen": -20.12925148010254, |
|
"rewards/margins": 20.86807632446289, |
|
"rewards/rejected": -40.9973258972168, |
|
"step": 260 |
|
}, |
|
{ |
|
"epoch": 0.07066213033237373, |
|
"grad_norm": 22.100525575015013, |
|
"learning_rate": 3.5248041775456916e-07, |
|
"logits/chosen": -2.6396119594573975, |
|
"logits/rejected": -2.627289295196533, |
|
"logps/chosen": -342.06005859375, |
|
"logps/rejected": -355.39569091796875, |
|
"loss": 0.6569, |
|
"rewards/accuracies": 0.699999988079071, |
|
"rewards/chosen": -27.9144287109375, |
|
"rewards/margins": 18.81797981262207, |
|
"rewards/rejected": -46.7324104309082, |
|
"step": 270 |
|
}, |
|
{ |
|
"epoch": 0.07327924627060979, |
|
"grad_norm": 14.329830569747937, |
|
"learning_rate": 3.6553524804177545e-07, |
|
"logits/chosen": -2.6874194145202637, |
|
"logits/rejected": -2.605147361755371, |
|
"logps/chosen": -261.7181091308594, |
|
"logps/rejected": -301.5287780761719, |
|
"loss": 0.6654, |
|
"rewards/accuracies": 0.625, |
|
"rewards/chosen": -1.3762327432632446, |
|
"rewards/margins": 13.867558479309082, |
|
"rewards/rejected": -15.243791580200195, |
|
"step": 280 |
|
}, |
|
{ |
|
"epoch": 0.07589636220884585, |
|
"grad_norm": 11.577264424915722, |
|
"learning_rate": 3.785900783289817e-07, |
|
"logits/chosen": -2.5518436431884766, |
|
"logits/rejected": -2.5594162940979004, |
|
"logps/chosen": -322.40875244140625, |
|
"logps/rejected": -293.8119201660156, |
|
"loss": 0.6646, |
|
"rewards/accuracies": 0.7250000238418579, |
|
"rewards/chosen": 5.861729621887207, |
|
"rewards/margins": 15.61244010925293, |
|
"rewards/rejected": -9.750709533691406, |
|
"step": 290 |
|
}, |
|
{ |
|
"epoch": 0.07851347814708191, |
|
"grad_norm": 12.227671537308453, |
|
"learning_rate": 3.91644908616188e-07, |
|
"logits/chosen": -2.728774309158325, |
|
"logits/rejected": -2.69193696975708, |
|
"logps/chosen": -234.78945922851562, |
|
"logps/rejected": -249.3471221923828, |
|
"loss": 0.6746, |
|
"rewards/accuracies": 0.6499999761581421, |
|
"rewards/chosen": -2.2874860763549805, |
|
"rewards/margins": 10.886515617370605, |
|
"rewards/rejected": -13.174001693725586, |
|
"step": 300 |
|
}, |
|
{ |
|
"epoch": 0.08113059408531798, |
|
"grad_norm": 10.865740609371638, |
|
"learning_rate": 4.046997389033943e-07, |
|
"logits/chosen": -2.755521297454834, |
|
"logits/rejected": -2.7999794483184814, |
|
"logps/chosen": -264.28765869140625, |
|
"logps/rejected": -330.6804504394531, |
|
"loss": 0.6515, |
|
"rewards/accuracies": 0.6000000238418579, |
|
"rewards/chosen": -9.909737586975098, |
|
"rewards/margins": 20.648042678833008, |
|
"rewards/rejected": -30.55777931213379, |
|
"step": 310 |
|
}, |
|
{ |
|
"epoch": 0.08374771002355404, |
|
"grad_norm": 17.405613711098255, |
|
"learning_rate": 4.1775456919060046e-07, |
|
"logits/chosen": -2.6872105598449707, |
|
"logits/rejected": -2.545166015625, |
|
"logps/chosen": -249.74252319335938, |
|
"logps/rejected": -246.7162628173828, |
|
"loss": 0.6584, |
|
"rewards/accuracies": 0.6000000238418579, |
|
"rewards/chosen": -11.012734413146973, |
|
"rewards/margins": 24.713336944580078, |
|
"rewards/rejected": -35.72606658935547, |
|
"step": 320 |
|
}, |
|
{ |
|
"epoch": 0.08636482596179011, |
|
"grad_norm": 16.876842691559403, |
|
"learning_rate": 4.3080939947780675e-07, |
|
"logits/chosen": -2.8166439533233643, |
|
"logits/rejected": -2.660918712615967, |
|
"logps/chosen": -333.4720153808594, |
|
"logps/rejected": -322.7220153808594, |
|
"loss": 0.6438, |
|
"rewards/accuracies": 0.8500000238418579, |
|
"rewards/chosen": -21.886640548706055, |
|
"rewards/margins": 36.4941520690918, |
|
"rewards/rejected": -58.38079833984375, |
|
"step": 330 |
|
}, |
|
{ |
|
"epoch": 0.08898194190002617, |
|
"grad_norm": 20.055916219424066, |
|
"learning_rate": 4.4386422976501305e-07, |
|
"logits/chosen": -2.64066219329834, |
|
"logits/rejected": -2.6652839183807373, |
|
"logps/chosen": -297.4921875, |
|
"logps/rejected": -326.902587890625, |
|
"loss": 0.6477, |
|
"rewards/accuracies": 0.75, |
|
"rewards/chosen": -30.227951049804688, |
|
"rewards/margins": 36.16551971435547, |
|
"rewards/rejected": -66.39347839355469, |
|
"step": 340 |
|
}, |
|
{ |
|
"epoch": 0.09159905783826224, |
|
"grad_norm": 18.03084139024805, |
|
"learning_rate": 4.569190600522193e-07, |
|
"logits/chosen": -2.5461313724517822, |
|
"logits/rejected": -2.3993871212005615, |
|
"logps/chosen": -333.36273193359375, |
|
"logps/rejected": -314.22796630859375, |
|
"loss": 0.661, |
|
"rewards/accuracies": 0.75, |
|
"rewards/chosen": -50.57813262939453, |
|
"rewards/margins": 34.174285888671875, |
|
"rewards/rejected": -84.7524185180664, |
|
"step": 350 |
|
}, |
|
{ |
|
"epoch": 0.0942161737764983, |
|
"grad_norm": 21.82315344421899, |
|
"learning_rate": 4.699738903394256e-07, |
|
"logits/chosen": -2.620579242706299, |
|
"logits/rejected": -2.389979839324951, |
|
"logps/chosen": -332.0915832519531, |
|
"logps/rejected": -292.027099609375, |
|
"loss": 0.6535, |
|
"rewards/accuracies": 0.675000011920929, |
|
"rewards/chosen": -47.5004997253418, |
|
"rewards/margins": 15.382966995239258, |
|
"rewards/rejected": -62.88347244262695, |
|
"step": 360 |
|
}, |
|
{ |
|
"epoch": 0.09683328971473436, |
|
"grad_norm": 15.226791043551732, |
|
"learning_rate": 4.830287206266319e-07, |
|
"logits/chosen": -2.47334623336792, |
|
"logits/rejected": -2.339829921722412, |
|
"logps/chosen": -314.9557189941406, |
|
"logps/rejected": -340.21624755859375, |
|
"loss": 0.6471, |
|
"rewards/accuracies": 0.6000000238418579, |
|
"rewards/chosen": -24.457332611083984, |
|
"rewards/margins": 20.158397674560547, |
|
"rewards/rejected": -44.61573028564453, |
|
"step": 370 |
|
}, |
|
{ |
|
"epoch": 0.09945040565297043, |
|
"grad_norm": 23.15807260624967, |
|
"learning_rate": 4.960835509138381e-07, |
|
"logits/chosen": -2.4241931438446045, |
|
"logits/rejected": -2.4154956340789795, |
|
"logps/chosen": -252.0162811279297, |
|
"logps/rejected": -317.27960205078125, |
|
"loss": 0.6577, |
|
"rewards/accuracies": 0.625, |
|
"rewards/chosen": -25.363056182861328, |
|
"rewards/margins": 27.764968872070312, |
|
"rewards/rejected": -53.128028869628906, |
|
"step": 380 |
|
}, |
|
{ |
|
"epoch": 0.1020675215912065, |
|
"grad_norm": 14.367634528764981, |
|
"learning_rate": 4.999948856244767e-07, |
|
"logits/chosen": -2.4574198722839355, |
|
"logits/rejected": -2.225590705871582, |
|
"logps/chosen": -278.4183349609375, |
|
"logps/rejected": -297.3124084472656, |
|
"loss": 0.6508, |
|
"rewards/accuracies": 0.7749999761581421, |
|
"rewards/chosen": -23.85447120666504, |
|
"rewards/margins": 30.555782318115234, |
|
"rewards/rejected": -54.410255432128906, |
|
"step": 390 |
|
}, |
|
{ |
|
"epoch": 0.10468463752944256, |
|
"grad_norm": 18.410363856260208, |
|
"learning_rate": 4.999698361256577e-07, |
|
"logits/chosen": -2.3174118995666504, |
|
"logits/rejected": -2.241575002670288, |
|
"logps/chosen": -293.21435546875, |
|
"logps/rejected": -377.48651123046875, |
|
"loss": 0.6646, |
|
"rewards/accuracies": 0.7250000238418579, |
|
"rewards/chosen": -21.861358642578125, |
|
"rewards/margins": 39.63842010498047, |
|
"rewards/rejected": -61.499778747558594, |
|
"step": 400 |
|
}, |
|
{ |
|
"epoch": 0.10730175346767862, |
|
"grad_norm": 21.26097793554471, |
|
"learning_rate": 4.99923914217458e-07, |
|
"logits/chosen": -2.6152307987213135, |
|
"logits/rejected": -2.2710635662078857, |
|
"logps/chosen": -290.6064453125, |
|
"logps/rejected": -356.77655029296875, |
|
"loss": 0.6567, |
|
"rewards/accuracies": 0.699999988079071, |
|
"rewards/chosen": -33.422264099121094, |
|
"rewards/margins": 62.78270721435547, |
|
"rewards/rejected": -96.20496368408203, |
|
"step": 410 |
|
}, |
|
{ |
|
"epoch": 0.10991886940591468, |
|
"grad_norm": 23.417576747738657, |
|
"learning_rate": 4.99857123734344e-07, |
|
"logits/chosen": -2.6153998374938965, |
|
"logits/rejected": -2.670661449432373, |
|
"logps/chosen": -321.48553466796875, |
|
"logps/rejected": -420.5201110839844, |
|
"loss": 0.6529, |
|
"rewards/accuracies": 0.625, |
|
"rewards/chosen": -60.42144012451172, |
|
"rewards/margins": 39.77070236206055, |
|
"rewards/rejected": -100.1921615600586, |
|
"step": 420 |
|
}, |
|
{ |
|
"epoch": 0.11253598534415074, |
|
"grad_norm": 32.63068698649212, |
|
"learning_rate": 4.997694702533016e-07, |
|
"logits/chosen": -2.52959942817688, |
|
"logits/rejected": -2.5021653175354004, |
|
"logps/chosen": -405.32867431640625, |
|
"logps/rejected": -397.52008056640625, |
|
"loss": 0.6494, |
|
"rewards/accuracies": 0.699999988079071, |
|
"rewards/chosen": -50.44035720825195, |
|
"rewards/margins": 55.264503479003906, |
|
"rewards/rejected": -105.7048568725586, |
|
"step": 430 |
|
}, |
|
{ |
|
"epoch": 0.11515310128238682, |
|
"grad_norm": 30.402667110459582, |
|
"learning_rate": 4.996609610933712e-07, |
|
"logits/chosen": -1.9839417934417725, |
|
"logits/rejected": -1.6886985301971436, |
|
"logps/chosen": -308.8043518066406, |
|
"logps/rejected": -316.9330749511719, |
|
"loss": 0.6746, |
|
"rewards/accuracies": 0.675000011920929, |
|
"rewards/chosen": -92.59898376464844, |
|
"rewards/margins": 24.77956771850586, |
|
"rewards/rejected": -117.3785400390625, |
|
"step": 440 |
|
}, |
|
{ |
|
"epoch": 0.11777021722062288, |
|
"grad_norm": 58.30863783887881, |
|
"learning_rate": 4.995316053150366e-07, |
|
"logits/chosen": -1.65292489528656, |
|
"logits/rejected": -1.3604259490966797, |
|
"logps/chosen": -400.2745361328125, |
|
"logps/rejected": -401.6268310546875, |
|
"loss": 0.6555, |
|
"rewards/accuracies": 0.75, |
|
"rewards/chosen": -98.50657653808594, |
|
"rewards/margins": 38.124855041503906, |
|
"rewards/rejected": -136.6314239501953, |
|
"step": 450 |
|
}, |
|
{ |
|
"epoch": 0.12038733315885894, |
|
"grad_norm": 30.609564022713695, |
|
"learning_rate": 4.99381413719468e-07, |
|
"logits/chosen": -2.0271124839782715, |
|
"logits/rejected": -1.5985970497131348, |
|
"logps/chosen": -309.2818603515625, |
|
"logps/rejected": -336.6201171875, |
|
"loss": 0.652, |
|
"rewards/accuracies": 0.7250000238418579, |
|
"rewards/chosen": -56.03574752807617, |
|
"rewards/margins": 49.059043884277344, |
|
"rewards/rejected": -105.09478759765625, |
|
"step": 460 |
|
}, |
|
{ |
|
"epoch": 0.123004449097095, |
|
"grad_norm": 25.115747814836954, |
|
"learning_rate": 4.992103988476205e-07, |
|
"logits/chosen": -1.7523378133773804, |
|
"logits/rejected": -1.3779999017715454, |
|
"logps/chosen": -346.60150146484375, |
|
"logps/rejected": -356.7440490722656, |
|
"loss": 0.6504, |
|
"rewards/accuracies": 0.675000011920929, |
|
"rewards/chosen": -83.8178939819336, |
|
"rewards/margins": 50.205230712890625, |
|
"rewards/rejected": -134.0231170654297, |
|
"step": 470 |
|
}, |
|
{ |
|
"epoch": 0.12562156503533106, |
|
"grad_norm": 30.622381148232613, |
|
"learning_rate": 4.990185749791864e-07, |
|
"logits/chosen": -1.7865359783172607, |
|
"logits/rejected": -1.0789787769317627, |
|
"logps/chosen": -356.77484130859375, |
|
"logps/rejected": -287.66925048828125, |
|
"loss": 0.6493, |
|
"rewards/accuracies": 0.699999988079071, |
|
"rewards/chosen": -70.98980712890625, |
|
"rewards/margins": 35.558433532714844, |
|
"rewards/rejected": -106.5482406616211, |
|
"step": 480 |
|
}, |
|
{ |
|
"epoch": 0.12823868097356714, |
|
"grad_norm": 24.535072604494577, |
|
"learning_rate": 4.988059581314039e-07, |
|
"logits/chosen": -1.7096989154815674, |
|
"logits/rejected": -1.8498868942260742, |
|
"logps/chosen": -335.67730712890625, |
|
"logps/rejected": -398.1258850097656, |
|
"loss": 0.6407, |
|
"rewards/accuracies": 0.675000011920929, |
|
"rewards/chosen": -55.123626708984375, |
|
"rewards/margins": 46.74440383911133, |
|
"rewards/rejected": -101.86802673339844, |
|
"step": 490 |
|
}, |
|
{ |
|
"epoch": 0.13085579691180318, |
|
"grad_norm": 91.50022744374398, |
|
"learning_rate": 4.985725660577184e-07, |
|
"logits/chosen": -1.8485355377197266, |
|
"logits/rejected": -1.5357762575149536, |
|
"logps/chosen": -317.83135986328125, |
|
"logps/rejected": -341.1956481933594, |
|
"loss": 0.6446, |
|
"rewards/accuracies": 0.6499999761581421, |
|
"rewards/chosen": -70.9972152709961, |
|
"rewards/margins": 18.12459373474121, |
|
"rewards/rejected": -89.1218032836914, |
|
"step": 500 |
|
}, |
|
{ |
|
"epoch": 0.13347291285003926, |
|
"grad_norm": 18.189747979700574, |
|
"learning_rate": 4.983184182463008e-07, |
|
"logits/chosen": -2.1394052505493164, |
|
"logits/rejected": -2.0640697479248047, |
|
"logps/chosen": -420.4029846191406, |
|
"logps/rejected": -425.334228515625, |
|
"loss": 0.6465, |
|
"rewards/accuracies": 0.675000011920929, |
|
"rewards/chosen": -73.95565032958984, |
|
"rewards/margins": 44.018211364746094, |
|
"rewards/rejected": -117.97386169433594, |
|
"step": 510 |
|
}, |
|
{ |
|
"epoch": 0.1360900287882753, |
|
"grad_norm": 43.16282028958125, |
|
"learning_rate": 4.980435359184203e-07, |
|
"logits/chosen": -2.4100048542022705, |
|
"logits/rejected": -2.3749964237213135, |
|
"logps/chosen": -322.7026062011719, |
|
"logps/rejected": -347.69366455078125, |
|
"loss": 0.6603, |
|
"rewards/accuracies": 0.574999988079071, |
|
"rewards/chosen": -54.79851531982422, |
|
"rewards/margins": 15.417330741882324, |
|
"rewards/rejected": -70.21583557128906, |
|
"step": 520 |
|
}, |
|
{ |
|
"epoch": 0.13870714472651138, |
|
"grad_norm": 29.05298017877607, |
|
"learning_rate": 4.977479420266723e-07, |
|
"logits/chosen": -2.0409185886383057, |
|
"logits/rejected": -1.8864901065826416, |
|
"logps/chosen": -267.41387939453125, |
|
"logps/rejected": -322.8434143066406, |
|
"loss": 0.649, |
|
"rewards/accuracies": 0.6499999761581421, |
|
"rewards/chosen": -43.029293060302734, |
|
"rewards/margins": 38.46556854248047, |
|
"rewards/rejected": -81.49485778808594, |
|
"step": 530 |
|
}, |
|
{ |
|
"epoch": 0.14132426066474746, |
|
"grad_norm": 31.793699019536305, |
|
"learning_rate": 4.974316612530614e-07, |
|
"logits/chosen": -1.8414909839630127, |
|
"logits/rejected": -1.3766376972198486, |
|
"logps/chosen": -335.9708557128906, |
|
"logps/rejected": -364.6490478515625, |
|
"loss": 0.6363, |
|
"rewards/accuracies": 0.675000011920929, |
|
"rewards/chosen": -55.383262634277344, |
|
"rewards/margins": 62.95615768432617, |
|
"rewards/rejected": -118.33943176269531, |
|
"step": 540 |
|
}, |
|
{ |
|
"epoch": 0.1439413766029835, |
|
"grad_norm": 44.57978704229952, |
|
"learning_rate": 4.970947200069415e-07, |
|
"logits/chosen": -1.5717941522598267, |
|
"logits/rejected": -0.947163462638855, |
|
"logps/chosen": -305.5942687988281, |
|
"logps/rejected": -333.8151550292969, |
|
"loss": 0.6415, |
|
"rewards/accuracies": 0.875, |
|
"rewards/chosen": -57.449851989746094, |
|
"rewards/margins": 67.77587890625, |
|
"rewards/rejected": -125.2257308959961, |
|
"step": 550 |
|
}, |
|
{ |
|
"epoch": 0.14655849254121958, |
|
"grad_norm": 20.95587939868339, |
|
"learning_rate": 4.967371464228095e-07, |
|
"logits/chosen": -1.2510230541229248, |
|
"logits/rejected": -0.9176260232925415, |
|
"logps/chosen": -392.3152160644531, |
|
"logps/rejected": -415.76568603515625, |
|
"loss": 0.6309, |
|
"rewards/accuracies": 0.75, |
|
"rewards/chosen": -86.01702880859375, |
|
"rewards/margins": 61.00504684448242, |
|
"rewards/rejected": -147.02207946777344, |
|
"step": 560 |
|
}, |
|
{ |
|
"epoch": 0.14917560847945563, |
|
"grad_norm": 21.753739466816693, |
|
"learning_rate": 4.963589703579569e-07, |
|
"logits/chosen": -1.6249122619628906, |
|
"logits/rejected": -1.1825764179229736, |
|
"logps/chosen": -324.12762451171875, |
|
"logps/rejected": -394.82171630859375, |
|
"loss": 0.6297, |
|
"rewards/accuracies": 0.699999988079071, |
|
"rewards/chosen": -60.056861877441406, |
|
"rewards/margins": 62.51560592651367, |
|
"rewards/rejected": -122.57247161865234, |
|
"step": 570 |
|
}, |
|
{ |
|
"epoch": 0.1517927244176917, |
|
"grad_norm": 29.488789126329046, |
|
"learning_rate": 4.959602233899761e-07, |
|
"logits/chosen": -1.5013480186462402, |
|
"logits/rejected": -0.5311890840530396, |
|
"logps/chosen": -343.14666748046875, |
|
"logps/rejected": -399.1929016113281, |
|
"loss": 0.6371, |
|
"rewards/accuracies": 0.699999988079071, |
|
"rewards/chosen": -66.63851928710938, |
|
"rewards/margins": 59.16364669799805, |
|
"rewards/rejected": -125.80216217041016, |
|
"step": 580 |
|
}, |
|
{ |
|
"epoch": 0.15440984035592778, |
|
"grad_norm": 19.7981594005066, |
|
"learning_rate": 4.955409388141243e-07, |
|
"logits/chosen": -1.3217118978500366, |
|
"logits/rejected": -0.3744869530200958, |
|
"logps/chosen": -431.08074951171875, |
|
"logps/rejected": -447.07708740234375, |
|
"loss": 0.6474, |
|
"rewards/accuracies": 0.75, |
|
"rewards/chosen": -82.43335723876953, |
|
"rewards/margins": 65.14683532714844, |
|
"rewards/rejected": -147.58018493652344, |
|
"step": 590 |
|
}, |
|
{ |
|
"epoch": 0.15702695629416383, |
|
"grad_norm": 36.104735427867645, |
|
"learning_rate": 4.951011516405429e-07, |
|
"logits/chosen": -2.091003894805908, |
|
"logits/rejected": -1.4518930912017822, |
|
"logps/chosen": -366.91790771484375, |
|
"logps/rejected": -385.13629150390625, |
|
"loss": 0.6269, |
|
"rewards/accuracies": 0.6499999761581421, |
|
"rewards/chosen": -68.24613952636719, |
|
"rewards/margins": 54.08154296875, |
|
"rewards/rejected": -122.32768249511719, |
|
"step": 600 |
|
}, |
|
{ |
|
"epoch": 0.1596440722323999, |
|
"grad_norm": 26.905150575209632, |
|
"learning_rate": 4.946408985913344e-07, |
|
"logits/chosen": -1.9128236770629883, |
|
"logits/rejected": -1.008561372756958, |
|
"logps/chosen": -354.6834716796875, |
|
"logps/rejected": -329.9129638671875, |
|
"loss": 0.6369, |
|
"rewards/accuracies": 0.800000011920929, |
|
"rewards/chosen": -48.14597702026367, |
|
"rewards/margins": 71.0589370727539, |
|
"rewards/rejected": -119.20491027832031, |
|
"step": 610 |
|
}, |
|
{ |
|
"epoch": 0.16226118817063595, |
|
"grad_norm": 34.58032776342021, |
|
"learning_rate": 4.941602180974958e-07, |
|
"logits/chosen": -1.3618040084838867, |
|
"logits/rejected": -0.6716572642326355, |
|
"logps/chosen": -314.2044982910156, |
|
"logps/rejected": -350.322021484375, |
|
"loss": 0.6578, |
|
"rewards/accuracies": 0.800000011920929, |
|
"rewards/chosen": -67.95203399658203, |
|
"rewards/margins": 61.99140548706055, |
|
"rewards/rejected": -129.9434356689453, |
|
"step": 620 |
|
}, |
|
{ |
|
"epoch": 0.16487830410887203, |
|
"grad_norm": 50.70972774376307, |
|
"learning_rate": 4.936591502957101e-07, |
|
"logits/chosen": -1.437552809715271, |
|
"logits/rejected": -1.3617020845413208, |
|
"logps/chosen": -226.73690795898438, |
|
"logps/rejected": -290.77044677734375, |
|
"loss": 0.6448, |
|
"rewards/accuracies": 0.625, |
|
"rewards/chosen": -49.09408950805664, |
|
"rewards/margins": 37.69247817993164, |
|
"rewards/rejected": -86.78656768798828, |
|
"step": 630 |
|
}, |
|
{ |
|
"epoch": 0.16749542004710807, |
|
"grad_norm": 15.357094809679237, |
|
"learning_rate": 4.931377370249945e-07, |
|
"logits/chosen": -1.46652090549469, |
|
"logits/rejected": -0.865250289440155, |
|
"logps/chosen": -293.2486572265625, |
|
"logps/rejected": -327.9940490722656, |
|
"loss": 0.6425, |
|
"rewards/accuracies": 0.75, |
|
"rewards/chosen": -45.150516510009766, |
|
"rewards/margins": 41.07079315185547, |
|
"rewards/rejected": -86.2213134765625, |
|
"step": 640 |
|
}, |
|
{ |
|
"epoch": 0.17011253598534415, |
|
"grad_norm": 24.168035949449166, |
|
"learning_rate": 4.925960218232072e-07, |
|
"logits/chosen": -0.6024962663650513, |
|
"logits/rejected": -0.08184865862131119, |
|
"logps/chosen": -361.3127746582031, |
|
"logps/rejected": -382.85845947265625, |
|
"loss": 0.6723, |
|
"rewards/accuracies": 0.824999988079071, |
|
"rewards/chosen": -85.62727355957031, |
|
"rewards/margins": 49.480709075927734, |
|
"rewards/rejected": -135.10800170898438, |
|
"step": 650 |
|
}, |
|
{ |
|
"epoch": 0.17272965192358022, |
|
"grad_norm": 27.513683012496575, |
|
"learning_rate": 4.920340499234116e-07, |
|
"logits/chosen": -0.9328662157058716, |
|
"logits/rejected": -0.03643731027841568, |
|
"logps/chosen": -390.78521728515625, |
|
"logps/rejected": -394.0958557128906, |
|
"loss": 0.6255, |
|
"rewards/accuracies": 0.699999988079071, |
|
"rewards/chosen": -104.32292175292969, |
|
"rewards/margins": 56.33015823364258, |
|
"rewards/rejected": -160.653076171875, |
|
"step": 660 |
|
}, |
|
{ |
|
"epoch": 0.17534676786181627, |
|
"grad_norm": 21.525597755133532, |
|
"learning_rate": 4.914518682500995e-07, |
|
"logits/chosen": -0.8200181722640991, |
|
"logits/rejected": -0.7997004985809326, |
|
"logps/chosen": -367.11407470703125, |
|
"logps/rejected": -425.20501708984375, |
|
"loss": 0.6432, |
|
"rewards/accuracies": 0.699999988079071, |
|
"rewards/chosen": -83.42225646972656, |
|
"rewards/margins": 41.75618362426758, |
|
"rewards/rejected": -125.17845153808594, |
|
"step": 670 |
|
}, |
|
{ |
|
"epoch": 0.17796388380005235, |
|
"grad_norm": 23.587490414758562, |
|
"learning_rate": 4.90849525415273e-07, |
|
"logits/chosen": -1.2536793947219849, |
|
"logits/rejected": -1.5548416376113892, |
|
"logps/chosen": -346.5150146484375, |
|
"logps/rejected": -420.87451171875, |
|
"loss": 0.6435, |
|
"rewards/accuracies": 0.675000011920929, |
|
"rewards/chosen": -84.81087493896484, |
|
"rewards/margins": 32.74394989013672, |
|
"rewards/rejected": -117.5548324584961, |
|
"step": 680 |
|
}, |
|
{ |
|
"epoch": 0.1805809997382884, |
|
"grad_norm": 16.140163273606795, |
|
"learning_rate": 4.902270717143858e-07, |
|
"logits/chosen": -1.2677083015441895, |
|
"logits/rejected": -0.6290455460548401, |
|
"logps/chosen": -392.08148193359375, |
|
"logps/rejected": -412.1546325683594, |
|
"loss": 0.6271, |
|
"rewards/accuracies": 0.6499999761581421, |
|
"rewards/chosen": -99.23873901367188, |
|
"rewards/margins": 40.86092758178711, |
|
"rewards/rejected": -140.0996856689453, |
|
"step": 690 |
|
}, |
|
{ |
|
"epoch": 0.18319811567652447, |
|
"grad_norm": 17.066483148041247, |
|
"learning_rate": 4.895845591221426e-07, |
|
"logits/chosen": -1.3644059896469116, |
|
"logits/rejected": -0.4180236756801605, |
|
"logps/chosen": -331.05426025390625, |
|
"logps/rejected": -343.99755859375, |
|
"loss": 0.6204, |
|
"rewards/accuracies": 0.75, |
|
"rewards/chosen": -71.15898895263672, |
|
"rewards/margins": 59.32234573364258, |
|
"rewards/rejected": -130.48133850097656, |
|
"step": 700 |
|
}, |
|
{ |
|
"epoch": 0.18581523161476055, |
|
"grad_norm": 22.437469152186377, |
|
"learning_rate": 4.8892204128816e-07, |
|
"logits/chosen": -1.5181987285614014, |
|
"logits/rejected": -0.2633799910545349, |
|
"logps/chosen": -363.61962890625, |
|
"logps/rejected": -313.43658447265625, |
|
"loss": 0.656, |
|
"rewards/accuracies": 0.625, |
|
"rewards/chosen": -85.1877670288086, |
|
"rewards/margins": 33.58545684814453, |
|
"rewards/rejected": -118.7732162475586, |
|
"step": 710 |
|
}, |
|
{ |
|
"epoch": 0.1884323475529966, |
|
"grad_norm": 16.0905054896879, |
|
"learning_rate": 4.882395735324863e-07, |
|
"logits/chosen": -1.1565659046173096, |
|
"logits/rejected": -0.8029291033744812, |
|
"logps/chosen": -322.3810729980469, |
|
"logps/rejected": -373.6277770996094, |
|
"loss": 0.6278, |
|
"rewards/accuracies": 0.7250000238418579, |
|
"rewards/chosen": -55.12693405151367, |
|
"rewards/margins": 71.59522247314453, |
|
"rewards/rejected": -126.7221450805664, |
|
"step": 720 |
|
}, |
|
{ |
|
"epoch": 0.19104946349123267, |
|
"grad_norm": 18.24845612413702, |
|
"learning_rate": 4.875372128409829e-07, |
|
"logits/chosen": -0.7359566688537598, |
|
"logits/rejected": -0.479675829410553, |
|
"logps/chosen": -234.16329956054688, |
|
"logps/rejected": -360.5873107910156, |
|
"loss": 0.6349, |
|
"rewards/accuracies": 0.800000011920929, |
|
"rewards/chosen": -82.40586853027344, |
|
"rewards/margins": 79.42875671386719, |
|
"rewards/rejected": -161.8346405029297, |
|
"step": 730 |
|
}, |
|
{ |
|
"epoch": 0.19366657942946872, |
|
"grad_norm": 31.57360611112559, |
|
"learning_rate": 4.868150178605653e-07, |
|
"logits/chosen": -1.8614606857299805, |
|
"logits/rejected": -0.5545592904090881, |
|
"logps/chosen": -468.30242919921875, |
|
"logps/rejected": -470.0863342285156, |
|
"loss": 0.6641, |
|
"rewards/accuracies": 0.7749999761581421, |
|
"rewards/chosen": -112.69195556640625, |
|
"rewards/margins": 80.2920150756836, |
|
"rewards/rejected": -192.9839630126953, |
|
"step": 740 |
|
}, |
|
{ |
|
"epoch": 0.1962836953677048, |
|
"grad_norm": 44.06651767708028, |
|
"learning_rate": 4.860730488943068e-07, |
|
"logits/chosen": -0.7021804451942444, |
|
"logits/rejected": -0.19743548333644867, |
|
"logps/chosen": -330.96148681640625, |
|
"logps/rejected": -390.07098388671875, |
|
"loss": 0.6437, |
|
"rewards/accuracies": 0.699999988079071, |
|
"rewards/chosen": -126.3841781616211, |
|
"rewards/margins": 59.72180938720703, |
|
"rewards/rejected": -186.10597229003906, |
|
"step": 750 |
|
}, |
|
{ |
|
"epoch": 0.19890081130594087, |
|
"grad_norm": 24.455997791623204, |
|
"learning_rate": 4.853113678964021e-07, |
|
"logits/chosen": -0.9393317103385925, |
|
"logits/rejected": -0.32588958740234375, |
|
"logps/chosen": -487.88897705078125, |
|
"logps/rejected": -475.53875732421875, |
|
"loss": 0.6242, |
|
"rewards/accuracies": 0.75, |
|
"rewards/chosen": -111.6043701171875, |
|
"rewards/margins": 43.061561584472656, |
|
"rewards/rejected": -154.6659393310547, |
|
"step": 760 |
|
}, |
|
{ |
|
"epoch": 0.20151792724417691, |
|
"grad_norm": 29.559259631889166, |
|
"learning_rate": 4.845300384669957e-07, |
|
"logits/chosen": -0.16026926040649414, |
|
"logits/rejected": 0.10852203518152237, |
|
"logps/chosen": -359.624267578125, |
|
"logps/rejected": -412.13970947265625, |
|
"loss": 0.6867, |
|
"rewards/accuracies": 0.699999988079071, |
|
"rewards/chosen": -108.4910659790039, |
|
"rewards/margins": 43.94545364379883, |
|
"rewards/rejected": -152.4365234375, |
|
"step": 770 |
|
}, |
|
{ |
|
"epoch": 0.204135043182413, |
|
"grad_norm": 38.08984952973831, |
|
"learning_rate": 4.8372912584687e-07, |
|
"logits/chosen": -1.0849199295043945, |
|
"logits/rejected": -0.6502343416213989, |
|
"logps/chosen": -363.4341735839844, |
|
"logps/rejected": -442.24053955078125, |
|
"loss": 0.6453, |
|
"rewards/accuracies": 0.699999988079071, |
|
"rewards/chosen": -82.14627838134766, |
|
"rewards/margins": 59.62005615234375, |
|
"rewards/rejected": -141.76632690429688, |
|
"step": 780 |
|
}, |
|
{ |
|
"epoch": 0.20675215912064904, |
|
"grad_norm": 17.40406336144934, |
|
"learning_rate": 4.829086969119983e-07, |
|
"logits/chosen": -0.8264859914779663, |
|
"logits/rejected": -0.45223379135131836, |
|
"logps/chosen": -364.9395446777344, |
|
"logps/rejected": -411.8614196777344, |
|
"loss": 0.6368, |
|
"rewards/accuracies": 0.6499999761581421, |
|
"rewards/chosen": -82.84831237792969, |
|
"rewards/margins": 35.919227600097656, |
|
"rewards/rejected": -118.76753997802734, |
|
"step": 790 |
|
}, |
|
{ |
|
"epoch": 0.2093692750588851, |
|
"grad_norm": 24.74636016899087, |
|
"learning_rate": 4.820688201679605e-07, |
|
"logits/chosen": -0.6875916719436646, |
|
"logits/rejected": -0.07015343010425568, |
|
"logps/chosen": -395.1434326171875, |
|
"logps/rejected": -461.1236877441406, |
|
"loss": 0.6157, |
|
"rewards/accuracies": 0.75, |
|
"rewards/chosen": -94.22093200683594, |
|
"rewards/margins": 64.74181365966797, |
|
"rewards/rejected": -158.96275329589844, |
|
"step": 800 |
|
}, |
|
{ |
|
"epoch": 0.21198639099712116, |
|
"grad_norm": 33.221118390849895, |
|
"learning_rate": 4.812095657442231e-07, |
|
"logits/chosen": -0.19572052359580994, |
|
"logits/rejected": 1.188407063484192, |
|
"logps/chosen": -333.38201904296875, |
|
"logps/rejected": -417.3399353027344, |
|
"loss": 0.6145, |
|
"rewards/accuracies": 0.875, |
|
"rewards/chosen": -74.15843200683594, |
|
"rewards/margins": 107.42332458496094, |
|
"rewards/rejected": -181.58175659179688, |
|
"step": 810 |
|
}, |
|
{ |
|
"epoch": 0.21460350693535724, |
|
"grad_norm": 30.78718682507575, |
|
"learning_rate": 4.803310053882831e-07, |
|
"logits/chosen": 1.3916046619415283, |
|
"logits/rejected": 1.80306077003479, |
|
"logps/chosen": -333.6537780761719, |
|
"logps/rejected": -432.14404296875, |
|
"loss": 0.6247, |
|
"rewards/accuracies": 0.800000011920929, |
|
"rewards/chosen": -104.1457290649414, |
|
"rewards/margins": 93.13396453857422, |
|
"rewards/rejected": -197.27969360351562, |
|
"step": 820 |
|
}, |
|
{ |
|
"epoch": 0.2172206228735933, |
|
"grad_norm": 32.330687937711005, |
|
"learning_rate": 4.794332124596775e-07, |
|
"logits/chosen": -0.8797634243965149, |
|
"logits/rejected": 0.2293863296508789, |
|
"logps/chosen": -385.99273681640625, |
|
"logps/rejected": -422.72760009765625, |
|
"loss": 0.6722, |
|
"rewards/accuracies": 0.6499999761581421, |
|
"rewards/chosen": -76.32684326171875, |
|
"rewards/margins": 44.87958526611328, |
|
"rewards/rejected": -121.20643615722656, |
|
"step": 830 |
|
}, |
|
{ |
|
"epoch": 0.21983773881182936, |
|
"grad_norm": 18.735973314129254, |
|
"learning_rate": 4.785162619238574e-07, |
|
"logits/chosen": -1.5916681289672852, |
|
"logits/rejected": 0.12047050148248672, |
|
"logps/chosen": -320.60382080078125, |
|
"logps/rejected": -326.4573974609375, |
|
"loss": 0.6174, |
|
"rewards/accuracies": 0.824999988079071, |
|
"rewards/chosen": -49.556217193603516, |
|
"rewards/margins": 59.73494338989258, |
|
"rewards/rejected": -109.2911605834961, |
|
"step": 840 |
|
}, |
|
{ |
|
"epoch": 0.22245485475006543, |
|
"grad_norm": 22.699909493169436, |
|
"learning_rate": 4.775802303459287e-07, |
|
"logits/chosen": -0.5288606286048889, |
|
"logits/rejected": 0.5344291925430298, |
|
"logps/chosen": -411.8746643066406, |
|
"logps/rejected": -471.15692138671875, |
|
"loss": 0.6233, |
|
"rewards/accuracies": 0.75, |
|
"rewards/chosen": -102.88179779052734, |
|
"rewards/margins": 61.04967498779297, |
|
"rewards/rejected": -163.9314727783203, |
|
"step": 850 |
|
}, |
|
{ |
|
"epoch": 0.22507197068830148, |
|
"grad_norm": 34.24159539521559, |
|
"learning_rate": 4.766251958842589e-07, |
|
"logits/chosen": 0.23371069133281708, |
|
"logits/rejected": 0.967072606086731, |
|
"logps/chosen": -335.1881103515625, |
|
"logps/rejected": -376.53192138671875, |
|
"loss": 0.632, |
|
"rewards/accuracies": 0.625, |
|
"rewards/chosen": -99.16200256347656, |
|
"rewards/margins": 47.29999542236328, |
|
"rewards/rejected": -146.46200561523438, |
|
"step": 860 |
|
}, |
|
{ |
|
"epoch": 0.22768908662653756, |
|
"grad_norm": 25.25556764754258, |
|
"learning_rate": 4.756512382839506e-07, |
|
"logits/chosen": -0.5754461884498596, |
|
"logits/rejected": 0.47906550765037537, |
|
"logps/chosen": -382.8406677246094, |
|
"logps/rejected": -452.028564453125, |
|
"loss": 0.6162, |
|
"rewards/accuracies": 0.699999988079071, |
|
"rewards/chosen": -93.5912857055664, |
|
"rewards/margins": 62.332862854003906, |
|
"rewards/rejected": -155.9241485595703, |
|
"step": 870 |
|
}, |
|
{ |
|
"epoch": 0.23030620256477363, |
|
"grad_norm": 37.027786250232936, |
|
"learning_rate": 4.746584388701831e-07, |
|
"logits/chosen": -0.11350803077220917, |
|
"logits/rejected": 0.7307132482528687, |
|
"logps/chosen": -340.4446716308594, |
|
"logps/rejected": -351.2494201660156, |
|
"loss": 0.6281, |
|
"rewards/accuracies": 0.75, |
|
"rewards/chosen": -87.96061706542969, |
|
"rewards/margins": 54.02471923828125, |
|
"rewards/rejected": -141.9853515625, |
|
"step": 880 |
|
}, |
|
{ |
|
"epoch": 0.23292331850300968, |
|
"grad_norm": 20.793289603998193, |
|
"learning_rate": 4.736468805414218e-07, |
|
"logits/chosen": -0.33384501934051514, |
|
"logits/rejected": 0.36782529950141907, |
|
"logps/chosen": -350.48291015625, |
|
"logps/rejected": -425.75006103515625, |
|
"loss": 0.6383, |
|
"rewards/accuracies": 0.75, |
|
"rewards/chosen": -86.30848693847656, |
|
"rewards/margins": 63.4256706237793, |
|
"rewards/rejected": -149.73416137695312, |
|
"step": 890 |
|
}, |
|
{ |
|
"epoch": 0.23554043444124576, |
|
"grad_norm": 29.395597470849673, |
|
"learning_rate": 4.7261664776249595e-07, |
|
"logits/chosen": -0.3997666537761688, |
|
"logits/rejected": -0.10973574221134186, |
|
"logps/chosen": -346.1295471191406, |
|
"logps/rejected": -398.69842529296875, |
|
"loss": 0.6225, |
|
"rewards/accuracies": 0.7749999761581421, |
|
"rewards/chosen": -99.20040893554688, |
|
"rewards/margins": 40.127742767333984, |
|
"rewards/rejected": -139.32815551757812, |
|
"step": 900 |
|
}, |
|
{ |
|
"epoch": 0.2381575503794818, |
|
"grad_norm": 49.28741036089969, |
|
"learning_rate": 4.7156782655754624e-07, |
|
"logits/chosen": -0.03910301998257637, |
|
"logits/rejected": 0.24898621439933777, |
|
"logps/chosen": -387.42022705078125, |
|
"logps/rejected": -459.38262939453125, |
|
"loss": 0.6485, |
|
"rewards/accuracies": 0.699999988079071, |
|
"rewards/chosen": -113.9815673828125, |
|
"rewards/margins": 61.421913146972656, |
|
"rewards/rejected": -175.40347290039062, |
|
"step": 910 |
|
}, |
|
{ |
|
"epoch": 0.24077466631771788, |
|
"grad_norm": 20.641644935857332, |
|
"learning_rate": 4.705005045028414e-07, |
|
"logits/chosen": 0.2061987817287445, |
|
"logits/rejected": 0.5889095664024353, |
|
"logps/chosen": -405.1602478027344, |
|
"logps/rejected": -368.80426025390625, |
|
"loss": 0.6533, |
|
"rewards/accuracies": 0.6499999761581421, |
|
"rewards/chosen": -141.77035522460938, |
|
"rewards/margins": 24.687891006469727, |
|
"rewards/rejected": -166.45823669433594, |
|
"step": 920 |
|
}, |
|
{ |
|
"epoch": 0.24339178225595393, |
|
"grad_norm": 42.863101044059015, |
|
"learning_rate": 4.694147707194659e-07, |
|
"logits/chosen": 0.5138073563575745, |
|
"logits/rejected": 1.4130924940109253, |
|
"logps/chosen": -334.7785339355469, |
|
"logps/rejected": -423.53521728515625, |
|
"loss": 0.62, |
|
"rewards/accuracies": 0.75, |
|
"rewards/chosen": -111.02415466308594, |
|
"rewards/margins": 68.88450622558594, |
|
"rewards/rejected": -179.90866088867188, |
|
"step": 930 |
|
}, |
|
{ |
|
"epoch": 0.24600889819419, |
|
"grad_norm": 17.582236860339187, |
|
"learning_rate": 4.683107158658781e-07, |
|
"logits/chosen": 0.26948803663253784, |
|
"logits/rejected": 1.579023838043213, |
|
"logps/chosen": -443.610107421875, |
|
"logps/rejected": -463.4444274902344, |
|
"loss": 0.6302, |
|
"rewards/accuracies": 0.699999988079071, |
|
"rewards/chosen": -142.50537109375, |
|
"rewards/margins": 68.03652954101562, |
|
"rewards/rejected": -210.54190063476562, |
|
"step": 940 |
|
}, |
|
{ |
|
"epoch": 0.24862601413242608, |
|
"grad_norm": 30.341277759240167, |
|
"learning_rate": 4.6718843213034066e-07, |
|
"logits/chosen": -0.16793589293956757, |
|
"logits/rejected": 0.207515150308609, |
|
"logps/chosen": -403.0389099121094, |
|
"logps/rejected": -482.0472106933594, |
|
"loss": 0.6609, |
|
"rewards/accuracies": 0.625, |
|
"rewards/chosen": -112.68156433105469, |
|
"rewards/margins": 50.542869567871094, |
|
"rewards/rejected": -163.22445678710938, |
|
"step": 950 |
|
}, |
|
{ |
|
"epoch": 0.2512431300706621, |
|
"grad_norm": 18.41663797308415, |
|
"learning_rate": 4.660480132232224e-07, |
|
"logits/chosen": -0.8736503720283508, |
|
"logits/rejected": 0.033030878752470016, |
|
"logps/chosen": -317.91522216796875, |
|
"logps/rejected": -392.4979553222656, |
|
"loss": 0.6237, |
|
"rewards/accuracies": 0.6000000238418579, |
|
"rewards/chosen": -76.37789916992188, |
|
"rewards/margins": 52.58796310424805, |
|
"rewards/rejected": -128.9658660888672, |
|
"step": 960 |
|
}, |
|
{ |
|
"epoch": 0.25386024600889817, |
|
"grad_norm": 23.17667895149893, |
|
"learning_rate": 4.64889554369174e-07, |
|
"logits/chosen": -0.007824582047760487, |
|
"logits/rejected": 0.285226970911026, |
|
"logps/chosen": -247.00167846679688, |
|
"logps/rejected": -351.49774169921875, |
|
"loss": 0.6313, |
|
"rewards/accuracies": 0.7749999761581421, |
|
"rewards/chosen": -67.98593139648438, |
|
"rewards/margins": 68.30857849121094, |
|
"rewards/rejected": -136.2945098876953, |
|
"step": 970 |
|
}, |
|
{ |
|
"epoch": 0.2564773619471343, |
|
"grad_norm": 25.935374616820546, |
|
"learning_rate": 4.637131522991764e-07, |
|
"logits/chosen": -0.6176963448524475, |
|
"logits/rejected": -0.6724565029144287, |
|
"logps/chosen": -349.24371337890625, |
|
"logps/rejected": -418.0118103027344, |
|
"loss": 0.6512, |
|
"rewards/accuracies": 0.625, |
|
"rewards/chosen": -93.6009292602539, |
|
"rewards/margins": 18.03414535522461, |
|
"rewards/rejected": -111.63508605957031, |
|
"step": 980 |
|
}, |
|
{ |
|
"epoch": 0.2590944778853703, |
|
"grad_norm": 60.752178743088585, |
|
"learning_rate": 4.6251890524246375e-07, |
|
"logits/chosen": 0.0293558482080698, |
|
"logits/rejected": 1.5651161670684814, |
|
"logps/chosen": -417.18255615234375, |
|
"logps/rejected": -387.0553283691406, |
|
"loss": 0.6242, |
|
"rewards/accuracies": 0.7749999761581421, |
|
"rewards/chosen": -90.68150329589844, |
|
"rewards/margins": 60.28722381591797, |
|
"rewards/rejected": -150.96873474121094, |
|
"step": 990 |
|
}, |
|
{ |
|
"epoch": 0.26171159382360637, |
|
"grad_norm": 50.547676030801206, |
|
"learning_rate": 4.613069129183218e-07, |
|
"logits/chosen": 1.1105889081954956, |
|
"logits/rejected": 1.4824490547180176, |
|
"logps/chosen": -373.8270263671875, |
|
"logps/rejected": -459.0462951660156, |
|
"loss": 0.6817, |
|
"rewards/accuracies": 0.7250000238418579, |
|
"rewards/chosen": -129.76486206054688, |
|
"rewards/margins": 66.0315933227539, |
|
"rewards/rejected": -195.7964324951172, |
|
"step": 1000 |
|
}, |
|
{ |
|
"epoch": 0.2643287097618425, |
|
"grad_norm": 31.19672071911541, |
|
"learning_rate": 4.6007727652776065e-07, |
|
"logits/chosen": 1.3123009204864502, |
|
"logits/rejected": 1.8814923763275146, |
|
"logps/chosen": -388.00616455078125, |
|
"logps/rejected": -473.76318359375, |
|
"loss": 0.6325, |
|
"rewards/accuracies": 0.7749999761581421, |
|
"rewards/chosen": -136.20584106445312, |
|
"rewards/margins": 68.31822204589844, |
|
"rewards/rejected": -204.5240478515625, |
|
"step": 1010 |
|
}, |
|
{ |
|
"epoch": 0.2669458257000785, |
|
"grad_norm": 24.38940664386023, |
|
"learning_rate": 4.588300987450652e-07, |
|
"logits/chosen": 0.7850022315979004, |
|
"logits/rejected": 1.2041956186294556, |
|
"logps/chosen": -416.10125732421875, |
|
"logps/rejected": -472.38397216796875, |
|
"loss": 0.6221, |
|
"rewards/accuracies": 0.7749999761581421, |
|
"rewards/chosen": -122.00775146484375, |
|
"rewards/margins": 70.46678161621094, |
|
"rewards/rejected": -192.4745330810547, |
|
"step": 1020 |
|
}, |
|
{ |
|
"epoch": 0.26956294163831457, |
|
"grad_norm": 32.57226417553651, |
|
"learning_rate": 4.5756548370922134e-07, |
|
"logits/chosen": 0.8883573412895203, |
|
"logits/rejected": 1.642805814743042, |
|
"logps/chosen": -326.7561950683594, |
|
"logps/rejected": -388.120849609375, |
|
"loss": 0.6242, |
|
"rewards/accuracies": 0.675000011920929, |
|
"rewards/chosen": -104.36289978027344, |
|
"rewards/margins": 54.60386276245117, |
|
"rewards/rejected": -158.96676635742188, |
|
"step": 1030 |
|
}, |
|
{ |
|
"epoch": 0.2721800575765506, |
|
"grad_norm": 46.81257101957739, |
|
"learning_rate": 4.5628353701522047e-07, |
|
"logits/chosen": -0.09301260858774185, |
|
"logits/rejected": 1.3069690465927124, |
|
"logps/chosen": -414.9811096191406, |
|
"logps/rejected": -459.68389892578125, |
|
"loss": 0.6488, |
|
"rewards/accuracies": 0.75, |
|
"rewards/chosen": -83.94697570800781, |
|
"rewards/margins": 63.28023147583008, |
|
"rewards/rejected": -147.22718811035156, |
|
"step": 1040 |
|
}, |
|
{ |
|
"epoch": 0.2747971735147867, |
|
"grad_norm": 28.130062724101716, |
|
"learning_rate": 4.549843657052429e-07, |
|
"logits/chosen": -0.6466625332832336, |
|
"logits/rejected": 0.41721582412719727, |
|
"logps/chosen": -322.29290771484375, |
|
"logps/rejected": -438.45501708984375, |
|
"loss": 0.6264, |
|
"rewards/accuracies": 0.7749999761581421, |
|
"rewards/chosen": -61.9997444152832, |
|
"rewards/margins": 81.4615249633789, |
|
"rewards/rejected": -143.46127319335938, |
|
"step": 1050 |
|
}, |
|
{ |
|
"epoch": 0.27741428945302277, |
|
"grad_norm": 40.68291107851666, |
|
"learning_rate": 4.5366807825971907e-07, |
|
"logits/chosen": -0.6572477221488953, |
|
"logits/rejected": 0.26327458024024963, |
|
"logps/chosen": -343.57025146484375, |
|
"logps/rejected": -353.9472351074219, |
|
"loss": 0.6314, |
|
"rewards/accuracies": 0.7250000238418579, |
|
"rewards/chosen": -78.26343536376953, |
|
"rewards/margins": 58.32725143432617, |
|
"rewards/rejected": -136.59066772460938, |
|
"step": 1060 |
|
}, |
|
{ |
|
"epoch": 0.2800314053912588, |
|
"grad_norm": 18.424013249011935, |
|
"learning_rate": 4.5233478458827176e-07, |
|
"logits/chosen": -1.0701769590377808, |
|
"logits/rejected": -0.38552147150039673, |
|
"logps/chosen": -371.5882263183594, |
|
"logps/rejected": -432.673583984375, |
|
"loss": 0.6271, |
|
"rewards/accuracies": 0.824999988079071, |
|
"rewards/chosen": -81.17890930175781, |
|
"rewards/margins": 78.04898834228516, |
|
"rewards/rejected": -159.22787475585938, |
|
"step": 1070 |
|
}, |
|
{ |
|
"epoch": 0.2826485213294949, |
|
"grad_norm": 29.259874543341486, |
|
"learning_rate": 4.509845960205389e-07, |
|
"logits/chosen": -0.3320876955986023, |
|
"logits/rejected": 0.8465584516525269, |
|
"logps/chosen": -347.73455810546875, |
|
"logps/rejected": -393.71173095703125, |
|
"loss": 0.6556, |
|
"rewards/accuracies": 0.7250000238418579, |
|
"rewards/chosen": -81.61093139648438, |
|
"rewards/margins": 78.703369140625, |
|
"rewards/rejected": -160.31430053710938, |
|
"step": 1080 |
|
}, |
|
{ |
|
"epoch": 0.28526563726773096, |
|
"grad_norm": 54.92918197903394, |
|
"learning_rate": 4.4961762529687736e-07, |
|
"logits/chosen": 1.3831452131271362, |
|
"logits/rejected": 2.0626230239868164, |
|
"logps/chosen": -394.3235168457031, |
|
"logps/rejected": -404.1439208984375, |
|
"loss": 0.653, |
|
"rewards/accuracies": 0.699999988079071, |
|
"rewards/chosen": -137.50064086914062, |
|
"rewards/margins": 56.01093673706055, |
|
"rewards/rejected": -193.51158142089844, |
|
"step": 1090 |
|
}, |
|
{ |
|
"epoch": 0.287882753205967, |
|
"grad_norm": 53.85836537057901, |
|
"learning_rate": 4.482339865589492e-07, |
|
"logits/chosen": 1.0399128198623657, |
|
"logits/rejected": 2.5773065090179443, |
|
"logps/chosen": -447.4933166503906, |
|
"logps/rejected": -476.53717041015625, |
|
"loss": 0.6421, |
|
"rewards/accuracies": 0.75, |
|
"rewards/chosen": -129.35110473632812, |
|
"rewards/margins": 97.5198745727539, |
|
"rewards/rejected": -226.8709716796875, |
|
"step": 1100 |
|
}, |
|
{ |
|
"epoch": 0.2904998691442031, |
|
"grad_norm": 29.02990654409812, |
|
"learning_rate": 4.4683379534019076e-07, |
|
"logits/chosen": 1.8409522771835327, |
|
"logits/rejected": 2.082953929901123, |
|
"logps/chosen": -398.29193115234375, |
|
"logps/rejected": -438.3072204589844, |
|
"loss": 0.6664, |
|
"rewards/accuracies": 0.550000011920929, |
|
"rewards/chosen": -144.30996704101562, |
|
"rewards/margins": 39.263938903808594, |
|
"rewards/rejected": -183.57391357421875, |
|
"step": 1110 |
|
}, |
|
{ |
|
"epoch": 0.29311698508243916, |
|
"grad_norm": 17.300862424332053, |
|
"learning_rate": 4.4541716855616593e-07, |
|
"logits/chosen": 1.154475450515747, |
|
"logits/rejected": 1.1236993074417114, |
|
"logps/chosen": -351.427978515625, |
|
"logps/rejected": -408.2262878417969, |
|
"loss": 0.6382, |
|
"rewards/accuracies": 0.6499999761581421, |
|
"rewards/chosen": -128.45620727539062, |
|
"rewards/margins": 44.44871139526367, |
|
"rewards/rejected": -172.90492248535156, |
|
"step": 1120 |
|
}, |
|
{ |
|
"epoch": 0.2957341010206752, |
|
"grad_norm": 31.459949619459707, |
|
"learning_rate": 4.4398422449480357e-07, |
|
"logits/chosen": 0.3457348048686981, |
|
"logits/rejected": 0.668042778968811, |
|
"logps/chosen": -417.75115966796875, |
|
"logps/rejected": -487.89794921875, |
|
"loss": 0.632, |
|
"rewards/accuracies": 0.699999988079071, |
|
"rewards/chosen": -119.79643249511719, |
|
"rewards/margins": 72.1993408203125, |
|
"rewards/rejected": -191.99578857421875, |
|
"step": 1130 |
|
}, |
|
{ |
|
"epoch": 0.29835121695891126, |
|
"grad_norm": 24.115738554276895, |
|
"learning_rate": 4.4253508280652036e-07, |
|
"logits/chosen": 0.2072986364364624, |
|
"logits/rejected": 2.0747737884521484, |
|
"logps/chosen": -435.58673095703125, |
|
"logps/rejected": -402.98089599609375, |
|
"loss": 0.6209, |
|
"rewards/accuracies": 0.875, |
|
"rewards/chosen": -113.14810943603516, |
|
"rewards/margins": 88.4394302368164, |
|
"rewards/rejected": -201.58753967285156, |
|
"step": 1140 |
|
}, |
|
{ |
|
"epoch": 0.30096833289714736, |
|
"grad_norm": 38.44970369580517, |
|
"learning_rate": 4.410698644942302e-07, |
|
"logits/chosen": 1.2417165040969849, |
|
"logits/rejected": 2.5007147789001465, |
|
"logps/chosen": -380.4283447265625, |
|
"logps/rejected": -374.45361328125, |
|
"loss": 0.6236, |
|
"rewards/accuracies": 0.7749999761581421, |
|
"rewards/chosen": -100.56785583496094, |
|
"rewards/margins": 60.188575744628906, |
|
"rewards/rejected": -160.7564239501953, |
|
"step": 1150 |
|
}, |
|
{ |
|
"epoch": 0.3035854488353834, |
|
"grad_norm": 23.20243908616475, |
|
"learning_rate": 4.3958869190324057e-07, |
|
"logits/chosen": 1.181898593902588, |
|
"logits/rejected": 1.9439668655395508, |
|
"logps/chosen": -388.64276123046875, |
|
"logps/rejected": -410.17242431640625, |
|
"loss": 0.6239, |
|
"rewards/accuracies": 0.699999988079071, |
|
"rewards/chosen": -98.65071105957031, |
|
"rewards/margins": 53.087440490722656, |
|
"rewards/rejected": -151.73812866210938, |
|
"step": 1160 |
|
}, |
|
{ |
|
"epoch": 0.30620256477361946, |
|
"grad_norm": 27.16772844182103, |
|
"learning_rate": 4.380916887110365e-07, |
|
"logits/chosen": 0.9662491679191589, |
|
"logits/rejected": 2.0781798362731934, |
|
"logps/chosen": -342.8768005371094, |
|
"logps/rejected": -418.36102294921875, |
|
"loss": 0.6404, |
|
"rewards/accuracies": 0.824999988079071, |
|
"rewards/chosen": -107.91712951660156, |
|
"rewards/margins": 76.5067138671875, |
|
"rewards/rejected": -184.42385864257812, |
|
"step": 1170 |
|
}, |
|
{ |
|
"epoch": 0.30881968071185556, |
|
"grad_norm": 23.646068288118425, |
|
"learning_rate": 4.3657897991695394e-07, |
|
"logits/chosen": 0.5617043375968933, |
|
"logits/rejected": 1.8490877151489258, |
|
"logps/chosen": -370.76507568359375, |
|
"logps/rejected": -392.9139099121094, |
|
"loss": 0.6494, |
|
"rewards/accuracies": 0.7749999761581421, |
|
"rewards/chosen": -107.6982421875, |
|
"rewards/margins": 75.37187957763672, |
|
"rewards/rejected": -183.0701446533203, |
|
"step": 1180 |
|
}, |
|
{ |
|
"epoch": 0.3114367966500916, |
|
"grad_norm": 17.745565087194286, |
|
"learning_rate": 4.350506918317416e-07, |
|
"logits/chosen": 0.6922825574874878, |
|
"logits/rejected": 1.2396411895751953, |
|
"logps/chosen": -435.5896911621094, |
|
"logps/rejected": -460.70233154296875, |
|
"loss": 0.6229, |
|
"rewards/accuracies": 0.699999988079071, |
|
"rewards/chosen": -120.14180755615234, |
|
"rewards/margins": 44.74254608154297, |
|
"rewards/rejected": -164.88436889648438, |
|
"step": 1190 |
|
}, |
|
{ |
|
"epoch": 0.31405391258832765, |
|
"grad_norm": 41.0845785288509, |
|
"learning_rate": 4.335069520670149e-07, |
|
"logits/chosen": 1.7557027339935303, |
|
"logits/rejected": 2.1554341316223145, |
|
"logps/chosen": -435.21697998046875, |
|
"logps/rejected": -479.82550048828125, |
|
"loss": 0.6491, |
|
"rewards/accuracies": 0.7250000238418579, |
|
"rewards/chosen": -166.35964965820312, |
|
"rewards/margins": 48.9127311706543, |
|
"rewards/rejected": -215.2723846435547, |
|
"step": 1200 |
|
}, |
|
{ |
|
"epoch": 0.3166710285265637, |
|
"grad_norm": 62.23468705652573, |
|
"learning_rate": 4.319478895245999e-07, |
|
"logits/chosen": 1.0253560543060303, |
|
"logits/rejected": 1.471136450767517, |
|
"logps/chosen": -379.4768981933594, |
|
"logps/rejected": -497.72113037109375, |
|
"loss": 0.5809, |
|
"rewards/accuracies": 0.824999988079071, |
|
"rewards/chosen": -130.55929565429688, |
|
"rewards/margins": 83.6629638671875, |
|
"rewards/rejected": -214.22225952148438, |
|
"step": 1210 |
|
}, |
|
{ |
|
"epoch": 0.3192881444647998, |
|
"grad_norm": 26.246391316141533, |
|
"learning_rate": 4.3037363438577036e-07, |
|
"logits/chosen": 0.5762010216712952, |
|
"logits/rejected": 1.9362785816192627, |
|
"logps/chosen": -396.3133544921875, |
|
"logps/rejected": -516.368408203125, |
|
"loss": 0.5856, |
|
"rewards/accuracies": 0.800000011920929, |
|
"rewards/chosen": -113.91856384277344, |
|
"rewards/margins": 118.1727523803711, |
|
"rewards/rejected": -232.09133911132812, |
|
"step": 1220 |
|
}, |
|
{ |
|
"epoch": 0.32190526040303585, |
|
"grad_norm": 49.15854501313321, |
|
"learning_rate": 4.2878431810037716e-07, |
|
"logits/chosen": 1.5246806144714355, |
|
"logits/rejected": 2.3451852798461914, |
|
"logps/chosen": -372.4715576171875, |
|
"logps/rejected": -474.74169921875, |
|
"loss": 0.6256, |
|
"rewards/accuracies": 0.675000011920929, |
|
"rewards/chosen": -156.90863037109375, |
|
"rewards/margins": 109.8390121459961, |
|
"rewards/rejected": -266.74761962890625, |
|
"step": 1230 |
|
}, |
|
{ |
|
"epoch": 0.3245223763412719, |
|
"grad_norm": 40.303888351983076, |
|
"learning_rate": 4.271800733758729e-07, |
|
"logits/chosen": 0.6723651885986328, |
|
"logits/rejected": 1.2088282108306885, |
|
"logps/chosen": -459.427490234375, |
|
"logps/rejected": -454.59136962890625, |
|
"loss": 0.6454, |
|
"rewards/accuracies": 0.550000011920929, |
|
"rewards/chosen": -121.82243347167969, |
|
"rewards/margins": 57.75952911376953, |
|
"rewards/rejected": -179.58197021484375, |
|
"step": 1240 |
|
}, |
|
{ |
|
"epoch": 0.327139492279508, |
|
"grad_norm": 25.566591095191676, |
|
"learning_rate": 4.255610341662304e-07, |
|
"logits/chosen": -0.0015428184997290373, |
|
"logits/rejected": 0.6781773567199707, |
|
"logps/chosen": -351.0030212402344, |
|
"logps/rejected": -389.29718017578125, |
|
"loss": 0.6228, |
|
"rewards/accuracies": 0.824999988079071, |
|
"rewards/chosen": -99.91766357421875, |
|
"rewards/margins": 89.89549255371094, |
|
"rewards/rejected": -189.81317138671875, |
|
"step": 1250 |
|
}, |
|
{ |
|
"epoch": 0.32975660821774405, |
|
"grad_norm": 76.72537095439779, |
|
"learning_rate": 4.2392733566075757e-07, |
|
"logits/chosen": -0.43047595024108887, |
|
"logits/rejected": 0.378257155418396, |
|
"logps/chosen": -412.3002014160156, |
|
"logps/rejected": -444.5391540527344, |
|
"loss": 0.6394, |
|
"rewards/accuracies": 0.699999988079071, |
|
"rewards/chosen": -121.55326843261719, |
|
"rewards/margins": 38.64310073852539, |
|
"rewards/rejected": -160.19638061523438, |
|
"step": 1260 |
|
}, |
|
{ |
|
"epoch": 0.3323737241559801, |
|
"grad_norm": 52.61567100724301, |
|
"learning_rate": 4.2227911427280973e-07, |
|
"logits/chosen": -0.7219769954681396, |
|
"logits/rejected": 0.48331594467163086, |
|
"logps/chosen": -427.62054443359375, |
|
"logps/rejected": -450.5807189941406, |
|
"loss": 0.6512, |
|
"rewards/accuracies": 0.675000011920929, |
|
"rewards/chosen": -113.51541900634766, |
|
"rewards/margins": 68.88684844970703, |
|
"rewards/rejected": -182.4022674560547, |
|
"step": 1270 |
|
}, |
|
{ |
|
"epoch": 0.33499084009421615, |
|
"grad_norm": 30.88397795823836, |
|
"learning_rate": 4.206165076283982e-07, |
|
"logits/chosen": -0.9375940561294556, |
|
"logits/rejected": 1.5283622741699219, |
|
"logps/chosen": -421.84320068359375, |
|
"logps/rejected": -440.0548400878906, |
|
"loss": 0.5959, |
|
"rewards/accuracies": 0.824999988079071, |
|
"rewards/chosen": -116.27479553222656, |
|
"rewards/margins": 98.38531494140625, |
|
"rewards/rejected": -214.6601104736328, |
|
"step": 1280 |
|
}, |
|
{ |
|
"epoch": 0.33760795603245225, |
|
"grad_norm": 57.576483810059, |
|
"learning_rate": 4.1893965455469946e-07, |
|
"logits/chosen": -0.68849116563797, |
|
"logits/rejected": 1.0034667253494263, |
|
"logps/chosen": -374.0795593261719, |
|
"logps/rejected": -387.61724853515625, |
|
"loss": 0.6091, |
|
"rewards/accuracies": 0.800000011920929, |
|
"rewards/chosen": -99.40745544433594, |
|
"rewards/margins": 77.72341918945312, |
|
"rewards/rejected": -177.13088989257812, |
|
"step": 1290 |
|
}, |
|
{ |
|
"epoch": 0.3402250719706883, |
|
"grad_norm": 31.72284091111631, |
|
"learning_rate": 4.172486950684626e-07, |
|
"logits/chosen": 0.16339489817619324, |
|
"logits/rejected": 1.6479781866073608, |
|
"logps/chosen": -434.8985900878906, |
|
"logps/rejected": -434.3905334472656, |
|
"loss": 0.662, |
|
"rewards/accuracies": 0.6499999761581421, |
|
"rewards/chosen": -159.21035766601562, |
|
"rewards/margins": 49.35946273803711, |
|
"rewards/rejected": -208.56982421875, |
|
"step": 1300 |
|
}, |
|
{ |
|
"epoch": 0.34284218790892435, |
|
"grad_norm": 24.64306569773067, |
|
"learning_rate": 4.155437703643181e-07, |
|
"logits/chosen": -0.7670415639877319, |
|
"logits/rejected": -0.2981819808483124, |
|
"logps/chosen": -374.2372741699219, |
|
"logps/rejected": -417.8731384277344, |
|
"loss": 0.6229, |
|
"rewards/accuracies": 0.800000011920929, |
|
"rewards/chosen": -111.1018295288086, |
|
"rewards/margins": 54.38102340698242, |
|
"rewards/rejected": -165.4828643798828, |
|
"step": 1310 |
|
}, |
|
{ |
|
"epoch": 0.34545930384716045, |
|
"grad_norm": 26.087342646568175, |
|
"learning_rate": 4.138250228029881e-07, |
|
"logits/chosen": -0.5596447587013245, |
|
"logits/rejected": 0.2671245038509369, |
|
"logps/chosen": -410.1627502441406, |
|
"logps/rejected": -433.45953369140625, |
|
"loss": 0.6282, |
|
"rewards/accuracies": 0.6499999761581421, |
|
"rewards/chosen": -120.9754867553711, |
|
"rewards/margins": 55.230552673339844, |
|
"rewards/rejected": -176.20603942871094, |
|
"step": 1320 |
|
}, |
|
{ |
|
"epoch": 0.3480764197853965, |
|
"grad_norm": 20.376442525686873, |
|
"learning_rate": 4.1209259589939935e-07, |
|
"logits/chosen": -0.38580647110939026, |
|
"logits/rejected": 0.4782138764858246, |
|
"logps/chosen": -339.5039367675781, |
|
"logps/rejected": -380.3269958496094, |
|
"loss": 0.6346, |
|
"rewards/accuracies": 0.7749999761581421, |
|
"rewards/chosen": -94.53594970703125, |
|
"rewards/margins": 66.47621154785156, |
|
"rewards/rejected": -161.01217651367188, |
|
"step": 1330 |
|
}, |
|
{ |
|
"epoch": 0.35069353572363254, |
|
"grad_norm": 21.37216900838956, |
|
"learning_rate": 4.103466343106998e-07, |
|
"logits/chosen": -0.649118185043335, |
|
"logits/rejected": 0.28338345885276794, |
|
"logps/chosen": -398.08270263671875, |
|
"logps/rejected": -392.17437744140625, |
|
"loss": 0.644, |
|
"rewards/accuracies": 0.7749999761581421, |
|
"rewards/chosen": -98.56443786621094, |
|
"rewards/margins": 50.19805145263672, |
|
"rewards/rejected": -148.76248168945312, |
|
"step": 1340 |
|
}, |
|
{ |
|
"epoch": 0.35331065166186865, |
|
"grad_norm": 28.767958733320434, |
|
"learning_rate": 4.085872838241796e-07, |
|
"logits/chosen": -0.13218168914318085, |
|
"logits/rejected": 0.9735630750656128, |
|
"logps/chosen": -355.2423095703125, |
|
"logps/rejected": -426.5146484375, |
|
"loss": 0.612, |
|
"rewards/accuracies": 0.75, |
|
"rewards/chosen": -110.88926696777344, |
|
"rewards/margins": 95.35261535644531, |
|
"rewards/rejected": -206.2418975830078, |
|
"step": 1350 |
|
}, |
|
{ |
|
"epoch": 0.3559277676001047, |
|
"grad_norm": 28.419877906639005, |
|
"learning_rate": 4.06814691345098e-07, |
|
"logits/chosen": -0.24849538505077362, |
|
"logits/rejected": 1.1262340545654297, |
|
"logps/chosen": -443.79180908203125, |
|
"logps/rejected": -455.39990234375, |
|
"loss": 0.6039, |
|
"rewards/accuracies": 0.800000011920929, |
|
"rewards/chosen": -112.5448226928711, |
|
"rewards/margins": 78.75032806396484, |
|
"rewards/rejected": -191.295166015625, |
|
"step": 1360 |
|
}, |
|
{ |
|
"epoch": 0.35854488353834074, |
|
"grad_norm": 68.82439419877379, |
|
"learning_rate": 4.0502900488441707e-07, |
|
"logits/chosen": 0.19888067245483398, |
|
"logits/rejected": 1.122232437133789, |
|
"logps/chosen": -423.8077697753906, |
|
"logps/rejected": -449.03509521484375, |
|
"loss": 0.6206, |
|
"rewards/accuracies": 0.699999988079071, |
|
"rewards/chosen": -138.73602294921875, |
|
"rewards/margins": 72.9966812133789, |
|
"rewards/rejected": -211.7327117919922, |
|
"step": 1370 |
|
}, |
|
{ |
|
"epoch": 0.3611619994765768, |
|
"grad_norm": 15.580407976066924, |
|
"learning_rate": 4.032303735464422e-07, |
|
"logits/chosen": 0.5571808218955994, |
|
"logits/rejected": 1.5645115375518799, |
|
"logps/chosen": -443.7701110839844, |
|
"logps/rejected": -538.642333984375, |
|
"loss": 0.6282, |
|
"rewards/accuracies": 0.7749999761581421, |
|
"rewards/chosen": -156.4287567138672, |
|
"rewards/margins": 87.27613830566406, |
|
"rewards/rejected": -243.7049102783203, |
|
"step": 1380 |
|
}, |
|
{ |
|
"epoch": 0.3637791154148129, |
|
"grad_norm": 25.711219077342538, |
|
"learning_rate": 4.014189475163726e-07, |
|
"logits/chosen": 0.42829257249832153, |
|
"logits/rejected": 2.6174747943878174, |
|
"logps/chosen": -480.5956115722656, |
|
"logps/rejected": -478.7676696777344, |
|
"loss": 0.6314, |
|
"rewards/accuracies": 0.7749999761581421, |
|
"rewards/chosen": -151.87530517578125, |
|
"rewards/margins": 86.75433349609375, |
|
"rewards/rejected": -238.629638671875, |
|
"step": 1390 |
|
}, |
|
{ |
|
"epoch": 0.36639623135304894, |
|
"grad_norm": 19.353569261594092, |
|
"learning_rate": 3.995948780477605e-07, |
|
"logits/chosen": 0.7079881429672241, |
|
"logits/rejected": 2.395602226257324, |
|
"logps/chosen": -477.9998474121094, |
|
"logps/rejected": -517.8780517578125, |
|
"loss": 0.6215, |
|
"rewards/accuracies": 0.75, |
|
"rewards/chosen": -151.6291961669922, |
|
"rewards/margins": 93.9462890625, |
|
"rewards/rejected": -245.5754852294922, |
|
"step": 1400 |
|
}, |
|
{ |
|
"epoch": 0.369013347291285, |
|
"grad_norm": 29.67895552774033, |
|
"learning_rate": 3.977583174498816e-07, |
|
"logits/chosen": 0.24756821990013123, |
|
"logits/rejected": 0.8279555439949036, |
|
"logps/chosen": -358.0359802246094, |
|
"logps/rejected": -396.7525939941406, |
|
"loss": 0.6623, |
|
"rewards/accuracies": 0.625, |
|
"rewards/chosen": -118.51368713378906, |
|
"rewards/margins": 39.870086669921875, |
|
"rewards/rejected": -158.38375854492188, |
|
"step": 1410 |
|
}, |
|
{ |
|
"epoch": 0.3716304632295211, |
|
"grad_norm": 22.478816315434774, |
|
"learning_rate": 3.9590941907501717e-07, |
|
"logits/chosen": 0.16131027042865753, |
|
"logits/rejected": 1.7322285175323486, |
|
"logps/chosen": -397.6242370605469, |
|
"logps/rejected": -410.8728942871094, |
|
"loss": 0.6435, |
|
"rewards/accuracies": 0.75, |
|
"rewards/chosen": -137.3665008544922, |
|
"rewards/margins": 59.99677658081055, |
|
"rewards/rejected": -197.36328125, |
|
"step": 1420 |
|
}, |
|
{ |
|
"epoch": 0.37424757916775714, |
|
"grad_norm": 25.09540882981143, |
|
"learning_rate": 3.9404833730564974e-07, |
|
"logits/chosen": 0.5373800992965698, |
|
"logits/rejected": 1.047599196434021, |
|
"logps/chosen": -433.1474609375, |
|
"logps/rejected": -441.2938537597656, |
|
"loss": 0.6193, |
|
"rewards/accuracies": 0.7250000238418579, |
|
"rewards/chosen": -142.21206665039062, |
|
"rewards/margins": 46.76278305053711, |
|
"rewards/rejected": -188.97482299804688, |
|
"step": 1430 |
|
}, |
|
{ |
|
"epoch": 0.3768646951059932, |
|
"grad_norm": 15.852691574589016, |
|
"learning_rate": 3.9217522754157117e-07, |
|
"logits/chosen": -0.2401774376630783, |
|
"logits/rejected": 0.3488456606864929, |
|
"logps/chosen": -358.58544921875, |
|
"logps/rejected": -401.83233642578125, |
|
"loss": 0.5961, |
|
"rewards/accuracies": 0.699999988079071, |
|
"rewards/chosen": -120.71138000488281, |
|
"rewards/margins": 69.83918762207031, |
|
"rewards/rejected": -190.55055236816406, |
|
"step": 1440 |
|
}, |
|
{ |
|
"epoch": 0.37948181104422923, |
|
"grad_norm": 47.29678098178137, |
|
"learning_rate": 3.9029024618690785e-07, |
|
"logits/chosen": -0.9757177233695984, |
|
"logits/rejected": 0.08731711655855179, |
|
"logps/chosen": -320.13580322265625, |
|
"logps/rejected": -409.79010009765625, |
|
"loss": 0.6304, |
|
"rewards/accuracies": 0.7749999761581421, |
|
"rewards/chosen": -109.36761474609375, |
|
"rewards/margins": 76.31434631347656, |
|
"rewards/rejected": -185.68197631835938, |
|
"step": 1450 |
|
}, |
|
{ |
|
"epoch": 0.38209892698246534, |
|
"grad_norm": 20.648940769058072, |
|
"learning_rate": 3.883935506370605e-07, |
|
"logits/chosen": -0.8205481767654419, |
|
"logits/rejected": 0.7159221172332764, |
|
"logps/chosen": -396.99591064453125, |
|
"logps/rejected": -503.263916015625, |
|
"loss": 0.6008, |
|
"rewards/accuracies": 0.875, |
|
"rewards/chosen": -131.62887573242188, |
|
"rewards/margins": 118.54338073730469, |
|
"rewards/rejected": -250.1722412109375, |
|
"step": 1460 |
|
}, |
|
{ |
|
"epoch": 0.3847160429207014, |
|
"grad_norm": 34.965513352204376, |
|
"learning_rate": 3.864852992655616e-07, |
|
"logits/chosen": -0.4378627836704254, |
|
"logits/rejected": 1.2069480419158936, |
|
"logps/chosen": -369.70904541015625, |
|
"logps/rejected": -504.4092712402344, |
|
"loss": 0.6134, |
|
"rewards/accuracies": 0.7749999761581421, |
|
"rewards/chosen": -133.9385986328125, |
|
"rewards/margins": 106.6314468383789, |
|
"rewards/rejected": -240.57003784179688, |
|
"step": 1470 |
|
}, |
|
{ |
|
"epoch": 0.38733315885893743, |
|
"grad_norm": 39.359286384644776, |
|
"learning_rate": 3.845656514108515e-07, |
|
"logits/chosen": -0.2930401861667633, |
|
"logits/rejected": 1.5885709524154663, |
|
"logps/chosen": -504.27288818359375, |
|
"logps/rejected": -531.5779418945312, |
|
"loss": 0.6244, |
|
"rewards/accuracies": 0.800000011920929, |
|
"rewards/chosen": -164.9402313232422, |
|
"rewards/margins": 104.5406494140625, |
|
"rewards/rejected": -269.48089599609375, |
|
"step": 1480 |
|
}, |
|
{ |
|
"epoch": 0.38995027479717354, |
|
"grad_norm": 133.50854420589513, |
|
"learning_rate": 3.8263476736297375e-07, |
|
"logits/chosen": -0.17435605823993683, |
|
"logits/rejected": 1.8973588943481445, |
|
"logps/chosen": -486.27569580078125, |
|
"logps/rejected": -542.3553466796875, |
|
"loss": 0.6385, |
|
"rewards/accuracies": 0.824999988079071, |
|
"rewards/chosen": -172.06533813476562, |
|
"rewards/margins": 126.09920501708984, |
|
"rewards/rejected": -298.1645202636719, |
|
"step": 1490 |
|
}, |
|
{ |
|
"epoch": 0.3925673907354096, |
|
"grad_norm": 51.73277823324011, |
|
"learning_rate": 3.8069280835019055e-07, |
|
"logits/chosen": 0.6358996033668518, |
|
"logits/rejected": 1.3472615480422974, |
|
"logps/chosen": -401.0054626464844, |
|
"logps/rejected": -480.7518005371094, |
|
"loss": 0.6402, |
|
"rewards/accuracies": 0.75, |
|
"rewards/chosen": -143.2891387939453, |
|
"rewards/margins": 92.3449478149414, |
|
"rewards/rejected": -235.6341094970703, |
|
"step": 1500 |
|
}, |
|
{ |
|
"epoch": 0.39518450667364563, |
|
"grad_norm": 21.226979866730023, |
|
"learning_rate": 3.7873993652552073e-07, |
|
"logits/chosen": 1.3380610942840576, |
|
"logits/rejected": 2.3835110664367676, |
|
"logps/chosen": -434.25274658203125, |
|
"logps/rejected": -512.7623901367188, |
|
"loss": 0.6167, |
|
"rewards/accuracies": 0.699999988079071, |
|
"rewards/chosen": -170.42994689941406, |
|
"rewards/margins": 104.90080261230469, |
|
"rewards/rejected": -275.3307189941406, |
|
"step": 1510 |
|
}, |
|
{ |
|
"epoch": 0.39780162261188173, |
|
"grad_norm": 25.919917334233475, |
|
"learning_rate": 3.767763149531995e-07, |
|
"logits/chosen": 1.0969598293304443, |
|
"logits/rejected": 2.068394660949707, |
|
"logps/chosen": -389.23974609375, |
|
"logps/rejected": -449.2406311035156, |
|
"loss": 0.6114, |
|
"rewards/accuracies": 0.7250000238418579, |
|
"rewards/chosen": -158.19741821289062, |
|
"rewards/margins": 76.74111938476562, |
|
"rewards/rejected": -234.9385223388672, |
|
"step": 1520 |
|
}, |
|
{ |
|
"epoch": 0.4004187385501178, |
|
"grad_norm": 16.290946941569523, |
|
"learning_rate": 3.7480210759506326e-07, |
|
"logits/chosen": 0.4282687306404114, |
|
"logits/rejected": 1.8052078485488892, |
|
"logps/chosen": -349.53875732421875, |
|
"logps/rejected": -432.3233337402344, |
|
"loss": 0.6152, |
|
"rewards/accuracies": 0.800000011920929, |
|
"rewards/chosen": -122.47404479980469, |
|
"rewards/margins": 89.38887023925781, |
|
"rewards/rejected": -211.86288452148438, |
|
"step": 1530 |
|
}, |
|
{ |
|
"epoch": 0.40303585448835383, |
|
"grad_norm": 22.714557755720946, |
|
"learning_rate": 3.728174792968582e-07, |
|
"logits/chosen": 0.4320814609527588, |
|
"logits/rejected": 0.4742346405982971, |
|
"logps/chosen": -325.5453186035156, |
|
"logps/rejected": -462.4740295410156, |
|
"loss": 0.6216, |
|
"rewards/accuracies": 0.7250000238418579, |
|
"rewards/chosen": -116.84428405761719, |
|
"rewards/margins": 77.38162994384766, |
|
"rewards/rejected": -194.2259063720703, |
|
"step": 1540 |
|
}, |
|
{ |
|
"epoch": 0.4056529704265899, |
|
"grad_norm": 26.358328629224562, |
|
"learning_rate": 3.70822595774476e-07, |
|
"logits/chosen": -0.13673920929431915, |
|
"logits/rejected": 0.8279534578323364, |
|
"logps/chosen": -367.3028259277344, |
|
"logps/rejected": -376.034912109375, |
|
"loss": 0.6665, |
|
"rewards/accuracies": 0.675000011920929, |
|
"rewards/chosen": -137.3899688720703, |
|
"rewards/margins": 37.6882209777832, |
|
"rewards/rejected": -175.0781707763672, |
|
"step": 1550 |
|
}, |
|
{ |
|
"epoch": 0.408270086364826, |
|
"grad_norm": 28.476109770564317, |
|
"learning_rate": 3.688176236001168e-07, |
|
"logits/chosen": -0.4365410804748535, |
|
"logits/rejected": 0.026941100135445595, |
|
"logps/chosen": -361.25994873046875, |
|
"logps/rejected": -420.5032653808594, |
|
"loss": 0.6545, |
|
"rewards/accuracies": 0.625, |
|
"rewards/chosen": -106.12479400634766, |
|
"rewards/margins": 39.8696174621582, |
|
"rewards/rejected": -145.99441528320312, |
|
"step": 1560 |
|
}, |
|
{ |
|
"epoch": 0.410887202303062, |
|
"grad_norm": 28.010241239383195, |
|
"learning_rate": 3.6680273018838016e-07, |
|
"logits/chosen": -1.1525790691375732, |
|
"logits/rejected": 0.04848083108663559, |
|
"logps/chosen": -332.90716552734375, |
|
"logps/rejected": -363.8558044433594, |
|
"loss": 0.6434, |
|
"rewards/accuracies": 0.7250000238418579, |
|
"rewards/chosen": -97.48597717285156, |
|
"rewards/margins": 61.830177307128906, |
|
"rewards/rejected": -159.31617736816406, |
|
"step": 1570 |
|
}, |
|
{ |
|
"epoch": 0.4135043182412981, |
|
"grad_norm": 20.182263158112, |
|
"learning_rate": 3.6477808378228596e-07, |
|
"logits/chosen": -0.7257155776023865, |
|
"logits/rejected": 0.9191628694534302, |
|
"logps/chosen": -387.0204162597656, |
|
"logps/rejected": -464.8163146972656, |
|
"loss": 0.5897, |
|
"rewards/accuracies": 0.7749999761581421, |
|
"rewards/chosen": -95.88233947753906, |
|
"rewards/margins": 116.20806884765625, |
|
"rewards/rejected": -212.0903778076172, |
|
"step": 1580 |
|
}, |
|
{ |
|
"epoch": 0.4161214341795342, |
|
"grad_norm": 23.991143684652364, |
|
"learning_rate": 3.6274385343922674e-07, |
|
"logits/chosen": -0.1758570373058319, |
|
"logits/rejected": 1.9301687479019165, |
|
"logps/chosen": -406.59161376953125, |
|
"logps/rejected": -472.0311584472656, |
|
"loss": 0.6064, |
|
"rewards/accuracies": 0.8999999761581421, |
|
"rewards/chosen": -98.99046325683594, |
|
"rewards/margins": 119.24066162109375, |
|
"rewards/rejected": -218.2311248779297, |
|
"step": 1590 |
|
}, |
|
{ |
|
"epoch": 0.4187385501177702, |
|
"grad_norm": 28.972630342872332, |
|
"learning_rate": 3.6070020901685057e-07, |
|
"logits/chosen": 0.7765110731124878, |
|
"logits/rejected": 1.9538475275039673, |
|
"logps/chosen": -423.0254821777344, |
|
"logps/rejected": -460.91815185546875, |
|
"loss": 0.6559, |
|
"rewards/accuracies": 0.625, |
|
"rewards/chosen": -169.84713745117188, |
|
"rewards/margins": 64.01656341552734, |
|
"rewards/rejected": -233.8636932373047, |
|
"step": 1600 |
|
}, |
|
{ |
|
"epoch": 0.4213556660560063, |
|
"grad_norm": 41.02288224605125, |
|
"learning_rate": 3.5864732115887863e-07, |
|
"logits/chosen": 0.2611493766307831, |
|
"logits/rejected": 2.171125888824463, |
|
"logps/chosen": -425.1031799316406, |
|
"logps/rejected": -486.8959045410156, |
|
"loss": 0.6102, |
|
"rewards/accuracies": 0.7749999761581421, |
|
"rewards/chosen": -140.05665588378906, |
|
"rewards/margins": 108.6607437133789, |
|
"rewards/rejected": -248.7174072265625, |
|
"step": 1610 |
|
}, |
|
{ |
|
"epoch": 0.4239727819942423, |
|
"grad_norm": 25.694369532356813, |
|
"learning_rate": 3.565853612808562e-07, |
|
"logits/chosen": -0.3281291127204895, |
|
"logits/rejected": 1.471435785293579, |
|
"logps/chosen": -434.2025451660156, |
|
"logps/rejected": -423.0579528808594, |
|
"loss": 0.617, |
|
"rewards/accuracies": 0.824999988079071, |
|
"rewards/chosen": -112.0459213256836, |
|
"rewards/margins": 81.47265625, |
|
"rewards/rejected": -193.51858520507812, |
|
"step": 1620 |
|
}, |
|
{ |
|
"epoch": 0.4265898979324784, |
|
"grad_norm": 22.37578326192518, |
|
"learning_rate": 3.5451450155583984e-07, |
|
"logits/chosen": -0.8572834134101868, |
|
"logits/rejected": 0.5741704702377319, |
|
"logps/chosen": -437.5694274902344, |
|
"logps/rejected": -434.1827087402344, |
|
"loss": 0.6029, |
|
"rewards/accuracies": 0.6499999761581421, |
|
"rewards/chosen": -104.87544250488281, |
|
"rewards/margins": 64.55244445800781, |
|
"rewards/rejected": -169.42788696289062, |
|
"step": 1630 |
|
}, |
|
{ |
|
"epoch": 0.42920701387071447, |
|
"grad_norm": 22.286497715425547, |
|
"learning_rate": 3.5243491490002055e-07, |
|
"logits/chosen": 0.08054514974355698, |
|
"logits/rejected": 0.3288571834564209, |
|
"logps/chosen": -337.58953857421875, |
|
"logps/rejected": -490.2518615722656, |
|
"loss": 0.6122, |
|
"rewards/accuracies": 0.75, |
|
"rewards/chosen": -115.36927795410156, |
|
"rewards/margins": 88.35755157470703, |
|
"rewards/rejected": -203.726806640625, |
|
"step": 1640 |
|
}, |
|
{ |
|
"epoch": 0.4318241298089505, |
|
"grad_norm": 17.18902655970043, |
|
"learning_rate": 3.503467749582857e-07, |
|
"logits/chosen": -0.857958972454071, |
|
"logits/rejected": 0.0014857888454571366, |
|
"logps/chosen": -389.71380615234375, |
|
"logps/rejected": -466.3826599121094, |
|
"loss": 0.6358, |
|
"rewards/accuracies": 0.7250000238418579, |
|
"rewards/chosen": -107.03041076660156, |
|
"rewards/margins": 70.32191467285156, |
|
"rewards/rejected": -177.35232543945312, |
|
"step": 1650 |
|
}, |
|
{ |
|
"epoch": 0.4344412457471866, |
|
"grad_norm": 102.49373012437187, |
|
"learning_rate": 3.482502560897194e-07, |
|
"logits/chosen": -0.3570778965950012, |
|
"logits/rejected": 0.19285421073436737, |
|
"logps/chosen": -311.11236572265625, |
|
"logps/rejected": -359.0173645019531, |
|
"loss": 0.6409, |
|
"rewards/accuracies": 0.6499999761581421, |
|
"rewards/chosen": -113.4951171875, |
|
"rewards/margins": 34.06425857543945, |
|
"rewards/rejected": -147.5593719482422, |
|
"step": 1660 |
|
}, |
|
{ |
|
"epoch": 0.43705836168542267, |
|
"grad_norm": 30.752504031354707, |
|
"learning_rate": 3.4614553335304403e-07, |
|
"logits/chosen": -0.8707353472709656, |
|
"logits/rejected": -0.01903078332543373, |
|
"logps/chosen": -457.0826110839844, |
|
"logps/rejected": -451.1153259277344, |
|
"loss": 0.6616, |
|
"rewards/accuracies": 0.699999988079071, |
|
"rewards/chosen": -121.56687927246094, |
|
"rewards/margins": 53.41261672973633, |
|
"rewards/rejected": -174.9794921875, |
|
"step": 1670 |
|
}, |
|
{ |
|
"epoch": 0.4396754776236587, |
|
"grad_norm": 22.035717198899118, |
|
"learning_rate": 3.440327824920022e-07, |
|
"logits/chosen": -0.7067769765853882, |
|
"logits/rejected": 0.44209250807762146, |
|
"logps/chosen": -370.5931091308594, |
|
"logps/rejected": -435.90020751953125, |
|
"loss": 0.6212, |
|
"rewards/accuracies": 0.75, |
|
"rewards/chosen": -93.64749908447266, |
|
"rewards/margins": 78.0377197265625, |
|
"rewards/rejected": -171.68521118164062, |
|
"step": 1680 |
|
}, |
|
{ |
|
"epoch": 0.44229259356189476, |
|
"grad_norm": 80.94342434820176, |
|
"learning_rate": 3.4191217992068287e-07, |
|
"logits/chosen": -0.795087456703186, |
|
"logits/rejected": -0.14704521000385284, |
|
"logps/chosen": -319.4860534667969, |
|
"logps/rejected": -388.4473571777344, |
|
"loss": 0.5818, |
|
"rewards/accuracies": 0.7250000238418579, |
|
"rewards/chosen": -114.21246337890625, |
|
"rewards/margins": 56.797027587890625, |
|
"rewards/rejected": -171.0094757080078, |
|
"step": 1690 |
|
}, |
|
{ |
|
"epoch": 0.44490970950013087, |
|
"grad_norm": 30.56889810788254, |
|
"learning_rate": 3.3978390270879056e-07, |
|
"logits/chosen": -0.2528185248374939, |
|
"logits/rejected": 1.1413016319274902, |
|
"logps/chosen": -351.8018493652344, |
|
"logps/rejected": -426.44696044921875, |
|
"loss": 0.64, |
|
"rewards/accuracies": 0.800000011920929, |
|
"rewards/chosen": -101.383056640625, |
|
"rewards/margins": 88.7364273071289, |
|
"rewards/rejected": -190.11947631835938, |
|
"step": 1700 |
|
}, |
|
{ |
|
"epoch": 0.4475268254383669, |
|
"grad_norm": 36.49606260048181, |
|
"learning_rate": 3.376481285668599e-07, |
|
"logits/chosen": -0.07895590364933014, |
|
"logits/rejected": 1.1439310312271118, |
|
"logps/chosen": -411.49530029296875, |
|
"logps/rejected": -513.4984130859375, |
|
"loss": 0.6144, |
|
"rewards/accuracies": 0.824999988079071, |
|
"rewards/chosen": -121.69512939453125, |
|
"rewards/margins": 86.84122467041016, |
|
"rewards/rejected": -208.53634643554688, |
|
"step": 1710 |
|
}, |
|
{ |
|
"epoch": 0.45014394137660296, |
|
"grad_norm": 66.13535061783702, |
|
"learning_rate": 3.355050358314172e-07, |
|
"logits/chosen": 0.14188793301582336, |
|
"logits/rejected": 1.6301027536392212, |
|
"logps/chosen": -429.80035400390625, |
|
"logps/rejected": -522.5330810546875, |
|
"loss": 0.6426, |
|
"rewards/accuracies": 0.875, |
|
"rewards/chosen": -124.7596664428711, |
|
"rewards/margins": 104.18574523925781, |
|
"rewards/rejected": -228.9454345703125, |
|
"step": 1720 |
|
}, |
|
{ |
|
"epoch": 0.45276105731483907, |
|
"grad_norm": 29.691570545902252, |
|
"learning_rate": 3.33354803450089e-07, |
|
"logits/chosen": 1.466657280921936, |
|
"logits/rejected": 1.8799492120742798, |
|
"logps/chosen": -456.29180908203125, |
|
"logps/rejected": -482.02435302734375, |
|
"loss": 0.6357, |
|
"rewards/accuracies": 0.75, |
|
"rewards/chosen": -154.3424072265625, |
|
"rewards/margins": 59.94362258911133, |
|
"rewards/rejected": -214.28604125976562, |
|
"step": 1730 |
|
}, |
|
{ |
|
"epoch": 0.4553781732530751, |
|
"grad_norm": 46.404207705281095, |
|
"learning_rate": 3.311976109666605e-07, |
|
"logits/chosen": 0.9557245373725891, |
|
"logits/rejected": 2.0985617637634277, |
|
"logps/chosen": -417.13018798828125, |
|
"logps/rejected": -554.0835571289062, |
|
"loss": 0.6334, |
|
"rewards/accuracies": 0.875, |
|
"rewards/chosen": -138.3618621826172, |
|
"rewards/margins": 120.8489990234375, |
|
"rewards/rejected": -259.21087646484375, |
|
"step": 1740 |
|
}, |
|
{ |
|
"epoch": 0.45799528919131116, |
|
"grad_norm": 31.419258030314534, |
|
"learning_rate": 3.2903363850608317e-07, |
|
"logits/chosen": 1.717154860496521, |
|
"logits/rejected": 2.31295108795166, |
|
"logps/chosen": -435.69207763671875, |
|
"logps/rejected": -515.6895751953125, |
|
"loss": 0.6113, |
|
"rewards/accuracies": 0.75, |
|
"rewards/chosen": -178.9014129638672, |
|
"rewards/margins": 81.10930633544922, |
|
"rewards/rejected": -260.0107116699219, |
|
"step": 1750 |
|
}, |
|
{ |
|
"epoch": 0.46061240512954726, |
|
"grad_norm": 22.348281616345037, |
|
"learning_rate": 3.2686306675943477e-07, |
|
"logits/chosen": 1.0877479314804077, |
|
"logits/rejected": 3.3799686431884766, |
|
"logps/chosen": -437.88922119140625, |
|
"logps/rejected": -477.70269775390625, |
|
"loss": 0.6238, |
|
"rewards/accuracies": 0.824999988079071, |
|
"rewards/chosen": -176.03854370117188, |
|
"rewards/margins": 105.66455078125, |
|
"rewards/rejected": -281.703125, |
|
"step": 1760 |
|
}, |
|
{ |
|
"epoch": 0.4632295210677833, |
|
"grad_norm": 29.335315749809492, |
|
"learning_rate": 3.2468607696883145e-07, |
|
"logits/chosen": 1.5161404609680176, |
|
"logits/rejected": 2.1498818397521973, |
|
"logps/chosen": -442.09552001953125, |
|
"logps/rejected": -503.4599609375, |
|
"loss": 0.6141, |
|
"rewards/accuracies": 0.800000011920929, |
|
"rewards/chosen": -176.09494018554688, |
|
"rewards/margins": 75.00154876708984, |
|
"rewards/rejected": -251.0964813232422, |
|
"step": 1770 |
|
}, |
|
{ |
|
"epoch": 0.46584663700601936, |
|
"grad_norm": 44.758684680886404, |
|
"learning_rate": 3.2250285091229435e-07, |
|
"logits/chosen": 1.6801475286483765, |
|
"logits/rejected": 2.2319846153259277, |
|
"logps/chosen": -435.25421142578125, |
|
"logps/rejected": -482.51409912109375, |
|
"loss": 0.6287, |
|
"rewards/accuracies": 0.7749999761581421, |
|
"rewards/chosen": -183.05035400390625, |
|
"rewards/margins": 72.37165832519531, |
|
"rewards/rejected": -255.4219970703125, |
|
"step": 1780 |
|
}, |
|
{ |
|
"epoch": 0.4684637529442554, |
|
"grad_norm": 31.858283788755106, |
|
"learning_rate": 3.2031357088857083e-07, |
|
"logits/chosen": 2.3322768211364746, |
|
"logits/rejected": 3.3507091999053955, |
|
"logps/chosen": -466.07000732421875, |
|
"logps/rejected": -508.0931701660156, |
|
"loss": 0.6111, |
|
"rewards/accuracies": 0.824999988079071, |
|
"rewards/chosen": -181.24215698242188, |
|
"rewards/margins": 83.99452209472656, |
|
"rewards/rejected": -265.23663330078125, |
|
"step": 1790 |
|
}, |
|
{ |
|
"epoch": 0.4710808688824915, |
|
"grad_norm": 29.74675313933934, |
|
"learning_rate": 3.1811841970191267e-07, |
|
"logits/chosen": 1.5117183923721313, |
|
"logits/rejected": 2.4991650581359863, |
|
"logps/chosen": -473.57568359375, |
|
"logps/rejected": -536.4353637695312, |
|
"loss": 0.6134, |
|
"rewards/accuracies": 0.699999988079071, |
|
"rewards/chosen": -162.0258331298828, |
|
"rewards/margins": 80.58038330078125, |
|
"rewards/rejected": -242.6062469482422, |
|
"step": 1800 |
|
}, |
|
{ |
|
"epoch": 0.47369798482072756, |
|
"grad_norm": 27.35977465905274, |
|
"learning_rate": 3.1591758064681257e-07, |
|
"logits/chosen": 0.7779955863952637, |
|
"logits/rejected": 2.2222166061401367, |
|
"logps/chosen": -376.4127502441406, |
|
"logps/rejected": -483.91790771484375, |
|
"loss": 0.6337, |
|
"rewards/accuracies": 0.8500000238418579, |
|
"rewards/chosen": -168.23818969726562, |
|
"rewards/margins": 109.63935852050781, |
|
"rewards/rejected": -277.8775634765625, |
|
"step": 1810 |
|
}, |
|
{ |
|
"epoch": 0.4763151007589636, |
|
"grad_norm": 19.99968017850521, |
|
"learning_rate": 3.13711237492698e-07, |
|
"logits/chosen": 0.8206868171691895, |
|
"logits/rejected": 2.1102559566497803, |
|
"logps/chosen": -443.5704040527344, |
|
"logps/rejected": -483.1748046875, |
|
"loss": 0.6148, |
|
"rewards/accuracies": 0.7749999761581421, |
|
"rewards/chosen": -154.49142456054688, |
|
"rewards/margins": 85.27095794677734, |
|
"rewards/rejected": -239.7623748779297, |
|
"step": 1820 |
|
}, |
|
{ |
|
"epoch": 0.4789322166971997, |
|
"grad_norm": 21.48288399086547, |
|
"learning_rate": 3.1149957446858767e-07, |
|
"logits/chosen": 0.4431106448173523, |
|
"logits/rejected": 1.139479398727417, |
|
"logps/chosen": -385.325927734375, |
|
"logps/rejected": -440.3143005371094, |
|
"loss": 0.5981, |
|
"rewards/accuracies": 0.7749999761581421, |
|
"rewards/chosen": -143.1088104248047, |
|
"rewards/margins": 81.86103057861328, |
|
"rewards/rejected": -224.9698486328125, |
|
"step": 1830 |
|
}, |
|
{ |
|
"epoch": 0.48154933263543576, |
|
"grad_norm": 27.109010660723932, |
|
"learning_rate": 3.0928277624770736e-07, |
|
"logits/chosen": 0.6995161771774292, |
|
"logits/rejected": 1.9806559085845947, |
|
"logps/chosen": -373.56597900390625, |
|
"logps/rejected": -450.35137939453125, |
|
"loss": 0.613, |
|
"rewards/accuracies": 0.75, |
|
"rewards/chosen": -123.50726318359375, |
|
"rewards/margins": 83.88824462890625, |
|
"rewards/rejected": -207.3955078125, |
|
"step": 1840 |
|
}, |
|
{ |
|
"epoch": 0.4841664485736718, |
|
"grad_norm": 22.180659701287773, |
|
"learning_rate": 3.0706102793207073e-07, |
|
"logits/chosen": 0.9623844027519226, |
|
"logits/rejected": 1.8174850940704346, |
|
"logps/chosen": -443.29730224609375, |
|
"logps/rejected": -453.452392578125, |
|
"loss": 0.6264, |
|
"rewards/accuracies": 0.7749999761581421, |
|
"rewards/chosen": -143.41941833496094, |
|
"rewards/margins": 51.09202194213867, |
|
"rewards/rejected": -194.51145935058594, |
|
"step": 1850 |
|
}, |
|
{ |
|
"epoch": 0.48678356451190785, |
|
"grad_norm": 26.242087699994133, |
|
"learning_rate": 3.048345150370226e-07, |
|
"logits/chosen": 0.8996448516845703, |
|
"logits/rejected": 1.7648754119873047, |
|
"logps/chosen": -344.4842529296875, |
|
"logps/rejected": -445.62921142578125, |
|
"loss": 0.6158, |
|
"rewards/accuracies": 0.7749999761581421, |
|
"rewards/chosen": -114.01617431640625, |
|
"rewards/margins": 96.75332641601562, |
|
"rewards/rejected": -210.76950073242188, |
|
"step": 1860 |
|
}, |
|
{ |
|
"epoch": 0.48940068045014395, |
|
"grad_norm": 24.019046468101173, |
|
"learning_rate": 3.0260342347574913e-07, |
|
"logits/chosen": 0.12005790323019028, |
|
"logits/rejected": 2.031740188598633, |
|
"logps/chosen": -410.7184143066406, |
|
"logps/rejected": -538.4361572265625, |
|
"loss": 0.6192, |
|
"rewards/accuracies": 0.824999988079071, |
|
"rewards/chosen": -140.0440216064453, |
|
"rewards/margins": 124.01192474365234, |
|
"rewards/rejected": -264.05596923828125, |
|
"step": 1870 |
|
}, |
|
{ |
|
"epoch": 0.49201779638838, |
|
"grad_norm": 23.465321376516627, |
|
"learning_rate": 3.0036793954375357e-07, |
|
"logits/chosen": -0.1856812685728073, |
|
"logits/rejected": 0.7818856239318848, |
|
"logps/chosen": -422.85186767578125, |
|
"logps/rejected": -462.67852783203125, |
|
"loss": 0.6195, |
|
"rewards/accuracies": 0.75, |
|
"rewards/chosen": -120.50177001953125, |
|
"rewards/margins": 73.6959228515625, |
|
"rewards/rejected": -194.19769287109375, |
|
"step": 1880 |
|
}, |
|
{ |
|
"epoch": 0.49463491232661605, |
|
"grad_norm": 34.7409595895184, |
|
"learning_rate": 2.9812824990330085e-07, |
|
"logits/chosen": 0.8234826922416687, |
|
"logits/rejected": 1.6676380634307861, |
|
"logps/chosen": -393.53399658203125, |
|
"logps/rejected": -445.2818908691406, |
|
"loss": 0.6249, |
|
"rewards/accuracies": 0.7749999761581421, |
|
"rewards/chosen": -143.31112670898438, |
|
"rewards/margins": 62.34083938598633, |
|
"rewards/rejected": -205.65194702148438, |
|
"step": 1890 |
|
}, |
|
{ |
|
"epoch": 0.49725202826485215, |
|
"grad_norm": 46.69831571470922, |
|
"learning_rate": 2.958845415678316e-07, |
|
"logits/chosen": 0.357837975025177, |
|
"logits/rejected": 1.3686195611953735, |
|
"logps/chosen": -350.6862487792969, |
|
"logps/rejected": -424.646728515625, |
|
"loss": 0.6191, |
|
"rewards/accuracies": 0.75, |
|
"rewards/chosen": -139.62425231933594, |
|
"rewards/margins": 73.19682312011719, |
|
"rewards/rejected": -212.8210906982422, |
|
"step": 1900 |
|
}, |
|
{ |
|
"epoch": 0.4998691442030882, |
|
"grad_norm": 31.257813261511235, |
|
"learning_rate": 2.936370018863459e-07, |
|
"logits/chosen": 0.6837180852890015, |
|
"logits/rejected": 1.4936373233795166, |
|
"logps/chosen": -393.9849548339844, |
|
"logps/rejected": -457.9052734375, |
|
"loss": 0.629, |
|
"rewards/accuracies": 0.6000000238418579, |
|
"rewards/chosen": -145.4834747314453, |
|
"rewards/margins": 73.64918518066406, |
|
"rewards/rejected": -219.1326446533203, |
|
"step": 1910 |
|
}, |
|
{ |
|
"epoch": 0.5024862601413242, |
|
"grad_norm": 20.96702719121057, |
|
"learning_rate": 2.913858185277605e-07, |
|
"logits/chosen": 0.0933571457862854, |
|
"logits/rejected": 1.821059226989746, |
|
"logps/chosen": -408.1490173339844, |
|
"logps/rejected": -472.2452697753906, |
|
"loss": 0.6138, |
|
"rewards/accuracies": 0.75, |
|
"rewards/chosen": -137.5945587158203, |
|
"rewards/margins": 102.65091705322266, |
|
"rewards/rejected": -240.24545288085938, |
|
"step": 1920 |
|
}, |
|
{ |
|
"epoch": 0.5051033760795604, |
|
"grad_norm": 48.88683198411564, |
|
"learning_rate": 2.89131179465238e-07, |
|
"logits/chosen": 0.2965295910835266, |
|
"logits/rejected": 1.3621985912322998, |
|
"logps/chosen": -435.49591064453125, |
|
"logps/rejected": -500.6187438964844, |
|
"loss": 0.639, |
|
"rewards/accuracies": 0.7250000238418579, |
|
"rewards/chosen": -165.7919158935547, |
|
"rewards/margins": 73.27894592285156, |
|
"rewards/rejected": -239.07089233398438, |
|
"step": 1930 |
|
}, |
|
{ |
|
"epoch": 0.5077204920177963, |
|
"grad_norm": 16.11236019830028, |
|
"learning_rate": 2.8687327296049125e-07, |
|
"logits/chosen": 0.9378612637519836, |
|
"logits/rejected": 2.076969623565674, |
|
"logps/chosen": -454.8284606933594, |
|
"logps/rejected": -496.15289306640625, |
|
"loss": 0.6137, |
|
"rewards/accuracies": 0.824999988079071, |
|
"rewards/chosen": -160.6776123046875, |
|
"rewards/margins": 83.29139709472656, |
|
"rewards/rejected": -243.96902465820312, |
|
"step": 1940 |
|
}, |
|
{ |
|
"epoch": 0.5103376079560324, |
|
"grad_norm": 31.755303999536356, |
|
"learning_rate": 2.846122875480637e-07, |
|
"logits/chosen": 1.4026004076004028, |
|
"logits/rejected": 2.3323285579681396, |
|
"logps/chosen": -454.24560546875, |
|
"logps/rejected": -539.9759521484375, |
|
"loss": 0.6102, |
|
"rewards/accuracies": 0.699999988079071, |
|
"rewards/chosen": -189.9915008544922, |
|
"rewards/margins": 90.0781021118164, |
|
"rewards/rejected": -280.0696105957031, |
|
"step": 1950 |
|
}, |
|
{ |
|
"epoch": 0.5129547238942685, |
|
"grad_norm": 18.193341038011287, |
|
"learning_rate": 2.8234841201958647e-07, |
|
"logits/chosen": 1.2526977062225342, |
|
"logits/rejected": 2.378753423690796, |
|
"logps/chosen": -448.1241149902344, |
|
"logps/rejected": -493.2060546875, |
|
"loss": 0.6047, |
|
"rewards/accuracies": 0.699999988079071, |
|
"rewards/chosen": -148.1785888671875, |
|
"rewards/margins": 85.16536712646484, |
|
"rewards/rejected": -233.343994140625, |
|
"step": 1960 |
|
}, |
|
{ |
|
"epoch": 0.5155718398325045, |
|
"grad_norm": 42.523096927869034, |
|
"learning_rate": 2.800818354080148e-07, |
|
"logits/chosen": 0.8163898587226868, |
|
"logits/rejected": 2.261420965194702, |
|
"logps/chosen": -426.493896484375, |
|
"logps/rejected": -436.91143798828125, |
|
"loss": 0.6468, |
|
"rewards/accuracies": 0.7250000238418579, |
|
"rewards/chosen": -162.7000732421875, |
|
"rewards/margins": 73.9859619140625, |
|
"rewards/rejected": -236.68606567382812, |
|
"step": 1970 |
|
}, |
|
{ |
|
"epoch": 0.5181889557707406, |
|
"grad_norm": 28.67012335174378, |
|
"learning_rate": 2.778127469718435e-07, |
|
"logits/chosen": 0.3960002362728119, |
|
"logits/rejected": 1.7132211923599243, |
|
"logps/chosen": -446.56378173828125, |
|
"logps/rejected": -473.88348388671875, |
|
"loss": 0.6284, |
|
"rewards/accuracies": 0.7250000238418579, |
|
"rewards/chosen": -140.1986083984375, |
|
"rewards/margins": 91.8968505859375, |
|
"rewards/rejected": -232.095458984375, |
|
"step": 1980 |
|
}, |
|
{ |
|
"epoch": 0.5208060717089767, |
|
"grad_norm": 46.10962534230826, |
|
"learning_rate": 2.755413361793039e-07, |
|
"logits/chosen": 0.23405644297599792, |
|
"logits/rejected": 1.4183191061019897, |
|
"logps/chosen": -392.7248840332031, |
|
"logps/rejected": -422.74835205078125, |
|
"loss": 0.6547, |
|
"rewards/accuracies": 0.675000011920929, |
|
"rewards/chosen": -164.8948211669922, |
|
"rewards/margins": 51.055076599121094, |
|
"rewards/rejected": -215.9499053955078, |
|
"step": 1990 |
|
}, |
|
{ |
|
"epoch": 0.5234231876472127, |
|
"grad_norm": 41.84711365480925, |
|
"learning_rate": 2.7326779269254356e-07, |
|
"logits/chosen": -0.1836518496274948, |
|
"logits/rejected": 0.5241420865058899, |
|
"logps/chosen": -464.1949157714844, |
|
"logps/rejected": -582.6480102539062, |
|
"loss": 0.6145, |
|
"rewards/accuracies": 0.675000011920929, |
|
"rewards/chosen": -143.33895874023438, |
|
"rewards/margins": 63.12605667114258, |
|
"rewards/rejected": -206.4650421142578, |
|
"step": 2000 |
|
}, |
|
{ |
|
"epoch": 0.5260403035854488, |
|
"grad_norm": 83.93444593272139, |
|
"learning_rate": 2.709923063517895e-07, |
|
"logits/chosen": 0.15282206237316132, |
|
"logits/rejected": 2.0228190422058105, |
|
"logps/chosen": -439.91326904296875, |
|
"logps/rejected": -409.6415100097656, |
|
"loss": 0.6282, |
|
"rewards/accuracies": 0.699999988079071, |
|
"rewards/chosen": -131.47225952148438, |
|
"rewards/margins": 62.43913650512695, |
|
"rewards/rejected": -193.91139221191406, |
|
"step": 2010 |
|
}, |
|
{ |
|
"epoch": 0.528657419523685, |
|
"grad_norm": 32.728531816402544, |
|
"learning_rate": 2.68715067159496e-07, |
|
"logits/chosen": 0.6938730478286743, |
|
"logits/rejected": 1.1131725311279297, |
|
"logps/chosen": -420.9944763183594, |
|
"logps/rejected": -477.00750732421875, |
|
"loss": 0.6323, |
|
"rewards/accuracies": 0.75, |
|
"rewards/chosen": -132.22010803222656, |
|
"rewards/margins": 75.46855163574219, |
|
"rewards/rejected": -207.6886444091797, |
|
"step": 2020 |
|
}, |
|
{ |
|
"epoch": 0.5312745354619209, |
|
"grad_norm": 18.18975295132374, |
|
"learning_rate": 2.664362652644806e-07, |
|
"logits/chosen": 0.9111794233322144, |
|
"logits/rejected": 1.23908531665802, |
|
"logps/chosen": -419.4034118652344, |
|
"logps/rejected": -420.58935546875, |
|
"loss": 0.6008, |
|
"rewards/accuracies": 0.675000011920929, |
|
"rewards/chosen": -156.475341796875, |
|
"rewards/margins": 43.2105712890625, |
|
"rewards/rejected": -199.68592834472656, |
|
"step": 2030 |
|
}, |
|
{ |
|
"epoch": 0.533891651400157, |
|
"grad_norm": 99.13883885306136, |
|
"learning_rate": 2.6415609094604555e-07, |
|
"logits/chosen": 0.37541747093200684, |
|
"logits/rejected": 1.5170711278915405, |
|
"logps/chosen": -436.736328125, |
|
"logps/rejected": -561.4639892578125, |
|
"loss": 0.6181, |
|
"rewards/accuracies": 0.7749999761581421, |
|
"rewards/chosen": -134.65609741210938, |
|
"rewards/margins": 109.64598083496094, |
|
"rewards/rejected": -244.3020477294922, |
|
"step": 2040 |
|
}, |
|
{ |
|
"epoch": 0.5365087673383931, |
|
"grad_norm": 24.440438938042078, |
|
"learning_rate": 2.618747345980904e-07, |
|
"logits/chosen": 0.6494592428207397, |
|
"logits/rejected": 2.528566598892212, |
|
"logps/chosen": -452.0528259277344, |
|
"logps/rejected": -499.9761657714844, |
|
"loss": 0.6047, |
|
"rewards/accuracies": 0.875, |
|
"rewards/chosen": -124.5580062866211, |
|
"rewards/margins": 98.02928161621094, |
|
"rewards/rejected": -222.5872802734375, |
|
"step": 2050 |
|
}, |
|
{ |
|
"epoch": 0.5391258832766291, |
|
"grad_norm": 41.64782826080612, |
|
"learning_rate": 2.595923867132136e-07, |
|
"logits/chosen": 2.0148017406463623, |
|
"logits/rejected": 2.202955961227417, |
|
"logps/chosen": -395.0124206542969, |
|
"logps/rejected": -520.9989013671875, |
|
"loss": 0.5933, |
|
"rewards/accuracies": 0.75, |
|
"rewards/chosen": -150.31761169433594, |
|
"rewards/margins": 89.17308807373047, |
|
"rewards/rejected": -239.49072265625, |
|
"step": 2060 |
|
}, |
|
{ |
|
"epoch": 0.5417429992148652, |
|
"grad_norm": 28.897098135028244, |
|
"learning_rate": 2.5730923786680667e-07, |
|
"logits/chosen": 0.9622161984443665, |
|
"logits/rejected": 2.1357316970825195, |
|
"logps/chosen": -442.57696533203125, |
|
"logps/rejected": -503.89483642578125, |
|
"loss": 0.596, |
|
"rewards/accuracies": 0.824999988079071, |
|
"rewards/chosen": -129.53762817382812, |
|
"rewards/margins": 110.2646713256836, |
|
"rewards/rejected": -239.8023223876953, |
|
"step": 2070 |
|
}, |
|
{ |
|
"epoch": 0.5443601151531012, |
|
"grad_norm": 37.61810979984936, |
|
"learning_rate": 2.5502547870114135e-07, |
|
"logits/chosen": 0.902941882610321, |
|
"logits/rejected": 1.3675785064697266, |
|
"logps/chosen": -324.65252685546875, |
|
"logps/rejected": -408.60992431640625, |
|
"loss": 0.6666, |
|
"rewards/accuracies": 0.824999988079071, |
|
"rewards/chosen": -123.71934509277344, |
|
"rewards/margins": 79.65061950683594, |
|
"rewards/rejected": -203.3699951171875, |
|
"step": 2080 |
|
}, |
|
{ |
|
"epoch": 0.5469772310913373, |
|
"grad_norm": 22.119217806890422, |
|
"learning_rate": 2.527412999094506e-07, |
|
"logits/chosen": 0.31701117753982544, |
|
"logits/rejected": 1.13541579246521, |
|
"logps/chosen": -423.2291564941406, |
|
"logps/rejected": -473.520263671875, |
|
"loss": 0.6529, |
|
"rewards/accuracies": 0.7250000238418579, |
|
"rewards/chosen": -144.65245056152344, |
|
"rewards/margins": 78.18202209472656, |
|
"rewards/rejected": -222.83447265625, |
|
"step": 2090 |
|
}, |
|
{ |
|
"epoch": 0.5495943470295734, |
|
"grad_norm": 12.04734469718736, |
|
"learning_rate": 2.5045689222000636e-07, |
|
"logits/chosen": -0.7169623374938965, |
|
"logits/rejected": -0.41885489225387573, |
|
"logps/chosen": -376.8286437988281, |
|
"logps/rejected": -467.8668518066406, |
|
"loss": 0.6003, |
|
"rewards/accuracies": 0.875, |
|
"rewards/chosen": -99.94172668457031, |
|
"rewards/margins": 90.81985473632812, |
|
"rewards/rejected": -190.76156616210938, |
|
"step": 2100 |
|
}, |
|
{ |
|
"epoch": 0.5522114629678094, |
|
"grad_norm": 45.74976103694579, |
|
"learning_rate": 2.481724463801933e-07, |
|
"logits/chosen": -0.09558014571666718, |
|
"logits/rejected": 1.1860709190368652, |
|
"logps/chosen": -360.15411376953125, |
|
"logps/rejected": -399.21685791015625, |
|
"loss": 0.6332, |
|
"rewards/accuracies": 0.7749999761581421, |
|
"rewards/chosen": -110.9947738647461, |
|
"rewards/margins": 68.16171264648438, |
|
"rewards/rejected": -179.15647888183594, |
|
"step": 2110 |
|
}, |
|
{ |
|
"epoch": 0.5548285789060455, |
|
"grad_norm": 30.058542677557412, |
|
"learning_rate": 2.4588815314058154e-07, |
|
"logits/chosen": -0.4582470953464508, |
|
"logits/rejected": 1.3120964765548706, |
|
"logps/chosen": -421.02655029296875, |
|
"logps/rejected": -456.4002380371094, |
|
"loss": 0.6392, |
|
"rewards/accuracies": 0.800000011920929, |
|
"rewards/chosen": -84.15247344970703, |
|
"rewards/margins": 106.6784439086914, |
|
"rewards/rejected": -190.83090209960938, |
|
"step": 2120 |
|
}, |
|
{ |
|
"epoch": 0.5574456948442816, |
|
"grad_norm": 24.331335326894454, |
|
"learning_rate": 2.4360420323899917e-07, |
|
"logits/chosen": 0.15577450394630432, |
|
"logits/rejected": 1.3761570453643799, |
|
"logps/chosen": -350.9698791503906, |
|
"logps/rejected": -469.49090576171875, |
|
"loss": 0.5898, |
|
"rewards/accuracies": 0.800000011920929, |
|
"rewards/chosen": -98.12188720703125, |
|
"rewards/margins": 96.63127136230469, |
|
"rewards/rejected": -194.75314331054688, |
|
"step": 2130 |
|
}, |
|
{ |
|
"epoch": 0.5600628107825176, |
|
"grad_norm": 35.87275267194999, |
|
"learning_rate": 2.4132078738460583e-07, |
|
"logits/chosen": 0.9583314061164856, |
|
"logits/rejected": 1.5858229398727417, |
|
"logps/chosen": -399.7995910644531, |
|
"logps/rejected": -457.26116943359375, |
|
"loss": 0.6056, |
|
"rewards/accuracies": 0.800000011920929, |
|
"rewards/chosen": -126.67093658447266, |
|
"rewards/margins": 60.547874450683594, |
|
"rewards/rejected": -187.21881103515625, |
|
"step": 2140 |
|
}, |
|
{ |
|
"epoch": 0.5626799267207537, |
|
"grad_norm": 39.761293537479226, |
|
"learning_rate": 2.390380962419682e-07, |
|
"logits/chosen": 0.7930958867073059, |
|
"logits/rejected": 2.482306957244873, |
|
"logps/chosen": -334.648193359375, |
|
"logps/rejected": -446.80438232421875, |
|
"loss": 0.6153, |
|
"rewards/accuracies": 0.824999988079071, |
|
"rewards/chosen": -103.2480239868164, |
|
"rewards/margins": 119.85728454589844, |
|
"rewards/rejected": -223.1052703857422, |
|
"step": 2150 |
|
}, |
|
{ |
|
"epoch": 0.5652970426589898, |
|
"grad_norm": 31.853624067752015, |
|
"learning_rate": 2.3675632041513977e-07, |
|
"logits/chosen": 0.531602680683136, |
|
"logits/rejected": 1.801783561706543, |
|
"logps/chosen": -399.61419677734375, |
|
"logps/rejected": -508.2198791503906, |
|
"loss": 0.6153, |
|
"rewards/accuracies": 0.800000011920929, |
|
"rewards/chosen": -125.03511810302734, |
|
"rewards/margins": 88.95806121826172, |
|
"rewards/rejected": -213.99319458007812, |
|
"step": 2160 |
|
}, |
|
{ |
|
"epoch": 0.5679141585972258, |
|
"grad_norm": 33.621655259329046, |
|
"learning_rate": 2.344756504317453e-07, |
|
"logits/chosen": 0.4952963888645172, |
|
"logits/rejected": 2.3254141807556152, |
|
"logps/chosen": -363.47210693359375, |
|
"logps/rejected": -426.3758850097656, |
|
"loss": 0.6349, |
|
"rewards/accuracies": 0.7749999761581421, |
|
"rewards/chosen": -104.16854095458984, |
|
"rewards/margins": 107.5788803100586, |
|
"rewards/rejected": -211.7474365234375, |
|
"step": 2170 |
|
}, |
|
{ |
|
"epoch": 0.5705312745354619, |
|
"grad_norm": 36.50906825096521, |
|
"learning_rate": 2.3219627672707237e-07, |
|
"logits/chosen": -0.048846520483493805, |
|
"logits/rejected": 1.8363500833511353, |
|
"logps/chosen": -395.2628479003906, |
|
"logps/rejected": -441.28546142578125, |
|
"loss": 0.6398, |
|
"rewards/accuracies": 0.800000011920929, |
|
"rewards/chosen": -83.27033996582031, |
|
"rewards/margins": 108.64981842041016, |
|
"rewards/rejected": -191.920166015625, |
|
"step": 2180 |
|
}, |
|
{ |
|
"epoch": 0.573148390473698, |
|
"grad_norm": 27.414128348892337, |
|
"learning_rate": 2.2991838962816918e-07, |
|
"logits/chosen": -0.003201258135959506, |
|
"logits/rejected": 1.6773014068603516, |
|
"logps/chosen": -340.6254577636719, |
|
"logps/rejected": -424.5982360839844, |
|
"loss": 0.5947, |
|
"rewards/accuracies": 0.824999988079071, |
|
"rewards/chosen": -71.3757095336914, |
|
"rewards/margins": 119.04563903808594, |
|
"rewards/rejected": -190.42135620117188, |
|
"step": 2190 |
|
}, |
|
{ |
|
"epoch": 0.575765506411934, |
|
"grad_norm": 20.12881929325743, |
|
"learning_rate": 2.2764217933795297e-07, |
|
"logits/chosen": 0.325003445148468, |
|
"logits/rejected": 2.2267208099365234, |
|
"logps/chosen": -318.68359375, |
|
"logps/rejected": -379.50787353515625, |
|
"loss": 0.5855, |
|
"rewards/accuracies": 0.7250000238418579, |
|
"rewards/chosen": -84.79862976074219, |
|
"rewards/margins": 93.99121856689453, |
|
"rewards/rejected": -178.7898406982422, |
|
"step": 2200 |
|
}, |
|
{ |
|
"epoch": 0.5783826223501701, |
|
"grad_norm": 28.1067572997552, |
|
"learning_rate": 2.253678359193278e-07, |
|
"logits/chosen": 0.6280830502510071, |
|
"logits/rejected": 2.3489346504211426, |
|
"logps/chosen": -494.9659729003906, |
|
"logps/rejected": -529.9819946289062, |
|
"loss": 0.6309, |
|
"rewards/accuracies": 0.75, |
|
"rewards/chosen": -138.376220703125, |
|
"rewards/margins": 81.79132843017578, |
|
"rewards/rejected": -220.1675262451172, |
|
"step": 2210 |
|
}, |
|
{ |
|
"epoch": 0.5809997382884062, |
|
"grad_norm": 19.494344171492532, |
|
"learning_rate": 2.230955492793149e-07, |
|
"logits/chosen": 1.8268368244171143, |
|
"logits/rejected": 2.58113956451416, |
|
"logps/chosen": -332.82220458984375, |
|
"logps/rejected": -433.9080505371094, |
|
"loss": 0.6259, |
|
"rewards/accuracies": 0.800000011920929, |
|
"rewards/chosen": -130.2157745361328, |
|
"rewards/margins": 85.47044372558594, |
|
"rewards/rejected": -215.6862335205078, |
|
"step": 2220 |
|
}, |
|
{ |
|
"epoch": 0.5836168542266422, |
|
"grad_norm": 51.63490469991745, |
|
"learning_rate": 2.2082550915319468e-07, |
|
"logits/chosen": 0.8697152137756348, |
|
"logits/rejected": 1.921521782875061, |
|
"logps/chosen": -420.7347106933594, |
|
"logps/rejected": -502.9576110839844, |
|
"loss": 0.6465, |
|
"rewards/accuracies": 0.75, |
|
"rewards/chosen": -123.50699615478516, |
|
"rewards/margins": 93.5182113647461, |
|
"rewards/rejected": -217.0252227783203, |
|
"step": 2230 |
|
}, |
|
{ |
|
"epoch": 0.5862339701648783, |
|
"grad_norm": 32.248455012221434, |
|
"learning_rate": 2.1855790508866433e-07, |
|
"logits/chosen": 0.9172471761703491, |
|
"logits/rejected": 2.2043850421905518, |
|
"logps/chosen": -384.3970031738281, |
|
"logps/rejected": -411.32373046875, |
|
"loss": 0.6135, |
|
"rewards/accuracies": 0.75, |
|
"rewards/chosen": -109.10707092285156, |
|
"rewards/margins": 77.65843200683594, |
|
"rewards/rejected": -186.76548767089844, |
|
"step": 2240 |
|
}, |
|
{ |
|
"epoch": 0.5888510861031143, |
|
"grad_norm": 34.745998393806474, |
|
"learning_rate": 2.162929264300107e-07, |
|
"logits/chosen": 1.7178211212158203, |
|
"logits/rejected": 1.943967580795288, |
|
"logps/chosen": -419.106689453125, |
|
"logps/rejected": -513.5217895507812, |
|
"loss": 0.619, |
|
"rewards/accuracies": 0.699999988079071, |
|
"rewards/chosen": -154.78704833984375, |
|
"rewards/margins": 75.63910675048828, |
|
"rewards/rejected": -230.4261474609375, |
|
"step": 2250 |
|
}, |
|
{ |
|
"epoch": 0.5914682020413504, |
|
"grad_norm": 39.28692190563846, |
|
"learning_rate": 2.1403076230230005e-07, |
|
"logits/chosen": 0.5459718108177185, |
|
"logits/rejected": 1.7720600366592407, |
|
"logps/chosen": -366.18939208984375, |
|
"logps/rejected": -477.49725341796875, |
|
"loss": 0.6097, |
|
"rewards/accuracies": 0.699999988079071, |
|
"rewards/chosen": -132.69638061523438, |
|
"rewards/margins": 80.7729263305664, |
|
"rewards/rejected": -213.4693145751953, |
|
"step": 2260 |
|
}, |
|
{ |
|
"epoch": 0.5940853179795865, |
|
"grad_norm": 35.79367820795886, |
|
"learning_rate": 2.1177160159558596e-07, |
|
"logits/chosen": 0.2836107313632965, |
|
"logits/rejected": 1.257423758506775, |
|
"logps/chosen": -405.7095947265625, |
|
"logps/rejected": -575.5140991210938, |
|
"loss": 0.5878, |
|
"rewards/accuracies": 0.800000011920929, |
|
"rewards/chosen": -130.59938049316406, |
|
"rewards/margins": 115.3625717163086, |
|
"rewards/rejected": -245.9619598388672, |
|
"step": 2270 |
|
}, |
|
{ |
|
"epoch": 0.5967024339178225, |
|
"grad_norm": 43.109152856420366, |
|
"learning_rate": 2.0951563294913734e-07, |
|
"logits/chosen": 0.8286579847335815, |
|
"logits/rejected": 1.5607203245162964, |
|
"logps/chosen": -433.5328674316406, |
|
"logps/rejected": -459.05615234375, |
|
"loss": 0.6721, |
|
"rewards/accuracies": 0.675000011920929, |
|
"rewards/chosen": -121.0738296508789, |
|
"rewards/margins": 54.7202262878418, |
|
"rewards/rejected": -175.7940673828125, |
|
"step": 2280 |
|
}, |
|
{ |
|
"epoch": 0.5993195498560586, |
|
"grad_norm": 44.27442075290467, |
|
"learning_rate": 2.072630447356869e-07, |
|
"logits/chosen": 1.2522351741790771, |
|
"logits/rejected": 2.194676160812378, |
|
"logps/chosen": -375.2573547363281, |
|
"logps/rejected": -454.4563903808594, |
|
"loss": 0.6213, |
|
"rewards/accuracies": 0.7250000238418579, |
|
"rewards/chosen": -133.56240844726562, |
|
"rewards/margins": 69.3228759765625, |
|
"rewards/rejected": -202.8852996826172, |
|
"step": 2290 |
|
}, |
|
{ |
|
"epoch": 0.6019366657942947, |
|
"grad_norm": 35.873403321724766, |
|
"learning_rate": 2.0501402504570232e-07, |
|
"logits/chosen": 1.2167752981185913, |
|
"logits/rejected": 2.3980419635772705, |
|
"logps/chosen": -382.34857177734375, |
|
"logps/rejected": -458.5743713378906, |
|
"loss": 0.6282, |
|
"rewards/accuracies": 0.7749999761581421, |
|
"rewards/chosen": -136.52151489257812, |
|
"rewards/margins": 77.73162841796875, |
|
"rewards/rejected": -214.253173828125, |
|
"step": 2300 |
|
}, |
|
{ |
|
"epoch": 0.6045537817325307, |
|
"grad_norm": 35.83497109520107, |
|
"learning_rate": 2.027687616716804e-07, |
|
"logits/chosen": 0.8540589213371277, |
|
"logits/rejected": 1.2983033657073975, |
|
"logps/chosen": -522.3756713867188, |
|
"logps/rejected": -499.83544921875, |
|
"loss": 0.6397, |
|
"rewards/accuracies": 0.5249999761581421, |
|
"rewards/chosen": -161.60308837890625, |
|
"rewards/margins": 24.720584869384766, |
|
"rewards/rejected": -186.3236541748047, |
|
"step": 2310 |
|
}, |
|
{ |
|
"epoch": 0.6071708976707668, |
|
"grad_norm": 21.18793615252519, |
|
"learning_rate": 2.005274420924668e-07, |
|
"logits/chosen": 0.7288818359375, |
|
"logits/rejected": 2.1675353050231934, |
|
"logps/chosen": -509.79248046875, |
|
"logps/rejected": -634.5296020507812, |
|
"loss": 0.634, |
|
"rewards/accuracies": 0.800000011920929, |
|
"rewards/chosen": -152.21360778808594, |
|
"rewards/margins": 104.65443420410156, |
|
"rewards/rejected": -256.8680114746094, |
|
"step": 2320 |
|
}, |
|
{ |
|
"epoch": 0.6097880136090029, |
|
"grad_norm": 22.50228145741483, |
|
"learning_rate": 1.9829025345760121e-07, |
|
"logits/chosen": 0.6618310213088989, |
|
"logits/rejected": 2.1998677253723145, |
|
"logps/chosen": -411.01348876953125, |
|
"logps/rejected": -443.0804138183594, |
|
"loss": 0.5998, |
|
"rewards/accuracies": 0.7250000238418579, |
|
"rewards/chosen": -146.52345275878906, |
|
"rewards/margins": 60.154937744140625, |
|
"rewards/rejected": -206.6783905029297, |
|
"step": 2330 |
|
}, |
|
{ |
|
"epoch": 0.6124051295472389, |
|
"grad_norm": 25.601857295410756, |
|
"learning_rate": 1.960573825716911e-07, |
|
"logits/chosen": 0.7051969766616821, |
|
"logits/rejected": 1.6920392513275146, |
|
"logps/chosen": -393.4776306152344, |
|
"logps/rejected": -550.2787475585938, |
|
"loss": 0.6275, |
|
"rewards/accuracies": 0.875, |
|
"rewards/chosen": -121.97821044921875, |
|
"rewards/margins": 120.07755279541016, |
|
"rewards/rejected": -242.0557403564453, |
|
"step": 2340 |
|
}, |
|
{ |
|
"epoch": 0.615022245485475, |
|
"grad_norm": 19.213556153517377, |
|
"learning_rate": 1.9382901587881273e-07, |
|
"logits/chosen": 1.1434944868087769, |
|
"logits/rejected": 1.3964121341705322, |
|
"logps/chosen": -384.5157470703125, |
|
"logps/rejected": -480.2007751464844, |
|
"loss": 0.594, |
|
"rewards/accuracies": 0.800000011920929, |
|
"rewards/chosen": -120.44758605957031, |
|
"rewards/margins": 86.69746398925781, |
|
"rewards/rejected": -207.1450653076172, |
|
"step": 2350 |
|
}, |
|
{ |
|
"epoch": 0.6176393614237111, |
|
"grad_norm": 25.557737674180782, |
|
"learning_rate": 1.9160533944694364e-07, |
|
"logits/chosen": 0.3543923497200012, |
|
"logits/rejected": 1.8675868511199951, |
|
"logps/chosen": -421.5625, |
|
"logps/rejected": -434.99102783203125, |
|
"loss": 0.6164, |
|
"rewards/accuracies": 0.675000011920929, |
|
"rewards/chosen": -150.8063507080078, |
|
"rewards/margins": 53.77281951904297, |
|
"rewards/rejected": -204.5791473388672, |
|
"step": 2360 |
|
}, |
|
{ |
|
"epoch": 0.6202564773619471, |
|
"grad_norm": 158.85223594365854, |
|
"learning_rate": 1.8938653895242602e-07, |
|
"logits/chosen": 1.4026604890823364, |
|
"logits/rejected": 2.489152193069458, |
|
"logps/chosen": -349.5987854003906, |
|
"logps/rejected": -491.1888732910156, |
|
"loss": 0.605, |
|
"rewards/accuracies": 0.875, |
|
"rewards/chosen": -127.90867614746094, |
|
"rewards/margins": 113.81087493896484, |
|
"rewards/rejected": -241.7195587158203, |
|
"step": 2370 |
|
}, |
|
{ |
|
"epoch": 0.6228735933001832, |
|
"grad_norm": 25.94527480965663, |
|
"learning_rate": 1.8717279966446264e-07, |
|
"logits/chosen": 0.9932606816291809, |
|
"logits/rejected": 1.2095084190368652, |
|
"logps/chosen": -425.6875, |
|
"logps/rejected": -495.3365173339844, |
|
"loss": 0.6252, |
|
"rewards/accuracies": 0.6499999761581421, |
|
"rewards/chosen": -143.76849365234375, |
|
"rewards/margins": 48.57000732421875, |
|
"rewards/rejected": -192.3385009765625, |
|
"step": 2380 |
|
}, |
|
{ |
|
"epoch": 0.6254907092384192, |
|
"grad_norm": 35.86272517082607, |
|
"learning_rate": 1.8496430642964694e-07, |
|
"logits/chosen": 1.6592071056365967, |
|
"logits/rejected": 3.2019355297088623, |
|
"logps/chosen": -449.941650390625, |
|
"logps/rejected": -503.06689453125, |
|
"loss": 0.6355, |
|
"rewards/accuracies": 0.7250000238418579, |
|
"rewards/chosen": -154.45082092285156, |
|
"rewards/margins": 97.35575866699219, |
|
"rewards/rejected": -251.8065948486328, |
|
"step": 2390 |
|
}, |
|
{ |
|
"epoch": 0.6281078251766553, |
|
"grad_norm": 33.36347925046869, |
|
"learning_rate": 1.8276124365652855e-07, |
|
"logits/chosen": 1.782403588294983, |
|
"logits/rejected": 2.6725101470947266, |
|
"logps/chosen": -414.8662109375, |
|
"logps/rejected": -513.3907470703125, |
|
"loss": 0.617, |
|
"rewards/accuracies": 0.800000011920929, |
|
"rewards/chosen": -152.33621215820312, |
|
"rewards/margins": 84.60549926757812, |
|
"rewards/rejected": -236.9416961669922, |
|
"step": 2400 |
|
}, |
|
{ |
|
"epoch": 0.6307249411148914, |
|
"grad_norm": 27.211887112083538, |
|
"learning_rate": 1.805637953002149e-07, |
|
"logits/chosen": 1.0229891538619995, |
|
"logits/rejected": 2.227069616317749, |
|
"logps/chosen": -412.5189514160156, |
|
"logps/rejected": -546.1232299804688, |
|
"loss": 0.6067, |
|
"rewards/accuracies": 0.824999988079071, |
|
"rewards/chosen": -142.4264373779297, |
|
"rewards/margins": 105.69502258300781, |
|
"rewards/rejected": -248.12142944335938, |
|
"step": 2410 |
|
}, |
|
{ |
|
"epoch": 0.6333420570531274, |
|
"grad_norm": 34.88818302440242, |
|
"learning_rate": 1.7837214484701153e-07, |
|
"logits/chosen": 1.1308300495147705, |
|
"logits/rejected": 1.0798611640930176, |
|
"logps/chosen": -440.8893127441406, |
|
"logps/rejected": -540.1508178710938, |
|
"loss": 0.6435, |
|
"rewards/accuracies": 0.800000011920929, |
|
"rewards/chosen": -157.52186584472656, |
|
"rewards/margins": 75.15589904785156, |
|
"rewards/rejected": -232.677734375, |
|
"step": 2420 |
|
}, |
|
{ |
|
"epoch": 0.6359591729913635, |
|
"grad_norm": 38.54161473588103, |
|
"learning_rate": 1.761864752991004e-07, |
|
"logits/chosen": 0.743876576423645, |
|
"logits/rejected": 2.1762986183166504, |
|
"logps/chosen": -427.7271423339844, |
|
"logps/rejected": -495.25909423828125, |
|
"loss": 0.6099, |
|
"rewards/accuracies": 0.7749999761581421, |
|
"rewards/chosen": -151.75076293945312, |
|
"rewards/margins": 84.32628631591797, |
|
"rewards/rejected": -236.0770263671875, |
|
"step": 2430 |
|
}, |
|
{ |
|
"epoch": 0.6385762889295996, |
|
"grad_norm": 30.683794106965227, |
|
"learning_rate": 1.7400696915925995e-07, |
|
"logits/chosen": 0.9094554781913757, |
|
"logits/rejected": 2.102067708969116, |
|
"logps/chosen": -393.0418395996094, |
|
"logps/rejected": -448.5322265625, |
|
"loss": 0.6029, |
|
"rewards/accuracies": 0.875, |
|
"rewards/chosen": -150.30026245117188, |
|
"rewards/margins": 81.52594757080078, |
|
"rewards/rejected": -231.8262176513672, |
|
"step": 2440 |
|
}, |
|
{ |
|
"epoch": 0.6411934048678356, |
|
"grad_norm": 120.87900499444974, |
|
"learning_rate": 1.718338084156254e-07, |
|
"logits/chosen": 0.7895253300666809, |
|
"logits/rejected": 1.4243078231811523, |
|
"logps/chosen": -478.90643310546875, |
|
"logps/rejected": -506.8853454589844, |
|
"loss": 0.6337, |
|
"rewards/accuracies": 0.75, |
|
"rewards/chosen": -147.2356414794922, |
|
"rewards/margins": 72.50621032714844, |
|
"rewards/rejected": -219.74185180664062, |
|
"step": 2450 |
|
}, |
|
{ |
|
"epoch": 0.6438105208060717, |
|
"grad_norm": 43.85428932744377, |
|
"learning_rate": 1.696671745264937e-07, |
|
"logits/chosen": 0.8812819719314575, |
|
"logits/rejected": 2.2113723754882812, |
|
"logps/chosen": -512.0079345703125, |
|
"logps/rejected": -543.0887451171875, |
|
"loss": 0.6242, |
|
"rewards/accuracies": 0.7250000238418579, |
|
"rewards/chosen": -182.76455688476562, |
|
"rewards/margins": 80.2233657836914, |
|
"rewards/rejected": -262.9878845214844, |
|
"step": 2460 |
|
}, |
|
{ |
|
"epoch": 0.6464276367443078, |
|
"grad_norm": 18.352614590491243, |
|
"learning_rate": 1.67507248405171e-07, |
|
"logits/chosen": 0.06380853801965714, |
|
"logits/rejected": 1.635575532913208, |
|
"logps/chosen": -411.78948974609375, |
|
"logps/rejected": -411.417724609375, |
|
"loss": 0.6258, |
|
"rewards/accuracies": 0.6499999761581421, |
|
"rewards/chosen": -135.79331970214844, |
|
"rewards/margins": 60.68121337890625, |
|
"rewards/rejected": -196.4745330810547, |
|
"step": 2470 |
|
}, |
|
{ |
|
"epoch": 0.6490447526825438, |
|
"grad_norm": 27.292806167535627, |
|
"learning_rate": 1.6535421040486683e-07, |
|
"logits/chosen": 0.4601976275444031, |
|
"logits/rejected": 1.9244674444198608, |
|
"logps/chosen": -420.33978271484375, |
|
"logps/rejected": -493.8761291503906, |
|
"loss": 0.5955, |
|
"rewards/accuracies": 0.8500000238418579, |
|
"rewards/chosen": -144.494384765625, |
|
"rewards/margins": 96.97511291503906, |
|
"rewards/rejected": -241.469482421875, |
|
"step": 2480 |
|
}, |
|
{ |
|
"epoch": 0.6516618686207799, |
|
"grad_norm": 44.574204509093036, |
|
"learning_rate": 1.6320824030363456e-07, |
|
"logits/chosen": 1.7913410663604736, |
|
"logits/rejected": 3.195528984069824, |
|
"logps/chosen": -437.32464599609375, |
|
"logps/rejected": -459.4554748535156, |
|
"loss": 0.6501, |
|
"rewards/accuracies": 0.625, |
|
"rewards/chosen": -181.6840362548828, |
|
"rewards/margins": 56.456878662109375, |
|
"rewards/rejected": -238.14089965820312, |
|
"step": 2490 |
|
}, |
|
{ |
|
"epoch": 0.654278984559016, |
|
"grad_norm": 24.375749373782856, |
|
"learning_rate": 1.6106951728936024e-07, |
|
"logits/chosen": 0.45715102553367615, |
|
"logits/rejected": 1.6011425256729126, |
|
"logps/chosen": -395.44354248046875, |
|
"logps/rejected": -513.138916015625, |
|
"loss": 0.6131, |
|
"rewards/accuracies": 0.625, |
|
"rewards/chosen": -115.36625671386719, |
|
"rewards/margins": 79.81534576416016, |
|
"rewards/rejected": -195.18161010742188, |
|
"step": 2500 |
|
}, |
|
{ |
|
"epoch": 0.656896100497252, |
|
"grad_norm": 21.478682367181154, |
|
"learning_rate": 1.5893821994479994e-07, |
|
"logits/chosen": 0.34590616822242737, |
|
"logits/rejected": 2.354616165161133, |
|
"logps/chosen": -446.23626708984375, |
|
"logps/rejected": -481.48394775390625, |
|
"loss": 0.6221, |
|
"rewards/accuracies": 0.800000011920929, |
|
"rewards/chosen": -119.68449401855469, |
|
"rewards/margins": 107.2177963256836, |
|
"rewards/rejected": -226.9022979736328, |
|
"step": 2510 |
|
}, |
|
{ |
|
"epoch": 0.6595132164354881, |
|
"grad_norm": 30.65322954729203, |
|
"learning_rate": 1.5681452623266867e-07, |
|
"logits/chosen": 1.2941632270812988, |
|
"logits/rejected": 2.3193411827087402, |
|
"logps/chosen": -414.8558044433594, |
|
"logps/rejected": -468.5655212402344, |
|
"loss": 0.6167, |
|
"rewards/accuracies": 0.625, |
|
"rewards/chosen": -155.61769104003906, |
|
"rewards/margins": 55.00055694580078, |
|
"rewards/rejected": -210.6182403564453, |
|
"step": 2520 |
|
}, |
|
{ |
|
"epoch": 0.6621303323737242, |
|
"grad_norm": 49.039161022991905, |
|
"learning_rate": 1.546986134807801e-07, |
|
"logits/chosen": 1.6722854375839233, |
|
"logits/rejected": 2.5984983444213867, |
|
"logps/chosen": -361.4550476074219, |
|
"logps/rejected": -491.7296447753906, |
|
"loss": 0.6123, |
|
"rewards/accuracies": 0.824999988079071, |
|
"rewards/chosen": -129.01124572753906, |
|
"rewards/margins": 128.69967651367188, |
|
"rewards/rejected": -257.7109069824219, |
|
"step": 2530 |
|
}, |
|
{ |
|
"epoch": 0.6647474483119602, |
|
"grad_norm": 34.85133335462039, |
|
"learning_rate": 1.5259065836724034e-07, |
|
"logits/chosen": 1.0750457048416138, |
|
"logits/rejected": 1.3905805349349976, |
|
"logps/chosen": -355.76092529296875, |
|
"logps/rejected": -473.81561279296875, |
|
"loss": 0.6037, |
|
"rewards/accuracies": 0.7749999761581421, |
|
"rewards/chosen": -121.11027526855469, |
|
"rewards/margins": 98.64629364013672, |
|
"rewards/rejected": -219.756591796875, |
|
"step": 2540 |
|
}, |
|
{ |
|
"epoch": 0.6673645642501963, |
|
"grad_norm": 18.55982939590747, |
|
"learning_rate": 1.5049083690569454e-07, |
|
"logits/chosen": 1.4674863815307617, |
|
"logits/rejected": 3.273242473602295, |
|
"logps/chosen": -380.1376953125, |
|
"logps/rejected": -481.0174865722656, |
|
"loss": 0.5796, |
|
"rewards/accuracies": 0.800000011920929, |
|
"rewards/chosen": -130.4925079345703, |
|
"rewards/margins": 113.21549224853516, |
|
"rewards/rejected": -243.7080078125, |
|
"step": 2550 |
|
}, |
|
{ |
|
"epoch": 0.6699816801884323, |
|
"grad_norm": 28.676002339732538, |
|
"learning_rate": 1.4839932443063056e-07, |
|
"logits/chosen": 0.23872914910316467, |
|
"logits/rejected": 1.4117581844329834, |
|
"logps/chosen": -429.30914306640625, |
|
"logps/rejected": -487.3106384277344, |
|
"loss": 0.6481, |
|
"rewards/accuracies": 0.7749999761581421, |
|
"rewards/chosen": -131.02191162109375, |
|
"rewards/margins": 91.13362121582031, |
|
"rewards/rejected": -222.15554809570312, |
|
"step": 2560 |
|
}, |
|
{ |
|
"epoch": 0.6725987961266684, |
|
"grad_norm": 54.9929297824748, |
|
"learning_rate": 1.46316295582738e-07, |
|
"logits/chosen": 1.2317416667938232, |
|
"logits/rejected": 2.6543679237365723, |
|
"logps/chosen": -458.9169921875, |
|
"logps/rejected": -479.111083984375, |
|
"loss": 0.6409, |
|
"rewards/accuracies": 0.7250000238418579, |
|
"rewards/chosen": -157.216796875, |
|
"rewards/margins": 83.09622955322266, |
|
"rewards/rejected": -240.3130340576172, |
|
"step": 2570 |
|
}, |
|
{ |
|
"epoch": 0.6752159120649045, |
|
"grad_norm": 23.433586682506476, |
|
"learning_rate": 1.4424192429432655e-07, |
|
"logits/chosen": 0.5045792460441589, |
|
"logits/rejected": 2.09639310836792, |
|
"logps/chosen": -445.75299072265625, |
|
"logps/rejected": -484.1240234375, |
|
"loss": 0.6155, |
|
"rewards/accuracies": 0.800000011920929, |
|
"rewards/chosen": -135.1338653564453, |
|
"rewards/margins": 92.08915710449219, |
|
"rewards/rejected": -227.22299194335938, |
|
"step": 2580 |
|
}, |
|
{ |
|
"epoch": 0.6778330280031405, |
|
"grad_norm": 23.583509411900305, |
|
"learning_rate": 1.4217638377480158e-07, |
|
"logits/chosen": 0.10572358220815659, |
|
"logits/rejected": 1.3806489706039429, |
|
"logps/chosen": -400.96270751953125, |
|
"logps/rejected": -460.14117431640625, |
|
"loss": 0.6009, |
|
"rewards/accuracies": 0.75, |
|
"rewards/chosen": -113.3682632446289, |
|
"rewards/margins": 80.95018005371094, |
|
"rewards/rejected": -194.31845092773438, |
|
"step": 2590 |
|
}, |
|
{ |
|
"epoch": 0.6804501439413766, |
|
"grad_norm": 40.40953868443985, |
|
"learning_rate": 1.401198464962021e-07, |
|
"logits/chosen": 0.2016058713197708, |
|
"logits/rejected": 1.130928874015808, |
|
"logps/chosen": -361.8517150878906, |
|
"logps/rejected": -494.3812561035156, |
|
"loss": 0.6294, |
|
"rewards/accuracies": 0.8999999761581421, |
|
"rewards/chosen": -103.8023452758789, |
|
"rewards/margins": 101.51799011230469, |
|
"rewards/rejected": -205.32034301757812, |
|
"step": 2600 |
|
}, |
|
{ |
|
"epoch": 0.6830672598796127, |
|
"grad_norm": 39.94792054480128, |
|
"learning_rate": 1.3807248417879894e-07, |
|
"logits/chosen": 0.6115899085998535, |
|
"logits/rejected": 1.6156005859375, |
|
"logps/chosen": -307.701904296875, |
|
"logps/rejected": -444.3831481933594, |
|
"loss": 0.6089, |
|
"rewards/accuracies": 0.7749999761581421, |
|
"rewards/chosen": -114.9823226928711, |
|
"rewards/margins": 100.7208023071289, |
|
"rewards/rejected": -215.703125, |
|
"step": 2610 |
|
}, |
|
{ |
|
"epoch": 0.6856843758178487, |
|
"grad_norm": 36.696272333131326, |
|
"learning_rate": 1.3603446777675665e-07, |
|
"logits/chosen": -0.2377341240644455, |
|
"logits/rejected": 1.0026599168777466, |
|
"logps/chosen": -350.7626953125, |
|
"logps/rejected": -412.9532775878906, |
|
"loss": 0.6097, |
|
"rewards/accuracies": 0.75, |
|
"rewards/chosen": -105.99015045166016, |
|
"rewards/margins": 73.70530700683594, |
|
"rewards/rejected": -179.6954803466797, |
|
"step": 2620 |
|
}, |
|
{ |
|
"epoch": 0.6883014917560848, |
|
"grad_norm": 39.398684962283816, |
|
"learning_rate": 1.3400596746385814e-07, |
|
"logits/chosen": 0.5835520029067993, |
|
"logits/rejected": 1.5241050720214844, |
|
"logps/chosen": -417.64990234375, |
|
"logps/rejected": -531.836669921875, |
|
"loss": 0.6309, |
|
"rewards/accuracies": 0.75, |
|
"rewards/chosen": -144.4230499267578, |
|
"rewards/margins": 75.8773193359375, |
|
"rewards/rejected": -220.3003692626953, |
|
"step": 2630 |
|
}, |
|
{ |
|
"epoch": 0.6909186076943209, |
|
"grad_norm": 44.431457896844876, |
|
"learning_rate": 1.3198715261929586e-07, |
|
"logits/chosen": 0.36666786670684814, |
|
"logits/rejected": 1.6828514337539673, |
|
"logps/chosen": -456.6180114746094, |
|
"logps/rejected": -464.532958984375, |
|
"loss": 0.636, |
|
"rewards/accuracies": 0.7250000238418579, |
|
"rewards/chosen": -151.42739868164062, |
|
"rewards/margins": 59.905517578125, |
|
"rewards/rejected": -211.3329315185547, |
|
"step": 2640 |
|
}, |
|
{ |
|
"epoch": 0.6935357236325569, |
|
"grad_norm": 25.17979333945541, |
|
"learning_rate": 1.299781918135282e-07, |
|
"logits/chosen": 0.24016058444976807, |
|
"logits/rejected": 1.928510069847107, |
|
"logps/chosen": -420.1914978027344, |
|
"logps/rejected": -453.20892333984375, |
|
"loss": 0.6323, |
|
"rewards/accuracies": 0.824999988079071, |
|
"rewards/chosen": -144.0685577392578, |
|
"rewards/margins": 97.08206176757812, |
|
"rewards/rejected": -241.15060424804688, |
|
"step": 2650 |
|
}, |
|
{ |
|
"epoch": 0.696152839570793, |
|
"grad_norm": 31.657621753900987, |
|
"learning_rate": 1.279792527942045e-07, |
|
"logits/chosen": -0.0180420633405447, |
|
"logits/rejected": 2.3416850566864014, |
|
"logps/chosen": -406.87652587890625, |
|
"logps/rejected": -449.5647888183594, |
|
"loss": 0.6203, |
|
"rewards/accuracies": 0.7749999761581421, |
|
"rewards/chosen": -133.4667510986328, |
|
"rewards/margins": 90.72755432128906, |
|
"rewards/rejected": -224.19430541992188, |
|
"step": 2660 |
|
}, |
|
{ |
|
"epoch": 0.6987699555090291, |
|
"grad_norm": 49.26258187961619, |
|
"learning_rate": 1.259905024721576e-07, |
|
"logits/chosen": 0.3727002739906311, |
|
"logits/rejected": 2.0285167694091797, |
|
"logps/chosen": -398.2438659667969, |
|
"logps/rejected": -459.0472106933594, |
|
"loss": 0.6157, |
|
"rewards/accuracies": 0.800000011920929, |
|
"rewards/chosen": -122.77784729003906, |
|
"rewards/margins": 113.05708312988281, |
|
"rewards/rejected": -235.83493041992188, |
|
"step": 2670 |
|
}, |
|
{ |
|
"epoch": 0.7013870714472651, |
|
"grad_norm": 83.37635034432546, |
|
"learning_rate": 1.2401210690746703e-07, |
|
"logits/chosen": 0.1346236914396286, |
|
"logits/rejected": 1.083974838256836, |
|
"logps/chosen": -421.3680725097656, |
|
"logps/rejected": -470.74627685546875, |
|
"loss": 0.6079, |
|
"rewards/accuracies": 0.75, |
|
"rewards/chosen": -139.5249481201172, |
|
"rewards/margins": 49.866912841796875, |
|
"rewards/rejected": -189.39187622070312, |
|
"step": 2680 |
|
}, |
|
{ |
|
"epoch": 0.7040041873855012, |
|
"grad_norm": 37.05947965910841, |
|
"learning_rate": 1.2204423129559305e-07, |
|
"logits/chosen": 0.9880054593086243, |
|
"logits/rejected": 2.4434711933135986, |
|
"logps/chosen": -341.87005615234375, |
|
"logps/rejected": -409.3817443847656, |
|
"loss": 0.635, |
|
"rewards/accuracies": 0.8500000238418579, |
|
"rewards/chosen": -123.16813659667969, |
|
"rewards/margins": 77.82776641845703, |
|
"rewards/rejected": -200.9958953857422, |
|
"step": 2690 |
|
}, |
|
{ |
|
"epoch": 0.7066213033237373, |
|
"grad_norm": 41.84013190369576, |
|
"learning_rate": 1.2008703995358299e-07, |
|
"logits/chosen": 2.119760274887085, |
|
"logits/rejected": 2.1831414699554443, |
|
"logps/chosen": -363.73211669921875, |
|
"logps/rejected": -506.10137939453125, |
|
"loss": 0.6087, |
|
"rewards/accuracies": 0.7749999761581421, |
|
"rewards/chosen": -136.8468475341797, |
|
"rewards/margins": 97.38895416259766, |
|
"rewards/rejected": -234.23580932617188, |
|
"step": 2700 |
|
}, |
|
{ |
|
"epoch": 0.7092384192619733, |
|
"grad_norm": 31.86581150130641, |
|
"learning_rate": 1.1814069630635068e-07, |
|
"logits/chosen": 0.14392466843128204, |
|
"logits/rejected": 1.7527008056640625, |
|
"logps/chosen": -454.44549560546875, |
|
"logps/rejected": -468.8114318847656, |
|
"loss": 0.6283, |
|
"rewards/accuracies": 0.800000011920929, |
|
"rewards/chosen": -157.68173217773438, |
|
"rewards/margins": 61.41668701171875, |
|
"rewards/rejected": -219.09841918945312, |
|
"step": 2710 |
|
}, |
|
{ |
|
"epoch": 0.7118555352002094, |
|
"grad_norm": 18.081063280078112, |
|
"learning_rate": 1.1620536287303051e-07, |
|
"logits/chosen": 1.341851830482483, |
|
"logits/rejected": 2.280597448348999, |
|
"logps/chosen": -310.7570495605469, |
|
"logps/rejected": -436.3260192871094, |
|
"loss": 0.6282, |
|
"rewards/accuracies": 0.675000011920929, |
|
"rewards/chosen": -132.96395874023438, |
|
"rewards/margins": 78.2131118774414, |
|
"rewards/rejected": -211.1770782470703, |
|
"step": 2720 |
|
}, |
|
{ |
|
"epoch": 0.7144726511384454, |
|
"grad_norm": 39.45453883471653, |
|
"learning_rate": 1.1428120125340716e-07, |
|
"logits/chosen": 0.7364514470100403, |
|
"logits/rejected": 2.3780157566070557, |
|
"logps/chosen": -415.115966796875, |
|
"logps/rejected": -506.86932373046875, |
|
"loss": 0.6397, |
|
"rewards/accuracies": 0.824999988079071, |
|
"rewards/chosen": -133.8503875732422, |
|
"rewards/margins": 97.6797866821289, |
|
"rewards/rejected": -231.5301971435547, |
|
"step": 2730 |
|
}, |
|
{ |
|
"epoch": 0.7170897670766815, |
|
"grad_norm": 28.535318320593547, |
|
"learning_rate": 1.123683721144223e-07, |
|
"logits/chosen": 0.23405185341835022, |
|
"logits/rejected": 1.1598809957504272, |
|
"logps/chosen": -408.23394775390625, |
|
"logps/rejected": -488.3985290527344, |
|
"loss": 0.6089, |
|
"rewards/accuracies": 0.7749999761581421, |
|
"rewards/chosen": -129.1389923095703, |
|
"rewards/margins": 65.66856384277344, |
|
"rewards/rejected": -194.80755615234375, |
|
"step": 2740 |
|
}, |
|
{ |
|
"epoch": 0.7197068830149176, |
|
"grad_norm": 57.03110684643374, |
|
"learning_rate": 1.1046703517675845e-07, |
|
"logits/chosen": 0.005275917239487171, |
|
"logits/rejected": 1.8837101459503174, |
|
"logps/chosen": -454.74847412109375, |
|
"logps/rejected": -430.2967834472656, |
|
"loss": 0.6373, |
|
"rewards/accuracies": 0.824999988079071, |
|
"rewards/chosen": -123.52076721191406, |
|
"rewards/margins": 77.50233459472656, |
|
"rewards/rejected": -201.02310180664062, |
|
"step": 2750 |
|
}, |
|
{ |
|
"epoch": 0.7223239989531536, |
|
"grad_norm": 16.31469315081151, |
|
"learning_rate": 1.085773492015028e-07, |
|
"logits/chosen": -0.029287749901413918, |
|
"logits/rejected": 0.9784728288650513, |
|
"logps/chosen": -424.54364013671875, |
|
"logps/rejected": -489.2088928222656, |
|
"loss": 0.6089, |
|
"rewards/accuracies": 0.699999988079071, |
|
"rewards/chosen": -115.7852783203125, |
|
"rewards/margins": 62.044944763183594, |
|
"rewards/rejected": -177.83023071289062, |
|
"step": 2760 |
|
}, |
|
{ |
|
"epoch": 0.7249411148913897, |
|
"grad_norm": 27.954300283078403, |
|
"learning_rate": 1.0669947197689033e-07, |
|
"logits/chosen": 0.25381985306739807, |
|
"logits/rejected": 1.3894766569137573, |
|
"logps/chosen": -375.91339111328125, |
|
"logps/rejected": -428.8695373535156, |
|
"loss": 0.597, |
|
"rewards/accuracies": 0.7749999761581421, |
|
"rewards/chosen": -127.7160415649414, |
|
"rewards/margins": 75.71189880371094, |
|
"rewards/rejected": -203.42791748046875, |
|
"step": 2770 |
|
}, |
|
{ |
|
"epoch": 0.7275582308296258, |
|
"grad_norm": 32.24312345073874, |
|
"learning_rate": 1.048335603051291e-07, |
|
"logits/chosen": 0.8968909382820129, |
|
"logits/rejected": 1.4077928066253662, |
|
"logps/chosen": -442.5194396972656, |
|
"logps/rejected": -486.0824279785156, |
|
"loss": 0.6251, |
|
"rewards/accuracies": 0.800000011920929, |
|
"rewards/chosen": -123.14749908447266, |
|
"rewards/margins": 78.89585876464844, |
|
"rewards/rejected": -202.04335021972656, |
|
"step": 2780 |
|
}, |
|
{ |
|
"epoch": 0.7301753467678618, |
|
"grad_norm": 36.23128514010418, |
|
"learning_rate": 1.0297976998930663e-07, |
|
"logits/chosen": 0.7249239683151245, |
|
"logits/rejected": 2.3154051303863525, |
|
"logps/chosen": -425.0227966308594, |
|
"logps/rejected": -466.5647888183594, |
|
"loss": 0.6061, |
|
"rewards/accuracies": 0.75, |
|
"rewards/chosen": -108.95176696777344, |
|
"rewards/margins": 75.4349365234375, |
|
"rewards/rejected": -184.38671875, |
|
"step": 2790 |
|
}, |
|
{ |
|
"epoch": 0.7327924627060979, |
|
"grad_norm": 35.159964513238755, |
|
"learning_rate": 1.0113825582038077e-07, |
|
"logits/chosen": 0.4708864688873291, |
|
"logits/rejected": 3.2727150917053223, |
|
"logps/chosen": -410.4869689941406, |
|
"logps/rejected": -432.30523681640625, |
|
"loss": 0.6079, |
|
"rewards/accuracies": 0.824999988079071, |
|
"rewards/chosen": -114.40809631347656, |
|
"rewards/margins": 107.46346282958984, |
|
"rewards/rejected": -221.87158203125, |
|
"step": 2800 |
|
}, |
|
{ |
|
"epoch": 0.735409578644334, |
|
"grad_norm": 71.25724626710895, |
|
"learning_rate": 9.930917156425475e-08, |
|
"logits/chosen": 0.4397871494293213, |
|
"logits/rejected": 2.1435513496398926, |
|
"logps/chosen": -392.5745849609375, |
|
"logps/rejected": -425.1902770996094, |
|
"loss": 0.6393, |
|
"rewards/accuracies": 0.75, |
|
"rewards/chosen": -96.77259826660156, |
|
"rewards/margins": 102.71319580078125, |
|
"rewards/rejected": -199.4857940673828, |
|
"step": 2810 |
|
}, |
|
{ |
|
"epoch": 0.73802669458257, |
|
"grad_norm": 44.71807626624307, |
|
"learning_rate": 9.749266994893754e-08, |
|
"logits/chosen": 0.3410795032978058, |
|
"logits/rejected": 1.3199317455291748, |
|
"logps/chosen": -358.898681640625, |
|
"logps/rejected": -448.3935546875, |
|
"loss": 0.616, |
|
"rewards/accuracies": 0.675000011920929, |
|
"rewards/chosen": -127.18278503417969, |
|
"rewards/margins": 72.87733459472656, |
|
"rewards/rejected": -200.0601043701172, |
|
"step": 2820 |
|
}, |
|
{ |
|
"epoch": 0.7406438105208061, |
|
"grad_norm": 30.761613537091247, |
|
"learning_rate": 9.568890265179128e-08, |
|
"logits/chosen": 0.4254278540611267, |
|
"logits/rejected": 1.124775767326355, |
|
"logps/chosen": -352.58270263671875, |
|
"logps/rejected": -433.5701599121094, |
|
"loss": 0.6495, |
|
"rewards/accuracies": 0.7250000238418579, |
|
"rewards/chosen": -125.1341781616211, |
|
"rewards/margins": 84.12373352050781, |
|
"rewards/rejected": -209.2578887939453, |
|
"step": 2830 |
|
}, |
|
{ |
|
"epoch": 0.7432609264590422, |
|
"grad_norm": 49.88197491305349, |
|
"learning_rate": 9.389802028686616e-08, |
|
"logits/chosen": -0.025458145886659622, |
|
"logits/rejected": 1.2856556177139282, |
|
"logps/chosen": -383.7842712402344, |
|
"logps/rejected": -466.41998291015625, |
|
"loss": 0.6271, |
|
"rewards/accuracies": 0.7250000238418579, |
|
"rewards/chosen": -114.00410461425781, |
|
"rewards/margins": 86.70926666259766, |
|
"rewards/rejected": -200.71334838867188, |
|
"step": 2840 |
|
}, |
|
{ |
|
"epoch": 0.7458780423972782, |
|
"grad_norm": 27.585109058465484, |
|
"learning_rate": 9.212017239232426e-08, |
|
"logits/chosen": 0.7934707403182983, |
|
"logits/rejected": 1.2405365705490112, |
|
"logps/chosen": -366.0028991699219, |
|
"logps/rejected": -449.9388732910156, |
|
"loss": 0.6256, |
|
"rewards/accuracies": 0.675000011920929, |
|
"rewards/chosen": -117.50825500488281, |
|
"rewards/margins": 56.368919372558594, |
|
"rewards/rejected": -173.87716674804688, |
|
"step": 2850 |
|
}, |
|
{ |
|
"epoch": 0.7484951583355143, |
|
"grad_norm": 19.174565551603006, |
|
"learning_rate": 9.035550741795328e-08, |
|
"logits/chosen": 0.19867125153541565, |
|
"logits/rejected": 1.6759397983551025, |
|
"logps/chosen": -413.1884765625, |
|
"logps/rejected": -487.4052734375, |
|
"loss": 0.6097, |
|
"rewards/accuracies": 0.824999988079071, |
|
"rewards/chosen": -103.04798889160156, |
|
"rewards/margins": 111.28741455078125, |
|
"rewards/rejected": -214.3354034423828, |
|
"step": 2860 |
|
}, |
|
{ |
|
"epoch": 0.7511122742737504, |
|
"grad_norm": 16.501011315290544, |
|
"learning_rate": 8.860417271277065e-08, |
|
"logits/chosen": 0.44556722044944763, |
|
"logits/rejected": 1.7269704341888428, |
|
"logps/chosen": -358.22796630859375, |
|
"logps/rejected": -463.13092041015625, |
|
"loss": 0.6076, |
|
"rewards/accuracies": 0.800000011920929, |
|
"rewards/chosen": -114.99739837646484, |
|
"rewards/margins": 107.62504577636719, |
|
"rewards/rejected": -222.62246704101562, |
|
"step": 2870 |
|
}, |
|
{ |
|
"epoch": 0.7537293902119864, |
|
"grad_norm": 16.630600643438857, |
|
"learning_rate": 8.686631451272029e-08, |
|
"logits/chosen": 0.36867132782936096, |
|
"logits/rejected": 1.5242793560028076, |
|
"logps/chosen": -487.36016845703125, |
|
"logps/rejected": -469.58074951171875, |
|
"loss": 0.6173, |
|
"rewards/accuracies": 0.75, |
|
"rewards/chosen": -132.63406372070312, |
|
"rewards/margins": 62.797386169433594, |
|
"rewards/rejected": -195.43142700195312, |
|
"step": 2880 |
|
}, |
|
{ |
|
"epoch": 0.7563465061502225, |
|
"grad_norm": 19.17394858355595, |
|
"learning_rate": 8.514207792846168e-08, |
|
"logits/chosen": 0.8502219319343567, |
|
"logits/rejected": 1.6160399913787842, |
|
"logps/chosen": -342.61981201171875, |
|
"logps/rejected": -443.2942810058594, |
|
"loss": 0.602, |
|
"rewards/accuracies": 0.7749999761581421, |
|
"rewards/chosen": -99.26615905761719, |
|
"rewards/margins": 106.50407409667969, |
|
"rewards/rejected": -205.7702178955078, |
|
"step": 2890 |
|
}, |
|
{ |
|
"epoch": 0.7589636220884585, |
|
"grad_norm": 26.890168448182035, |
|
"learning_rate": 8.343160693325355e-08, |
|
"logits/chosen": 1.1203569173812866, |
|
"logits/rejected": 2.6382992267608643, |
|
"logps/chosen": -440.35699462890625, |
|
"logps/rejected": -502.62176513671875, |
|
"loss": 0.6176, |
|
"rewards/accuracies": 0.75, |
|
"rewards/chosen": -144.5258026123047, |
|
"rewards/margins": 112.56309509277344, |
|
"rewards/rejected": -257.0888977050781, |
|
"step": 2900 |
|
}, |
|
{ |
|
"epoch": 0.7615807380266946, |
|
"grad_norm": 43.572658787963654, |
|
"learning_rate": 8.173504435093173e-08, |
|
"logits/chosen": 0.8563163876533508, |
|
"logits/rejected": 3.0953686237335205, |
|
"logps/chosen": -408.85089111328125, |
|
"logps/rejected": -474.3164978027344, |
|
"loss": 0.6352, |
|
"rewards/accuracies": 0.75, |
|
"rewards/chosen": -127.74039459228516, |
|
"rewards/margins": 110.32560729980469, |
|
"rewards/rejected": -238.0659942626953, |
|
"step": 2910 |
|
}, |
|
{ |
|
"epoch": 0.7641978539649307, |
|
"grad_norm": 25.30166750340952, |
|
"learning_rate": 8.005253184398359e-08, |
|
"logits/chosen": 1.089434266090393, |
|
"logits/rejected": 2.4163076877593994, |
|
"logps/chosen": -365.9088439941406, |
|
"logps/rejected": -480.734130859375, |
|
"loss": 0.6251, |
|
"rewards/accuracies": 0.75, |
|
"rewards/chosen": -120.29222106933594, |
|
"rewards/margins": 89.46209716796875, |
|
"rewards/rejected": -209.7543182373047, |
|
"step": 2920 |
|
}, |
|
{ |
|
"epoch": 0.7668149699031667, |
|
"grad_norm": 34.78404791298794, |
|
"learning_rate": 7.838420990171926e-08, |
|
"logits/chosen": 0.16231641173362732, |
|
"logits/rejected": 1.6783320903778076, |
|
"logps/chosen": -398.6322021484375, |
|
"logps/rejected": -470.6861267089844, |
|
"loss": 0.6007, |
|
"rewards/accuracies": 0.800000011920929, |
|
"rewards/chosen": -111.038330078125, |
|
"rewards/margins": 101.57747650146484, |
|
"rewards/rejected": -212.6157989501953, |
|
"step": 2930 |
|
}, |
|
{ |
|
"epoch": 0.7694320858414028, |
|
"grad_norm": 14.809008382614701, |
|
"learning_rate": 7.673021782854083e-08, |
|
"logits/chosen": -0.15028102695941925, |
|
"logits/rejected": 1.6816718578338623, |
|
"logps/chosen": -429.3408203125, |
|
"logps/rejected": -436.5340270996094, |
|
"loss": 0.5864, |
|
"rewards/accuracies": 0.699999988079071, |
|
"rewards/chosen": -116.80708312988281, |
|
"rewards/margins": 84.45670318603516, |
|
"rewards/rejected": -201.26376342773438, |
|
"step": 2940 |
|
}, |
|
{ |
|
"epoch": 0.7720492017796389, |
|
"grad_norm": 21.97790850293821, |
|
"learning_rate": 7.509069373231039e-08, |
|
"logits/chosen": 0.1503334790468216, |
|
"logits/rejected": 1.3877406120300293, |
|
"logps/chosen": -426.85418701171875, |
|
"logps/rejected": -458.5160217285156, |
|
"loss": 0.612, |
|
"rewards/accuracies": 0.7749999761581421, |
|
"rewards/chosen": -109.48735046386719, |
|
"rewards/margins": 87.34893798828125, |
|
"rewards/rejected": -196.83627319335938, |
|
"step": 2950 |
|
}, |
|
{ |
|
"epoch": 0.7746663177178749, |
|
"grad_norm": 58.57085668506599, |
|
"learning_rate": 7.346577451281821e-08, |
|
"logits/chosen": 0.13314999639987946, |
|
"logits/rejected": 1.4621391296386719, |
|
"logps/chosen": -433.54974365234375, |
|
"logps/rejected": -509.16864013671875, |
|
"loss": 0.6009, |
|
"rewards/accuracies": 0.824999988079071, |
|
"rewards/chosen": -123.75052642822266, |
|
"rewards/margins": 90.73429870605469, |
|
"rewards/rejected": -214.4848175048828, |
|
"step": 2960 |
|
}, |
|
{ |
|
"epoch": 0.777283433656111, |
|
"grad_norm": 23.872007670066296, |
|
"learning_rate": 7.185559585035136e-08, |
|
"logits/chosen": -0.1507561057806015, |
|
"logits/rejected": 1.7900638580322266, |
|
"logps/chosen": -398.67156982421875, |
|
"logps/rejected": -442.6659240722656, |
|
"loss": 0.6187, |
|
"rewards/accuracies": 0.7749999761581421, |
|
"rewards/chosen": -98.9404525756836, |
|
"rewards/margins": 82.65269470214844, |
|
"rewards/rejected": -181.5931396484375, |
|
"step": 2970 |
|
}, |
|
{ |
|
"epoch": 0.7799005495943471, |
|
"grad_norm": 44.43635867539923, |
|
"learning_rate": 7.026029219436502e-08, |
|
"logits/chosen": -0.27151232957839966, |
|
"logits/rejected": 1.824271559715271, |
|
"logps/chosen": -402.84942626953125, |
|
"logps/rejected": -433.583984375, |
|
"loss": 0.5892, |
|
"rewards/accuracies": 0.800000011920929, |
|
"rewards/chosen": -112.6480941772461, |
|
"rewards/margins": 101.14834594726562, |
|
"rewards/rejected": -213.7964324951172, |
|
"step": 2980 |
|
}, |
|
{ |
|
"epoch": 0.7825176655325831, |
|
"grad_norm": 39.07899805263987, |
|
"learning_rate": 6.867999675225522e-08, |
|
"logits/chosen": 0.15538421273231506, |
|
"logits/rejected": 0.8754119873046875, |
|
"logps/chosen": -396.12249755859375, |
|
"logps/rejected": -429.5633239746094, |
|
"loss": 0.6509, |
|
"rewards/accuracies": 0.6499999761581421, |
|
"rewards/chosen": -110.3372573852539, |
|
"rewards/margins": 73.39195251464844, |
|
"rewards/rejected": -183.7292022705078, |
|
"step": 2990 |
|
}, |
|
{ |
|
"epoch": 0.7851347814708192, |
|
"grad_norm": 25.62485658793096, |
|
"learning_rate": 6.711484147823662e-08, |
|
"logits/chosen": 0.48910078406333923, |
|
"logits/rejected": 1.69696843624115, |
|
"logps/chosen": -332.47314453125, |
|
"logps/rejected": -342.8687744140625, |
|
"loss": 0.6306, |
|
"rewards/accuracies": 0.75, |
|
"rewards/chosen": -110.85118103027344, |
|
"rewards/margins": 58.67308807373047, |
|
"rewards/rejected": -169.52426147460938, |
|
"step": 3000 |
|
}, |
|
{ |
|
"epoch": 0.7877518974090553, |
|
"grad_norm": 37.83197091357618, |
|
"learning_rate": 6.556495706232412e-08, |
|
"logits/chosen": 0.1738726645708084, |
|
"logits/rejected": 0.9801591038703918, |
|
"logps/chosen": -419.72479248046875, |
|
"logps/rejected": -499.14410400390625, |
|
"loss": 0.6207, |
|
"rewards/accuracies": 0.8500000238418579, |
|
"rewards/chosen": -119.627685546875, |
|
"rewards/margins": 98.64978790283203, |
|
"rewards/rejected": -218.2774658203125, |
|
"step": 3010 |
|
}, |
|
{ |
|
"epoch": 0.7903690133472913, |
|
"grad_norm": 29.292991665537237, |
|
"learning_rate": 6.403047291942057e-08, |
|
"logits/chosen": 0.2022714614868164, |
|
"logits/rejected": 0.7730263471603394, |
|
"logps/chosen": -394.0956726074219, |
|
"logps/rejected": -425.01104736328125, |
|
"loss": 0.643, |
|
"rewards/accuracies": 0.675000011920929, |
|
"rewards/chosen": -106.90836334228516, |
|
"rewards/margins": 35.84281921386719, |
|
"rewards/rejected": -142.75119018554688, |
|
"step": 3020 |
|
}, |
|
{ |
|
"epoch": 0.7929861292855274, |
|
"grad_norm": 28.64177935728824, |
|
"learning_rate": 6.251151717851021e-08, |
|
"logits/chosen": -0.10760748386383057, |
|
"logits/rejected": 1.3448083400726318, |
|
"logps/chosen": -371.3802185058594, |
|
"logps/rejected": -411.80682373046875, |
|
"loss": 0.6036, |
|
"rewards/accuracies": 0.824999988079071, |
|
"rewards/chosen": -106.61136627197266, |
|
"rewards/margins": 80.564208984375, |
|
"rewards/rejected": -187.17555236816406, |
|
"step": 3030 |
|
}, |
|
{ |
|
"epoch": 0.7956032452237635, |
|
"grad_norm": 46.31822111959148, |
|
"learning_rate": 6.100821667196041e-08, |
|
"logits/chosen": 0.24122682213783264, |
|
"logits/rejected": 0.4868651330471039, |
|
"logps/chosen": -392.434814453125, |
|
"logps/rejected": -435.9532165527344, |
|
"loss": 0.6202, |
|
"rewards/accuracies": 0.7250000238418579, |
|
"rewards/chosen": -101.78141021728516, |
|
"rewards/margins": 51.99553680419922, |
|
"rewards/rejected": -153.77696228027344, |
|
"step": 3040 |
|
}, |
|
{ |
|
"epoch": 0.7982203611619995, |
|
"grad_norm": 20.18830972334164, |
|
"learning_rate": 5.952069692493061e-08, |
|
"logits/chosen": 0.18420252203941345, |
|
"logits/rejected": 1.016076683998108, |
|
"logps/chosen": -332.6474304199219, |
|
"logps/rejected": -428.02618408203125, |
|
"loss": 0.6216, |
|
"rewards/accuracies": 0.800000011920929, |
|
"rewards/chosen": -83.14781188964844, |
|
"rewards/margins": 99.45719909667969, |
|
"rewards/rejected": -182.60501098632812, |
|
"step": 3050 |
|
}, |
|
{ |
|
"epoch": 0.8008374771002356, |
|
"grad_norm": 32.9312271398435, |
|
"learning_rate": 5.8049082144891794e-08, |
|
"logits/chosen": -0.34789204597473145, |
|
"logits/rejected": 1.1649951934814453, |
|
"logps/chosen": -350.571044921875, |
|
"logps/rejected": -384.31982421875, |
|
"loss": 0.5948, |
|
"rewards/accuracies": 0.7749999761581421, |
|
"rewards/chosen": -100.79808044433594, |
|
"rewards/margins": 72.1947250366211, |
|
"rewards/rejected": -172.9927978515625, |
|
"step": 3060 |
|
}, |
|
{ |
|
"epoch": 0.8034545930384716, |
|
"grad_norm": 24.538460956530265, |
|
"learning_rate": 5.659349521125459e-08, |
|
"logits/chosen": 0.20586685836315155, |
|
"logits/rejected": 1.3391412496566772, |
|
"logps/chosen": -398.23126220703125, |
|
"logps/rejected": -544.0811767578125, |
|
"loss": 0.6167, |
|
"rewards/accuracies": 0.75, |
|
"rewards/chosen": -118.44612884521484, |
|
"rewards/margins": 88.53105163574219, |
|
"rewards/rejected": -206.97720336914062, |
|
"step": 3070 |
|
}, |
|
{ |
|
"epoch": 0.8060717089767077, |
|
"grad_norm": 18.607731049230058, |
|
"learning_rate": 5.5154057665109e-08, |
|
"logits/chosen": -0.3509600758552551, |
|
"logits/rejected": 0.9203709363937378, |
|
"logps/chosen": -419.15960693359375, |
|
"logps/rejected": -474.97119140625, |
|
"loss": 0.6016, |
|
"rewards/accuracies": 0.7749999761581421, |
|
"rewards/chosen": -99.57820892333984, |
|
"rewards/margins": 82.64437103271484, |
|
"rewards/rejected": -182.22256469726562, |
|
"step": 3080 |
|
}, |
|
{ |
|
"epoch": 0.8086888249149438, |
|
"grad_norm": 34.143748417302646, |
|
"learning_rate": 5.3730889699075853e-08, |
|
"logits/chosen": -0.47578373551368713, |
|
"logits/rejected": 0.4792146682739258, |
|
"logps/chosen": -436.461669921875, |
|
"logps/rejected": -462.1795349121094, |
|
"loss": 0.6033, |
|
"rewards/accuracies": 0.7749999761581421, |
|
"rewards/chosen": -106.6905517578125, |
|
"rewards/margins": 73.09249114990234, |
|
"rewards/rejected": -179.78305053710938, |
|
"step": 3090 |
|
}, |
|
{ |
|
"epoch": 0.8113059408531798, |
|
"grad_norm": 38.11095562004462, |
|
"learning_rate": 5.2324110147270893e-08, |
|
"logits/chosen": 0.583615243434906, |
|
"logits/rejected": 1.4006626605987549, |
|
"logps/chosen": -347.5642395019531, |
|
"logps/rejected": -463.29962158203125, |
|
"loss": 0.6141, |
|
"rewards/accuracies": 0.7250000238418579, |
|
"rewards/chosen": -113.69773864746094, |
|
"rewards/margins": 85.3753662109375, |
|
"rewards/rejected": -199.0731201171875, |
|
"step": 3100 |
|
}, |
|
{ |
|
"epoch": 0.8139230567914159, |
|
"grad_norm": 44.946656962405264, |
|
"learning_rate": 5.0933836475381795e-08, |
|
"logits/chosen": -0.39866572618484497, |
|
"logits/rejected": 2.506138801574707, |
|
"logps/chosen": -429.55694580078125, |
|
"logps/rejected": -445.2572326660156, |
|
"loss": 0.6195, |
|
"rewards/accuracies": 0.800000011920929, |
|
"rewards/chosen": -100.6939926147461, |
|
"rewards/margins": 108.70045471191406, |
|
"rewards/rejected": -209.39443969726562, |
|
"step": 3110 |
|
}, |
|
{ |
|
"epoch": 0.816540172729652, |
|
"grad_norm": 28.151760961027662, |
|
"learning_rate": 4.956018477086005e-08, |
|
"logits/chosen": -0.3393104076385498, |
|
"logits/rejected": 0.8863036036491394, |
|
"logps/chosen": -425.35443115234375, |
|
"logps/rejected": -503.589111328125, |
|
"loss": 0.6068, |
|
"rewards/accuracies": 0.7749999761581421, |
|
"rewards/chosen": -127.26164245605469, |
|
"rewards/margins": 89.95985412597656, |
|
"rewards/rejected": -217.2214813232422, |
|
"step": 3120 |
|
}, |
|
{ |
|
"epoch": 0.819157288667888, |
|
"grad_norm": 17.270672002448382, |
|
"learning_rate": 4.820326973322763e-08, |
|
"logits/chosen": 0.06319572031497955, |
|
"logits/rejected": 1.5195165872573853, |
|
"logps/chosen": -352.53643798828125, |
|
"logps/rejected": -366.3856506347656, |
|
"loss": 0.6126, |
|
"rewards/accuracies": 0.7250000238418579, |
|
"rewards/chosen": -124.1020278930664, |
|
"rewards/margins": 65.88347625732422, |
|
"rewards/rejected": -189.98550415039062, |
|
"step": 3130 |
|
}, |
|
{ |
|
"epoch": 0.821774404606124, |
|
"grad_norm": 22.56622768926043, |
|
"learning_rate": 4.686320466449981e-08, |
|
"logits/chosen": -0.03642309829592705, |
|
"logits/rejected": 0.8057546615600586, |
|
"logps/chosen": -409.79571533203125, |
|
"logps/rejected": -441.3692321777344, |
|
"loss": 0.6214, |
|
"rewards/accuracies": 0.699999988079071, |
|
"rewards/chosen": -135.99322509765625, |
|
"rewards/margins": 58.7821159362793, |
|
"rewards/rejected": -194.7753448486328, |
|
"step": 3140 |
|
}, |
|
{ |
|
"epoch": 0.8243915205443602, |
|
"grad_norm": 22.244625038969573, |
|
"learning_rate": 4.554010145972417e-08, |
|
"logits/chosen": -0.41408711671829224, |
|
"logits/rejected": 0.9231753349304199, |
|
"logps/chosen": -449.20281982421875, |
|
"logps/rejected": -483.89593505859375, |
|
"loss": 0.6128, |
|
"rewards/accuracies": 0.7250000238418579, |
|
"rewards/chosen": -104.51177978515625, |
|
"rewards/margins": 86.42676544189453, |
|
"rewards/rejected": -190.9385223388672, |
|
"step": 3150 |
|
}, |
|
{ |
|
"epoch": 0.8270086364825961, |
|
"grad_norm": 30.60778101586563, |
|
"learning_rate": 4.423407059763745e-08, |
|
"logits/chosen": -0.17581668496131897, |
|
"logits/rejected": 0.6096712350845337, |
|
"logps/chosen": -415.3594665527344, |
|
"logps/rejected": -438.58526611328125, |
|
"loss": 0.6221, |
|
"rewards/accuracies": 0.7250000238418579, |
|
"rewards/chosen": -95.22805786132812, |
|
"rewards/margins": 79.04048156738281, |
|
"rewards/rejected": -174.26852416992188, |
|
"step": 3160 |
|
}, |
|
{ |
|
"epoch": 0.8296257524208323, |
|
"grad_norm": 29.18479634749517, |
|
"learning_rate": 4.294522113144078e-08, |
|
"logits/chosen": 0.7951468229293823, |
|
"logits/rejected": 1.7467435598373413, |
|
"logps/chosen": -419.86871337890625, |
|
"logps/rejected": -456.6111755371094, |
|
"loss": 0.6103, |
|
"rewards/accuracies": 0.75, |
|
"rewards/chosen": -145.0784149169922, |
|
"rewards/margins": 73.13197326660156, |
|
"rewards/rejected": -218.2103729248047, |
|
"step": 3170 |
|
}, |
|
{ |
|
"epoch": 0.8322428683590684, |
|
"grad_norm": 133.54621745583137, |
|
"learning_rate": 4.1673660679693804e-08, |
|
"logits/chosen": -0.258161723613739, |
|
"logits/rejected": 0.2669413089752197, |
|
"logps/chosen": -401.003173828125, |
|
"logps/rejected": -459.4292907714844, |
|
"loss": 0.6436, |
|
"rewards/accuracies": 0.75, |
|
"rewards/chosen": -109.79402923583984, |
|
"rewards/margins": 53.30994415283203, |
|
"rewards/rejected": -163.10397338867188, |
|
"step": 3180 |
|
}, |
|
{ |
|
"epoch": 0.8348599842973043, |
|
"grad_norm": 22.45159047211893, |
|
"learning_rate": 4.041949541732825e-08, |
|
"logits/chosen": 0.8274353742599487, |
|
"logits/rejected": 1.4107186794281006, |
|
"logps/chosen": -396.6387634277344, |
|
"logps/rejected": -528.96484375, |
|
"loss": 0.6218, |
|
"rewards/accuracies": 0.75, |
|
"rewards/chosen": -139.893310546875, |
|
"rewards/margins": 150.52256774902344, |
|
"rewards/rejected": -290.4158630371094, |
|
"step": 3190 |
|
}, |
|
{ |
|
"epoch": 0.8374771002355405, |
|
"grad_norm": 85.9004333178433, |
|
"learning_rate": 3.9182830066782605e-08, |
|
"logits/chosen": -0.225154310464859, |
|
"logits/rejected": 0.8962351679801941, |
|
"logps/chosen": -411.37213134765625, |
|
"logps/rejected": -457.3607482910156, |
|
"loss": 0.605, |
|
"rewards/accuracies": 0.824999988079071, |
|
"rewards/chosen": -114.58937072753906, |
|
"rewards/margins": 89.79297637939453, |
|
"rewards/rejected": -204.38232421875, |
|
"step": 3200 |
|
}, |
|
{ |
|
"epoch": 0.8400942161737766, |
|
"grad_norm": 54.26136466947174, |
|
"learning_rate": 3.79637678892577e-08, |
|
"logits/chosen": 0.8109161257743835, |
|
"logits/rejected": 1.667267084121704, |
|
"logps/chosen": -436.07281494140625, |
|
"logps/rejected": -462.241943359375, |
|
"loss": 0.6251, |
|
"rewards/accuracies": 0.7250000238418579, |
|
"rewards/chosen": -144.87509155273438, |
|
"rewards/margins": 60.97523880004883, |
|
"rewards/rejected": -205.85031127929688, |
|
"step": 3210 |
|
}, |
|
{ |
|
"epoch": 0.8427113321120125, |
|
"grad_norm": 20.724578281603087, |
|
"learning_rate": 3.6762410676094645e-08, |
|
"logits/chosen": 0.34443768858909607, |
|
"logits/rejected": 1.82961905002594, |
|
"logps/chosen": -458.07513427734375, |
|
"logps/rejected": -504.356201171875, |
|
"loss": 0.5864, |
|
"rewards/accuracies": 0.75, |
|
"rewards/chosen": -123.5264663696289, |
|
"rewards/margins": 89.05165100097656, |
|
"rewards/rejected": -212.578125, |
|
"step": 3220 |
|
}, |
|
{ |
|
"epoch": 0.8453284480502486, |
|
"grad_norm": 40.41533481576333, |
|
"learning_rate": 3.557885874027497e-08, |
|
"logits/chosen": 0.2875576913356781, |
|
"logits/rejected": 1.515847086906433, |
|
"logps/chosen": -420.4130859375, |
|
"logps/rejected": -446.76922607421875, |
|
"loss": 0.6213, |
|
"rewards/accuracies": 0.6499999761581421, |
|
"rewards/chosen": -118.218994140625, |
|
"rewards/margins": 75.82159423828125, |
|
"rewards/rejected": -194.04058837890625, |
|
"step": 3230 |
|
}, |
|
{ |
|
"epoch": 0.8479455639884846, |
|
"grad_norm": 33.34083742420766, |
|
"learning_rate": 3.441321090804469e-08, |
|
"logits/chosen": -0.43460145592689514, |
|
"logits/rejected": 1.3918662071228027, |
|
"logps/chosen": -406.4483642578125, |
|
"logps/rejected": -525.8848876953125, |
|
"loss": 0.5975, |
|
"rewards/accuracies": 0.699999988079071, |
|
"rewards/chosen": -110.67840576171875, |
|
"rewards/margins": 94.2422866821289, |
|
"rewards/rejected": -204.92068481445312, |
|
"step": 3240 |
|
}, |
|
{ |
|
"epoch": 0.8505626799267207, |
|
"grad_norm": 17.25663786210711, |
|
"learning_rate": 3.326556451066234e-08, |
|
"logits/chosen": -0.41477689146995544, |
|
"logits/rejected": 1.7711904048919678, |
|
"logps/chosen": -430.942138671875, |
|
"logps/rejected": -448.9490661621094, |
|
"loss": 0.6206, |
|
"rewards/accuracies": 0.75, |
|
"rewards/chosen": -124.95573425292969, |
|
"rewards/margins": 84.45895385742188, |
|
"rewards/rejected": -209.41470336914062, |
|
"step": 3250 |
|
}, |
|
{ |
|
"epoch": 0.8531797958649568, |
|
"grad_norm": 33.06045690226845, |
|
"learning_rate": 3.2136015376271946e-08, |
|
"logits/chosen": 0.573313295841217, |
|
"logits/rejected": 2.416731595993042, |
|
"logps/chosen": -350.485595703125, |
|
"logps/rejected": -356.3022155761719, |
|
"loss": 0.5999, |
|
"rewards/accuracies": 0.625, |
|
"rewards/chosen": -122.4647216796875, |
|
"rewards/margins": 62.025794982910156, |
|
"rewards/rejected": -184.49050903320312, |
|
"step": 3260 |
|
}, |
|
{ |
|
"epoch": 0.8557969118031928, |
|
"grad_norm": 33.26261425090372, |
|
"learning_rate": 3.102465782190106e-08, |
|
"logits/chosen": 1.0045592784881592, |
|
"logits/rejected": 1.1752761602401733, |
|
"logps/chosen": -355.07769775390625, |
|
"logps/rejected": -607.1895751953125, |
|
"loss": 0.6018, |
|
"rewards/accuracies": 0.675000011920929, |
|
"rewards/chosen": -141.97750854492188, |
|
"rewards/margins": 80.67582702636719, |
|
"rewards/rejected": -222.65335083007812, |
|
"step": 3270 |
|
}, |
|
{ |
|
"epoch": 0.8584140277414289, |
|
"grad_norm": 25.329125812833063, |
|
"learning_rate": 2.993158464558565e-08, |
|
"logits/chosen": 0.6127084493637085, |
|
"logits/rejected": 0.8664273023605347, |
|
"logps/chosen": -393.71392822265625, |
|
"logps/rejected": -506.6620178222656, |
|
"loss": 0.6253, |
|
"rewards/accuracies": 0.75, |
|
"rewards/chosen": -131.31350708007812, |
|
"rewards/margins": 82.22903442382812, |
|
"rewards/rejected": -213.5425262451172, |
|
"step": 3280 |
|
}, |
|
{ |
|
"epoch": 0.861031143679665, |
|
"grad_norm": 18.25047969135475, |
|
"learning_rate": 2.8856887118621358e-08, |
|
"logits/chosen": 0.11902396380901337, |
|
"logits/rejected": 1.4481818675994873, |
|
"logps/chosen": -437.8792419433594, |
|
"logps/rejected": -560.4588012695312, |
|
"loss": 0.5944, |
|
"rewards/accuracies": 0.824999988079071, |
|
"rewards/chosen": -111.86763763427734, |
|
"rewards/margins": 117.12901306152344, |
|
"rewards/rejected": -228.9966583251953, |
|
"step": 3290 |
|
}, |
|
{ |
|
"epoch": 0.863648259617901, |
|
"grad_norm": 26.6063313450085, |
|
"learning_rate": 2.7800654977942482e-08, |
|
"logits/chosen": 0.6087424159049988, |
|
"logits/rejected": 0.7114389538764954, |
|
"logps/chosen": -371.51397705078125, |
|
"logps/rejected": -431.497802734375, |
|
"loss": 0.6332, |
|
"rewards/accuracies": 0.6499999761581421, |
|
"rewards/chosen": -125.68513488769531, |
|
"rewards/margins": 42.292999267578125, |
|
"rewards/rejected": -167.97811889648438, |
|
"step": 3300 |
|
}, |
|
{ |
|
"epoch": 0.8662653755561371, |
|
"grad_norm": 38.897936775137644, |
|
"learning_rate": 2.676297641862879e-08, |
|
"logits/chosen": -0.13244356215000153, |
|
"logits/rejected": 0.7202237844467163, |
|
"logps/chosen": -447.2171325683594, |
|
"logps/rejected": -462.3369140625, |
|
"loss": 0.6265, |
|
"rewards/accuracies": 0.7250000238418579, |
|
"rewards/chosen": -115.806396484375, |
|
"rewards/margins": 49.385658264160156, |
|
"rewards/rejected": -165.19203186035156, |
|
"step": 3310 |
|
}, |
|
{ |
|
"epoch": 0.8688824914943732, |
|
"grad_norm": 22.24134018200084, |
|
"learning_rate": 2.5743938086541352e-08, |
|
"logits/chosen": 0.31034788489341736, |
|
"logits/rejected": 1.7712135314941406, |
|
"logps/chosen": -361.48370361328125, |
|
"logps/rejected": -408.6527404785156, |
|
"loss": 0.6042, |
|
"rewards/accuracies": 0.875, |
|
"rewards/chosen": -123.82491302490234, |
|
"rewards/margins": 101.67516326904297, |
|
"rewards/rejected": -225.50009155273438, |
|
"step": 3320 |
|
}, |
|
{ |
|
"epoch": 0.8714996074326092, |
|
"grad_norm": 76.72292034254704, |
|
"learning_rate": 2.474362507108757e-08, |
|
"logits/chosen": -0.2383597195148468, |
|
"logits/rejected": 1.3851786851882935, |
|
"logps/chosen": -359.73858642578125, |
|
"logps/rejected": -491.29266357421875, |
|
"loss": 0.6068, |
|
"rewards/accuracies": 0.824999988079071, |
|
"rewards/chosen": -94.1988525390625, |
|
"rewards/margins": 134.3372344970703, |
|
"rewards/rejected": -228.5360870361328, |
|
"step": 3330 |
|
}, |
|
{ |
|
"epoch": 0.8741167233708453, |
|
"grad_norm": 48.10790913779111, |
|
"learning_rate": 2.3762120898116495e-08, |
|
"logits/chosen": 0.07395198196172714, |
|
"logits/rejected": 1.8523792028427124, |
|
"logps/chosen": -409.458984375, |
|
"logps/rejected": -479.45465087890625, |
|
"loss": 0.6149, |
|
"rewards/accuracies": 0.7250000238418579, |
|
"rewards/chosen": -129.4833984375, |
|
"rewards/margins": 103.62904357910156, |
|
"rewards/rejected": -233.1124267578125, |
|
"step": 3340 |
|
}, |
|
{ |
|
"epoch": 0.8767338393090814, |
|
"grad_norm": 35.52309379757307, |
|
"learning_rate": 2.2799507522944044e-08, |
|
"logits/chosen": -0.05852848291397095, |
|
"logits/rejected": 1.087908387184143, |
|
"logps/chosen": -470.86181640625, |
|
"logps/rejected": -509.21435546875, |
|
"loss": 0.6248, |
|
"rewards/accuracies": 0.800000011920929, |
|
"rewards/chosen": -129.98536682128906, |
|
"rewards/margins": 83.6360855102539, |
|
"rewards/rejected": -213.6214599609375, |
|
"step": 3350 |
|
}, |
|
{ |
|
"epoch": 0.8793509552473174, |
|
"grad_norm": 30.69116981637674, |
|
"learning_rate": 2.1855865323510054e-08, |
|
"logits/chosen": -0.04171602800488472, |
|
"logits/rejected": 1.6160989999771118, |
|
"logps/chosen": -417.27166748046875, |
|
"logps/rejected": -514.6968994140625, |
|
"loss": 0.6023, |
|
"rewards/accuracies": 0.7749999761581421, |
|
"rewards/chosen": -129.82444763183594, |
|
"rewards/margins": 108.20865631103516, |
|
"rewards/rejected": -238.0331573486328, |
|
"step": 3360 |
|
}, |
|
{ |
|
"epoch": 0.8819680711855535, |
|
"grad_norm": 18.432172416615423, |
|
"learning_rate": 2.0931273093666573e-08, |
|
"logits/chosen": -0.05673789232969284, |
|
"logits/rejected": 1.1431539058685303, |
|
"logps/chosen": -387.72882080078125, |
|
"logps/rejected": -360.6510009765625, |
|
"loss": 0.6221, |
|
"rewards/accuracies": 0.6499999761581421, |
|
"rewards/chosen": -122.041748046875, |
|
"rewards/margins": 46.56056594848633, |
|
"rewards/rejected": -168.60232543945312, |
|
"step": 3370 |
|
}, |
|
{ |
|
"epoch": 0.8845851871237895, |
|
"grad_norm": 36.97449002155012, |
|
"learning_rate": 2.002580803659873e-08, |
|
"logits/chosen": 0.47802838683128357, |
|
"logits/rejected": 1.560175895690918, |
|
"logps/chosen": -367.7857971191406, |
|
"logps/rejected": -479.096435546875, |
|
"loss": 0.6068, |
|
"rewards/accuracies": 0.675000011920929, |
|
"rewards/chosen": -131.46559143066406, |
|
"rewards/margins": 93.1476821899414, |
|
"rewards/rejected": -224.61325073242188, |
|
"step": 3380 |
|
}, |
|
{ |
|
"epoch": 0.8872023030620256, |
|
"grad_norm": 41.84440019806065, |
|
"learning_rate": 1.9139545758378256e-08, |
|
"logits/chosen": -0.31822529435157776, |
|
"logits/rejected": 2.1346435546875, |
|
"logps/chosen": -434.1947326660156, |
|
"logps/rejected": -516.7833251953125, |
|
"loss": 0.6006, |
|
"rewards/accuracies": 0.925000011920929, |
|
"rewards/chosen": -107.50236511230469, |
|
"rewards/margins": 126.99251556396484, |
|
"rewards/rejected": -234.49490356445312, |
|
"step": 3390 |
|
}, |
|
{ |
|
"epoch": 0.8898194190002617, |
|
"grad_norm": 37.403239081733155, |
|
"learning_rate": 1.8272560261650277e-08, |
|
"logits/chosen": 0.23876461386680603, |
|
"logits/rejected": 2.3519275188446045, |
|
"logps/chosen": -370.8079528808594, |
|
"logps/rejected": -457.3379821777344, |
|
"loss": 0.6089, |
|
"rewards/accuracies": 0.8500000238418579, |
|
"rewards/chosen": -125.28385925292969, |
|
"rewards/margins": 101.42870330810547, |
|
"rewards/rejected": -226.7125701904297, |
|
"step": 3400 |
|
}, |
|
{ |
|
"epoch": 0.8924365349384977, |
|
"grad_norm": 67.36541321581835, |
|
"learning_rate": 1.742492393945427e-08, |
|
"logits/chosen": 0.07925743609666824, |
|
"logits/rejected": 1.592063069343567, |
|
"logps/chosen": -428.7183532714844, |
|
"logps/rejected": -457.3017578125, |
|
"loss": 0.6062, |
|
"rewards/accuracies": 0.7250000238418579, |
|
"rewards/chosen": -125.50584411621094, |
|
"rewards/margins": 84.96311950683594, |
|
"rewards/rejected": -210.4689483642578, |
|
"step": 3410 |
|
}, |
|
{ |
|
"epoch": 0.8950536508767338, |
|
"grad_norm": 20.44206177273988, |
|
"learning_rate": 1.6596707569179302e-08, |
|
"logits/chosen": -0.014702320098876953, |
|
"logits/rejected": 1.5920372009277344, |
|
"logps/chosen": -409.2933654785156, |
|
"logps/rejected": -514.6095581054688, |
|
"loss": 0.5756, |
|
"rewards/accuracies": 0.875, |
|
"rewards/chosen": -115.48329162597656, |
|
"rewards/margins": 133.8512725830078, |
|
"rewards/rejected": -249.3345489501953, |
|
"step": 3420 |
|
}, |
|
{ |
|
"epoch": 0.8976707668149699, |
|
"grad_norm": 28.55538975432989, |
|
"learning_rate": 1.5787980306653848e-08, |
|
"logits/chosen": -0.006506321020424366, |
|
"logits/rejected": 2.24983549118042, |
|
"logps/chosen": -427.6556091308594, |
|
"logps/rejected": -487.06060791015625, |
|
"loss": 0.5991, |
|
"rewards/accuracies": 0.8500000238418579, |
|
"rewards/chosen": -133.6538848876953, |
|
"rewards/margins": 104.88616943359375, |
|
"rewards/rejected": -238.54006958007812, |
|
"step": 3430 |
|
}, |
|
{ |
|
"epoch": 0.9002878827532059, |
|
"grad_norm": 24.356643588279173, |
|
"learning_rate": 1.499880968037165e-08, |
|
"logits/chosen": 0.4594038426876068, |
|
"logits/rejected": 2.023451805114746, |
|
"logps/chosen": -435.5736389160156, |
|
"logps/rejected": -524.2735595703125, |
|
"loss": 0.6262, |
|
"rewards/accuracies": 0.6499999761581421, |
|
"rewards/chosen": -140.4995574951172, |
|
"rewards/margins": 81.06678771972656, |
|
"rewards/rejected": -221.56637573242188, |
|
"step": 3440 |
|
}, |
|
{ |
|
"epoch": 0.902904998691442, |
|
"grad_norm": 20.783729767212474, |
|
"learning_rate": 1.4229261585852803e-08, |
|
"logits/chosen": 0.43992137908935547, |
|
"logits/rejected": 2.0338668823242188, |
|
"logps/chosen": -427.1771545410156, |
|
"logps/rejected": -489.74688720703125, |
|
"loss": 0.6145, |
|
"rewards/accuracies": 0.7749999761581421, |
|
"rewards/chosen": -127.97509765625, |
|
"rewards/margins": 101.35646057128906, |
|
"rewards/rejected": -229.3315887451172, |
|
"step": 3450 |
|
}, |
|
{ |
|
"epoch": 0.9055221146296781, |
|
"grad_norm": 15.969187300278293, |
|
"learning_rate": 1.3479400280141883e-08, |
|
"logits/chosen": 0.6696885228157043, |
|
"logits/rejected": 1.1811548471450806, |
|
"logps/chosen": -424.75689697265625, |
|
"logps/rejected": -527.1947021484375, |
|
"loss": 0.5997, |
|
"rewards/accuracies": 0.7250000238418579, |
|
"rewards/chosen": -132.3514404296875, |
|
"rewards/margins": 60.17137908935547, |
|
"rewards/rejected": -192.52281188964844, |
|
"step": 3460 |
|
}, |
|
{ |
|
"epoch": 0.9081392305679141, |
|
"grad_norm": 47.729403494478504, |
|
"learning_rate": 1.2749288376442042e-08, |
|
"logits/chosen": -0.05798863619565964, |
|
"logits/rejected": 0.5691540837287903, |
|
"logps/chosen": -359.86822509765625, |
|
"logps/rejected": -430.391357421875, |
|
"loss": 0.6333, |
|
"rewards/accuracies": 0.6499999761581421, |
|
"rewards/chosen": -104.583984375, |
|
"rewards/margins": 69.30809020996094, |
|
"rewards/rejected": -173.89205932617188, |
|
"step": 3470 |
|
}, |
|
{ |
|
"epoch": 0.9107563465061502, |
|
"grad_norm": 30.620975249350334, |
|
"learning_rate": 1.2038986838887127e-08, |
|
"logits/chosen": 0.2651011645793915, |
|
"logits/rejected": 1.6772241592407227, |
|
"logps/chosen": -347.3929443359375, |
|
"logps/rejected": -412.2870178222656, |
|
"loss": 0.6092, |
|
"rewards/accuracies": 0.800000011920929, |
|
"rewards/chosen": -113.95643615722656, |
|
"rewards/margins": 88.0910415649414, |
|
"rewards/rejected": -202.0474853515625, |
|
"step": 3480 |
|
}, |
|
{ |
|
"epoch": 0.9133734624443863, |
|
"grad_norm": 53.121108927701215, |
|
"learning_rate": 1.1348554977451131e-08, |
|
"logits/chosen": 0.2223062962293625, |
|
"logits/rejected": 1.1075026988983154, |
|
"logps/chosen": -435.0687561035156, |
|
"logps/rejected": -574.1682739257812, |
|
"loss": 0.5968, |
|
"rewards/accuracies": 0.800000011920929, |
|
"rewards/chosen": -120.94419860839844, |
|
"rewards/margins": 113.83845520019531, |
|
"rewards/rejected": -234.7826385498047, |
|
"step": 3490 |
|
}, |
|
{ |
|
"epoch": 0.9159905783826223, |
|
"grad_norm": 18.84746498341957, |
|
"learning_rate": 1.06780504429958e-08, |
|
"logits/chosen": 0.6739507913589478, |
|
"logits/rejected": 0.42967432737350464, |
|
"logps/chosen": -349.63128662109375, |
|
"logps/rejected": -455.380859375, |
|
"loss": 0.6419, |
|
"rewards/accuracies": 0.7250000238418579, |
|
"rewards/chosen": -125.32344818115234, |
|
"rewards/margins": 75.88658905029297, |
|
"rewards/rejected": -201.21005249023438, |
|
"step": 3500 |
|
}, |
|
{ |
|
"epoch": 0.9186076943208584, |
|
"grad_norm": 36.78114307261151, |
|
"learning_rate": 1.0027529222456754e-08, |
|
"logits/chosen": 0.017544955015182495, |
|
"logits/rejected": 0.7979161143302917, |
|
"logps/chosen": -395.4692687988281, |
|
"logps/rejected": -522.6412963867188, |
|
"loss": 0.6098, |
|
"rewards/accuracies": 0.800000011920929, |
|
"rewards/chosen": -127.53913879394531, |
|
"rewards/margins": 100.97230529785156, |
|
"rewards/rejected": -228.51144409179688, |
|
"step": 3510 |
|
}, |
|
{ |
|
"epoch": 0.9212248102590945, |
|
"grad_norm": 32.458861273146255, |
|
"learning_rate": 9.397045634168766e-09, |
|
"logits/chosen": -0.1015222892165184, |
|
"logits/rejected": 1.0661077499389648, |
|
"logps/chosen": -444.6565856933594, |
|
"logps/rejected": -434.8824768066406, |
|
"loss": 0.6052, |
|
"rewards/accuracies": 0.699999988079071, |
|
"rewards/chosen": -134.20489501953125, |
|
"rewards/margins": 66.43836975097656, |
|
"rewards/rejected": -200.6432647705078, |
|
"step": 3520 |
|
}, |
|
{ |
|
"epoch": 0.9238419261973305, |
|
"grad_norm": 42.81956388214368, |
|
"learning_rate": 8.78665232332998e-09, |
|
"logits/chosen": 0.6678240299224854, |
|
"logits/rejected": 2.1921708583831787, |
|
"logps/chosen": -428.3387145996094, |
|
"logps/rejected": -553.4244995117188, |
|
"loss": 0.5909, |
|
"rewards/accuracies": 0.7749999761581421, |
|
"rewards/chosen": -119.02767181396484, |
|
"rewards/margins": 116.71366119384766, |
|
"rewards/rejected": -235.74130249023438, |
|
"step": 3530 |
|
}, |
|
{ |
|
"epoch": 0.9264590421355666, |
|
"grad_norm": 34.21637872966215, |
|
"learning_rate": 8.196400257606206e-09, |
|
"logits/chosen": 0.47500696778297424, |
|
"logits/rejected": 1.3353134393692017, |
|
"logps/chosen": -376.7292785644531, |
|
"logps/rejected": -447.9065856933594, |
|
"loss": 0.6407, |
|
"rewards/accuracies": 0.7749999761581421, |
|
"rewards/chosen": -136.65072631835938, |
|
"rewards/margins": 82.5105972290039, |
|
"rewards/rejected": -219.1613311767578, |
|
"step": 3540 |
|
}, |
|
{ |
|
"epoch": 0.9290761580738026, |
|
"grad_norm": 53.978193486224676, |
|
"learning_rate": 7.626338722875075e-09, |
|
"logits/chosen": -0.10515992343425751, |
|
"logits/rejected": 1.1506468057632446, |
|
"logps/chosen": -461.34552001953125, |
|
"logps/rejected": -540.5074462890625, |
|
"loss": 0.6092, |
|
"rewards/accuracies": 0.75, |
|
"rewards/chosen": -160.37290954589844, |
|
"rewards/margins": 87.07331848144531, |
|
"rewards/rejected": -247.4462432861328, |
|
"step": 3550 |
|
}, |
|
{ |
|
"epoch": 0.9316932740120387, |
|
"grad_norm": 16.239388562963665, |
|
"learning_rate": 7.0765153191106875e-09, |
|
"logits/chosen": 1.0949875116348267, |
|
"logits/rejected": 0.985447883605957, |
|
"logps/chosen": -387.4863586425781, |
|
"logps/rejected": -427.345703125, |
|
"loss": 0.6532, |
|
"rewards/accuracies": 0.75, |
|
"rewards/chosen": -128.79241943359375, |
|
"rewards/margins": 64.76528930664062, |
|
"rewards/rejected": -193.55770874023438, |
|
"step": 3560 |
|
}, |
|
{ |
|
"epoch": 0.9343103899502748, |
|
"grad_norm": 25.74478178481144, |
|
"learning_rate": 6.54697595640899e-09, |
|
"logits/chosen": 0.4125773310661316, |
|
"logits/rejected": 1.1157901287078857, |
|
"logps/chosen": -324.0262451171875, |
|
"logps/rejected": -398.330078125, |
|
"loss": 0.5769, |
|
"rewards/accuracies": 0.7749999761581421, |
|
"rewards/chosen": -101.90724182128906, |
|
"rewards/margins": 86.06889343261719, |
|
"rewards/rejected": -187.97613525390625, |
|
"step": 3570 |
|
}, |
|
{ |
|
"epoch": 0.9369275058885108, |
|
"grad_norm": 32.83789705445322, |
|
"learning_rate": 6.037764851154425e-09, |
|
"logits/chosen": -0.2145831137895584, |
|
"logits/rejected": 1.321026086807251, |
|
"logps/chosen": -455.0643615722656, |
|
"logps/rejected": -509.22119140625, |
|
"loss": 0.5892, |
|
"rewards/accuracies": 0.800000011920929, |
|
"rewards/chosen": -125.3864517211914, |
|
"rewards/margins": 87.76890563964844, |
|
"rewards/rejected": -213.1553497314453, |
|
"step": 3580 |
|
}, |
|
{ |
|
"epoch": 0.9395446218267469, |
|
"grad_norm": 32.398022325965066, |
|
"learning_rate": 5.548924522327747e-09, |
|
"logits/chosen": -0.08072943985462189, |
|
"logits/rejected": 1.052816390991211, |
|
"logps/chosen": -424.6456604003906, |
|
"logps/rejected": -463.2438049316406, |
|
"loss": 0.6267, |
|
"rewards/accuracies": 0.824999988079071, |
|
"rewards/chosen": -117.72221374511719, |
|
"rewards/margins": 101.6100845336914, |
|
"rewards/rejected": -219.3323211669922, |
|
"step": 3590 |
|
}, |
|
{ |
|
"epoch": 0.942161737764983, |
|
"grad_norm": 43.737861260512695, |
|
"learning_rate": 5.080495787955691e-09, |
|
"logits/chosen": 0.2663705050945282, |
|
"logits/rejected": 1.5858561992645264, |
|
"logps/chosen": -440.0337829589844, |
|
"logps/rejected": -425.4478454589844, |
|
"loss": 0.6225, |
|
"rewards/accuracies": 0.6499999761581421, |
|
"rewards/chosen": -116.02880859375, |
|
"rewards/margins": 71.27129364013672, |
|
"rewards/rejected": -187.3000946044922, |
|
"step": 3600 |
|
}, |
|
{ |
|
"epoch": 0.944778853703219, |
|
"grad_norm": 19.943029188551513, |
|
"learning_rate": 4.632517761702814e-09, |
|
"logits/chosen": 0.42500191926956177, |
|
"logits/rejected": 1.425812005996704, |
|
"logps/chosen": -473.26641845703125, |
|
"logps/rejected": -651.4464111328125, |
|
"loss": 0.6189, |
|
"rewards/accuracies": 0.8500000238418579, |
|
"rewards/chosen": -147.15560913085938, |
|
"rewards/margins": 129.23179626464844, |
|
"rewards/rejected": -276.387451171875, |
|
"step": 3610 |
|
}, |
|
{ |
|
"epoch": 0.9473959696414551, |
|
"grad_norm": 24.74762269805631, |
|
"learning_rate": 4.205027849605358e-09, |
|
"logits/chosen": 0.15811410546302795, |
|
"logits/rejected": 1.1120755672454834, |
|
"logps/chosen": -396.139404296875, |
|
"logps/rejected": -448.7173767089844, |
|
"loss": 0.6211, |
|
"rewards/accuracies": 0.6499999761581421, |
|
"rewards/chosen": -135.4348602294922, |
|
"rewards/margins": 68.15974426269531, |
|
"rewards/rejected": -203.59461975097656, |
|
"step": 3620 |
|
}, |
|
{ |
|
"epoch": 0.9500130855796912, |
|
"grad_norm": 22.40191130638278, |
|
"learning_rate": 3.798061746947995e-09, |
|
"logits/chosen": 0.37473368644714355, |
|
"logits/rejected": 1.1352459192276, |
|
"logps/chosen": -445.27044677734375, |
|
"logps/rejected": -539.070556640625, |
|
"loss": 0.615, |
|
"rewards/accuracies": 0.800000011920929, |
|
"rewards/chosen": -136.21102905273438, |
|
"rewards/margins": 94.1988754272461, |
|
"rewards/rejected": -230.409912109375, |
|
"step": 3630 |
|
}, |
|
{ |
|
"epoch": 0.9526302015179272, |
|
"grad_norm": 28.695280469787797, |
|
"learning_rate": 3.411653435283157e-09, |
|
"logits/chosen": -0.19782522320747375, |
|
"logits/rejected": 1.6401824951171875, |
|
"logps/chosen": -430.9485778808594, |
|
"logps/rejected": -515.1265869140625, |
|
"loss": 0.6242, |
|
"rewards/accuracies": 0.8500000238418579, |
|
"rewards/chosen": -125.76761627197266, |
|
"rewards/margins": 113.26475524902344, |
|
"rewards/rejected": -239.0323486328125, |
|
"step": 3640 |
|
}, |
|
{ |
|
"epoch": 0.9552473174561633, |
|
"grad_norm": 33.000510950766326, |
|
"learning_rate": 3.0458351795936698e-09, |
|
"logits/chosen": 0.09500176459550858, |
|
"logits/rejected": 1.7043838500976562, |
|
"logps/chosen": -439.33831787109375, |
|
"logps/rejected": -497.22515869140625, |
|
"loss": 0.5922, |
|
"rewards/accuracies": 0.699999988079071, |
|
"rewards/chosen": -136.70016479492188, |
|
"rewards/margins": 84.02696228027344, |
|
"rewards/rejected": -220.7271270751953, |
|
"step": 3650 |
|
}, |
|
{ |
|
"epoch": 0.9578644333943994, |
|
"grad_norm": 27.216858014189555, |
|
"learning_rate": 2.700637525598598e-09, |
|
"logits/chosen": 0.1739320009946823, |
|
"logits/rejected": 1.9256073236465454, |
|
"logps/chosen": -415.5773010253906, |
|
"logps/rejected": -464.6927185058594, |
|
"loss": 0.6142, |
|
"rewards/accuracies": 0.675000011920929, |
|
"rewards/chosen": -122.09500885009766, |
|
"rewards/margins": 85.67970275878906, |
|
"rewards/rejected": -207.7747039794922, |
|
"step": 3660 |
|
}, |
|
{ |
|
"epoch": 0.9604815493326354, |
|
"grad_norm": 41.15234660865958, |
|
"learning_rate": 2.3760892972027324e-09, |
|
"logits/chosen": 0.7606725096702576, |
|
"logits/rejected": 2.247407913208008, |
|
"logps/chosen": -464.6285095214844, |
|
"logps/rejected": -494.5741271972656, |
|
"loss": 0.6657, |
|
"rewards/accuracies": 0.800000011920929, |
|
"rewards/chosen": -143.5747528076172, |
|
"rewards/margins": 92.42018127441406, |
|
"rewards/rejected": -235.9949188232422, |
|
"step": 3670 |
|
}, |
|
{ |
|
"epoch": 0.9630986652708715, |
|
"grad_norm": 37.89432017028242, |
|
"learning_rate": 2.0722175940897645e-09, |
|
"logits/chosen": -0.2540181279182434, |
|
"logits/rejected": 1.4863791465759277, |
|
"logps/chosen": -487.421875, |
|
"logps/rejected": -469.3511657714844, |
|
"loss": 0.6164, |
|
"rewards/accuracies": 0.699999988079071, |
|
"rewards/chosen": -140.024169921875, |
|
"rewards/margins": 66.67285919189453, |
|
"rewards/rejected": -206.69699096679688, |
|
"step": 3680 |
|
}, |
|
{ |
|
"epoch": 0.9657157812091076, |
|
"grad_norm": 28.841623740215717, |
|
"learning_rate": 1.7890477894593748e-09, |
|
"logits/chosen": 0.5983523726463318, |
|
"logits/rejected": 2.3860888481140137, |
|
"logps/chosen": -413.36090087890625, |
|
"logps/rejected": -475.72998046875, |
|
"loss": 0.6261, |
|
"rewards/accuracies": 0.75, |
|
"rewards/chosen": -129.737548828125, |
|
"rewards/margins": 118.51426696777344, |
|
"rewards/rejected": -248.25180053710938, |
|
"step": 3690 |
|
}, |
|
{ |
|
"epoch": 0.9683328971473436, |
|
"grad_norm": 30.85848190499018, |
|
"learning_rate": 1.5266035279088708e-09, |
|
"logits/chosen": 0.642953634262085, |
|
"logits/rejected": 1.7377115488052368, |
|
"logps/chosen": -435.58172607421875, |
|
"logps/rejected": -495.81524658203125, |
|
"loss": 0.6194, |
|
"rewards/accuracies": 0.7749999761581421, |
|
"rewards/chosen": -122.56893157958984, |
|
"rewards/margins": 103.9132308959961, |
|
"rewards/rejected": -226.48214721679688, |
|
"step": 3700 |
|
}, |
|
{ |
|
"epoch": 0.9709500130855797, |
|
"grad_norm": 38.33508936295693, |
|
"learning_rate": 1.2849067234584621e-09, |
|
"logits/chosen": 0.22355318069458008, |
|
"logits/rejected": 1.7621004581451416, |
|
"logps/chosen": -331.8682556152344, |
|
"logps/rejected": -411.5686950683594, |
|
"loss": 0.6057, |
|
"rewards/accuracies": 0.824999988079071, |
|
"rewards/chosen": -128.43795776367188, |
|
"rewards/margins": 97.2995376586914, |
|
"rewards/rejected": -225.7374725341797, |
|
"step": 3710 |
|
}, |
|
{ |
|
"epoch": 0.9735671290238157, |
|
"grad_norm": 32.865483577563424, |
|
"learning_rate": 1.0639775577218625e-09, |
|
"logits/chosen": -0.21633200347423553, |
|
"logits/rejected": 1.0267454385757446, |
|
"logps/chosen": -437.278564453125, |
|
"logps/rejected": -542.7936401367188, |
|
"loss": 0.5941, |
|
"rewards/accuracies": 0.800000011920929, |
|
"rewards/chosen": -127.906494140625, |
|
"rewards/margins": 112.18253326416016, |
|
"rewards/rejected": -240.08901977539062, |
|
"step": 3720 |
|
}, |
|
{ |
|
"epoch": 0.9761842449620518, |
|
"grad_norm": 55.69172200477488, |
|
"learning_rate": 8.638344782207485e-10, |
|
"logits/chosen": 0.11773265898227692, |
|
"logits/rejected": 1.5340635776519775, |
|
"logps/chosen": -337.8634948730469, |
|
"logps/rejected": -417.99383544921875, |
|
"loss": 0.5949, |
|
"rewards/accuracies": 0.800000011920929, |
|
"rewards/chosen": -123.92936706542969, |
|
"rewards/margins": 97.03419494628906, |
|
"rewards/rejected": -220.9635772705078, |
|
"step": 3730 |
|
}, |
|
{ |
|
"epoch": 0.9788013609002879, |
|
"grad_norm": 29.33864063438161, |
|
"learning_rate": 6.844941968447149e-10, |
|
"logits/chosen": 0.4394141137599945, |
|
"logits/rejected": 0.8948599696159363, |
|
"logps/chosen": -397.30047607421875, |
|
"logps/rejected": -521.3082885742188, |
|
"loss": 0.6459, |
|
"rewards/accuracies": 0.699999988079071, |
|
"rewards/chosen": -135.5926513671875, |
|
"rewards/margins": 71.82649993896484, |
|
"rewards/rejected": -207.4191436767578, |
|
"step": 3740 |
|
}, |
|
{ |
|
"epoch": 0.9814184768385239, |
|
"grad_norm": 32.254807951743736, |
|
"learning_rate": 5.25971688455612e-10, |
|
"logits/chosen": 0.3935531675815582, |
|
"logits/rejected": 1.233229398727417, |
|
"logps/chosen": -420.3367614746094, |
|
"logps/rejected": -451.939697265625, |
|
"loss": 0.602, |
|
"rewards/accuracies": 0.699999988079071, |
|
"rewards/chosen": -139.55093383789062, |
|
"rewards/margins": 70.234130859375, |
|
"rewards/rejected": -209.78506469726562, |
|
"step": 3750 |
|
}, |
|
{ |
|
"epoch": 0.98403559277676, |
|
"grad_norm": 16.725344955385015, |
|
"learning_rate": 3.882801896372967e-10, |
|
"logits/chosen": -0.2951185405254364, |
|
"logits/rejected": 1.3272792100906372, |
|
"logps/chosen": -458.79693603515625, |
|
"logps/rejected": -501.19525146484375, |
|
"loss": 0.5835, |
|
"rewards/accuracies": 0.800000011920929, |
|
"rewards/chosen": -117.83502197265625, |
|
"rewards/margins": 97.3798828125, |
|
"rewards/rejected": -215.2148895263672, |
|
"step": 3760 |
|
}, |
|
{ |
|
"epoch": 0.9866527087149961, |
|
"grad_norm": 23.047469492542017, |
|
"learning_rate": 2.714311975902661e-10, |
|
"logits/chosen": 0.4829816222190857, |
|
"logits/rejected": 0.9257513284683228, |
|
"logps/chosen": -404.45098876953125, |
|
"logps/rejected": -491.4329528808594, |
|
"loss": 0.6195, |
|
"rewards/accuracies": 0.7749999761581421, |
|
"rewards/chosen": -126.78916931152344, |
|
"rewards/margins": 86.20039367675781, |
|
"rewards/rejected": -212.9895477294922, |
|
"step": 3770 |
|
}, |
|
{ |
|
"epoch": 0.9892698246532321, |
|
"grad_norm": 42.77852100209512, |
|
"learning_rate": 1.754344691717591e-10, |
|
"logits/chosen": 0.8735979199409485, |
|
"logits/rejected": 1.6277217864990234, |
|
"logps/chosen": -358.1724548339844, |
|
"logps/rejected": -410.5965881347656, |
|
"loss": 0.6026, |
|
"rewards/accuracies": 0.699999988079071, |
|
"rewards/chosen": -123.83882141113281, |
|
"rewards/margins": 71.39359283447266, |
|
"rewards/rejected": -195.23239135742188, |
|
"step": 3780 |
|
}, |
|
{ |
|
"epoch": 0.9918869405914682, |
|
"grad_norm": 48.71547389714528, |
|
"learning_rate": 1.0029802008096333e-10, |
|
"logits/chosen": -0.3842397928237915, |
|
"logits/rejected": 1.9089739322662354, |
|
"logps/chosen": -412.76605224609375, |
|
"logps/rejected": -472.7958068847656, |
|
"loss": 0.5939, |
|
"rewards/accuracies": 0.824999988079071, |
|
"rewards/chosen": -129.0559539794922, |
|
"rewards/margins": 96.40486907958984, |
|
"rewards/rejected": -225.46084594726562, |
|
"step": 3790 |
|
}, |
|
{ |
|
"epoch": 0.9945040565297043, |
|
"grad_norm": 35.70591610414197, |
|
"learning_rate": 4.602812418974533e-11, |
|
"logits/chosen": 0.3812098503112793, |
|
"logits/rejected": 2.2290968894958496, |
|
"logps/chosen": -390.21673583984375, |
|
"logps/rejected": -426.26788330078125, |
|
"loss": 0.6217, |
|
"rewards/accuracies": 0.675000011920929, |
|
"rewards/chosen": -138.75518798828125, |
|
"rewards/margins": 76.2783203125, |
|
"rewards/rejected": -215.0334930419922, |
|
"step": 3800 |
|
}, |
|
{ |
|
"epoch": 0.9971211724679403, |
|
"grad_norm": 34.935749182467326, |
|
"learning_rate": 1.2629313018819309e-11, |
|
"logits/chosen": 0.44964084029197693, |
|
"logits/rejected": 1.517663598060608, |
|
"logps/chosen": -444.845703125, |
|
"logps/rejected": -547.6998901367188, |
|
"loss": 0.6326, |
|
"rewards/accuracies": 0.7250000238418579, |
|
"rewards/chosen": -141.57296752929688, |
|
"rewards/margins": 83.2319107055664, |
|
"rewards/rejected": -224.8048858642578, |
|
"step": 3810 |
|
}, |
|
{ |
|
"epoch": 0.9997382884061764, |
|
"grad_norm": 19.681913314378978, |
|
"learning_rate": 1.0437535929996855e-13, |
|
"logits/chosen": 0.43567976355552673, |
|
"logits/rejected": 2.002690315246582, |
|
"logps/chosen": -437.58770751953125, |
|
"logps/rejected": -477.990966796875, |
|
"loss": 0.6177, |
|
"rewards/accuracies": 0.7250000238418579, |
|
"rewards/chosen": -148.7041778564453, |
|
"rewards/margins": 90.53707122802734, |
|
"rewards/rejected": -239.2412567138672, |
|
"step": 3820 |
|
}, |
|
{ |
|
"epoch": 1.0, |
|
"step": 3821, |
|
"total_flos": 0.0, |
|
"train_loss": 0.6291336198965001, |
|
"train_runtime": 16881.5542, |
|
"train_samples_per_second": 3.621, |
|
"train_steps_per_second": 0.226 |
|
} |
|
], |
|
"logging_steps": 10, |
|
"max_steps": 3821, |
|
"num_input_tokens_seen": 0, |
|
"num_train_epochs": 1, |
|
"save_steps": 2000, |
|
"stateful_callbacks": { |
|
"TrainerControl": { |
|
"args": { |
|
"should_epoch_stop": false, |
|
"should_evaluate": false, |
|
"should_log": false, |
|
"should_save": true, |
|
"should_training_stop": false |
|
}, |
|
"attributes": {} |
|
} |
|
}, |
|
"total_flos": 0.0, |
|
"train_batch_size": 4, |
|
"trial_name": null, |
|
"trial_params": null |
|
} |
|
|