|
{ |
|
"best_metric": null, |
|
"best_model_checkpoint": null, |
|
"epoch": 2.0, |
|
"eval_steps": 100, |
|
"global_step": 1000, |
|
"is_hyper_param_search": false, |
|
"is_local_process_zero": true, |
|
"is_world_process_zero": true, |
|
"log_history": [ |
|
{ |
|
"epoch": 0.0, |
|
"learning_rate": 5.0000000000000004e-08, |
|
"logits/chosen": -3.1812663078308105, |
|
"logits/rejected": -2.810722827911377, |
|
"logps/chosen": -118.07083129882812, |
|
"logps/rejected": -100.80718994140625, |
|
"loss": 0.0011, |
|
"rewards/accuracies": 0.0, |
|
"rewards/chosen": 0.0, |
|
"rewards/margins": 0.0, |
|
"rewards/rejected": 0.0, |
|
"step": 1 |
|
}, |
|
{ |
|
"epoch": 0.02, |
|
"learning_rate": 5.000000000000001e-07, |
|
"logits/chosen": -2.9193410873413086, |
|
"logits/rejected": -2.7373409271240234, |
|
"logps/chosen": -116.89999389648438, |
|
"logps/rejected": -116.49290466308594, |
|
"loss": 0.0011, |
|
"rewards/accuracies": 0.3888888955116272, |
|
"rewards/chosen": -0.00038699465221725404, |
|
"rewards/margins": -0.0005435158964246511, |
|
"rewards/rejected": 0.00015652130241505802, |
|
"step": 10 |
|
}, |
|
{ |
|
"epoch": 0.04, |
|
"learning_rate": 1.0000000000000002e-06, |
|
"logits/chosen": -2.981767416000366, |
|
"logits/rejected": -2.6092238426208496, |
|
"logps/chosen": -164.24819946289062, |
|
"logps/rejected": -164.7591552734375, |
|
"loss": 0.0011, |
|
"rewards/accuracies": 0.4000000059604645, |
|
"rewards/chosen": -6.044005203875713e-05, |
|
"rewards/margins": -0.0003916244604624808, |
|
"rewards/rejected": 0.0003311844193376601, |
|
"step": 20 |
|
}, |
|
{ |
|
"epoch": 0.06, |
|
"learning_rate": 1.5e-06, |
|
"logits/chosen": -2.913992404937744, |
|
"logits/rejected": -2.6422486305236816, |
|
"logps/chosen": -153.99203491210938, |
|
"logps/rejected": -141.27601623535156, |
|
"loss": 0.0011, |
|
"rewards/accuracies": 0.44999998807907104, |
|
"rewards/chosen": -0.00013318247511051595, |
|
"rewards/margins": 0.0008146190084517002, |
|
"rewards/rejected": -0.0009478016872890294, |
|
"step": 30 |
|
}, |
|
{ |
|
"epoch": 0.08, |
|
"learning_rate": 2.0000000000000003e-06, |
|
"logits/chosen": -2.9979870319366455, |
|
"logits/rejected": -2.5024120807647705, |
|
"logps/chosen": -180.6337890625, |
|
"logps/rejected": -168.78204345703125, |
|
"loss": 0.001, |
|
"rewards/accuracies": 0.699999988079071, |
|
"rewards/chosen": 0.002415223978459835, |
|
"rewards/margins": 0.002099759876728058, |
|
"rewards/rejected": 0.0003154640144202858, |
|
"step": 40 |
|
}, |
|
{ |
|
"epoch": 0.1, |
|
"learning_rate": 2.5e-06, |
|
"logits/chosen": -2.7826578617095947, |
|
"logits/rejected": -2.437479019165039, |
|
"logps/chosen": -167.60964965820312, |
|
"logps/rejected": -154.45205688476562, |
|
"loss": 0.001, |
|
"rewards/accuracies": 0.550000011920929, |
|
"rewards/chosen": 0.0028098905459046364, |
|
"rewards/margins": 0.0014571474166586995, |
|
"rewards/rejected": 0.0013527432456612587, |
|
"step": 50 |
|
}, |
|
{ |
|
"epoch": 0.12, |
|
"learning_rate": 3e-06, |
|
"logits/chosen": -2.715552806854248, |
|
"logits/rejected": -2.5350234508514404, |
|
"logps/chosen": -147.0255126953125, |
|
"logps/rejected": -136.99874877929688, |
|
"loss": 0.0011, |
|
"rewards/accuracies": 0.6499999761581421, |
|
"rewards/chosen": 0.001564659527502954, |
|
"rewards/margins": 0.0012098618317395449, |
|
"rewards/rejected": 0.0003547971718944609, |
|
"step": 60 |
|
}, |
|
{ |
|
"epoch": 0.14, |
|
"learning_rate": 3.5e-06, |
|
"logits/chosen": -3.064145088195801, |
|
"logits/rejected": -2.715378522872925, |
|
"logps/chosen": -163.35397338867188, |
|
"logps/rejected": -152.8401336669922, |
|
"loss": 0.001, |
|
"rewards/accuracies": 0.6000000238418579, |
|
"rewards/chosen": 0.011319306679069996, |
|
"rewards/margins": 0.002820510882884264, |
|
"rewards/rejected": 0.00849879626184702, |
|
"step": 70 |
|
}, |
|
{ |
|
"epoch": 0.16, |
|
"learning_rate": 4.000000000000001e-06, |
|
"logits/chosen": -3.031562089920044, |
|
"logits/rejected": -2.762894868850708, |
|
"logps/chosen": -154.64096069335938, |
|
"logps/rejected": -142.45066833496094, |
|
"loss": 0.001, |
|
"rewards/accuracies": 0.699999988079071, |
|
"rewards/chosen": 0.015446094796061516, |
|
"rewards/margins": 0.0030269301496446133, |
|
"rewards/rejected": 0.012419164180755615, |
|
"step": 80 |
|
}, |
|
{ |
|
"epoch": 0.18, |
|
"learning_rate": 4.5e-06, |
|
"logits/chosen": -3.0730316638946533, |
|
"logits/rejected": -2.5636696815490723, |
|
"logps/chosen": -126.04837799072266, |
|
"logps/rejected": -110.87501525878906, |
|
"loss": 0.0009, |
|
"rewards/accuracies": 0.6499999761581421, |
|
"rewards/chosen": 0.018308930099010468, |
|
"rewards/margins": 0.008363587781786919, |
|
"rewards/rejected": 0.009945342317223549, |
|
"step": 90 |
|
}, |
|
{ |
|
"epoch": 0.2, |
|
"learning_rate": 5e-06, |
|
"logits/chosen": -2.9546635150909424, |
|
"logits/rejected": -2.5928895473480225, |
|
"logps/chosen": -133.506103515625, |
|
"logps/rejected": -123.97562408447266, |
|
"loss": 0.0008, |
|
"rewards/accuracies": 0.75, |
|
"rewards/chosen": 0.014314673840999603, |
|
"rewards/margins": 0.005265203304588795, |
|
"rewards/rejected": 0.009049469605088234, |
|
"step": 100 |
|
}, |
|
{ |
|
"epoch": 0.2, |
|
"eval_logits/chosen": -2.4007108211517334, |
|
"eval_logits/rejected": -2.2025678157806396, |
|
"eval_logps/chosen": -269.69903564453125, |
|
"eval_logps/rejected": -250.2169952392578, |
|
"eval_loss": 0.0019292256329208612, |
|
"eval_rewards/accuracies": 0.5299999713897705, |
|
"eval_rewards/chosen": -0.01111243013292551, |
|
"eval_rewards/margins": 0.002662122482433915, |
|
"eval_rewards/rejected": -0.013774552382528782, |
|
"eval_runtime": 1423.4882, |
|
"eval_samples_per_second": 1.405, |
|
"eval_steps_per_second": 0.702, |
|
"step": 100 |
|
}, |
|
{ |
|
"epoch": 0.22, |
|
"learning_rate": 4.99847706754774e-06, |
|
"logits/chosen": -3.0832509994506836, |
|
"logits/rejected": -2.6540307998657227, |
|
"logps/chosen": -158.02037048339844, |
|
"logps/rejected": -128.09632873535156, |
|
"loss": 0.0012, |
|
"rewards/accuracies": 0.800000011920929, |
|
"rewards/chosen": 0.03201522305607796, |
|
"rewards/margins": 0.014926651492714882, |
|
"rewards/rejected": 0.017088569700717926, |
|
"step": 110 |
|
}, |
|
{ |
|
"epoch": 0.24, |
|
"learning_rate": 4.993910125649561e-06, |
|
"logits/chosen": -3.000650644302368, |
|
"logits/rejected": -2.5093066692352295, |
|
"logps/chosen": -111.5718765258789, |
|
"logps/rejected": -91.67008209228516, |
|
"loss": 0.0007, |
|
"rewards/accuracies": 0.8500000238418579, |
|
"rewards/chosen": 0.017279163002967834, |
|
"rewards/margins": 0.016767729073762894, |
|
"rewards/rejected": 0.0005114328814670444, |
|
"step": 120 |
|
}, |
|
{ |
|
"epoch": 0.26, |
|
"learning_rate": 4.986304738420684e-06, |
|
"logits/chosen": -2.8899765014648438, |
|
"logits/rejected": -2.545956611633301, |
|
"logps/chosen": -157.5283660888672, |
|
"logps/rejected": -135.06536865234375, |
|
"loss": 0.001, |
|
"rewards/accuracies": 0.699999988079071, |
|
"rewards/chosen": 0.006769341416656971, |
|
"rewards/margins": 0.013469241559505463, |
|
"rewards/rejected": -0.006699901074171066, |
|
"step": 130 |
|
}, |
|
{ |
|
"epoch": 0.28, |
|
"learning_rate": 4.975670171853926e-06, |
|
"logits/chosen": -3.0038022994995117, |
|
"logits/rejected": -2.8060638904571533, |
|
"logps/chosen": -172.43551635742188, |
|
"logps/rejected": -161.0656280517578, |
|
"loss": 0.001, |
|
"rewards/accuracies": 0.5, |
|
"rewards/chosen": -0.006354503333568573, |
|
"rewards/margins": 0.004105663392692804, |
|
"rewards/rejected": -0.01046016626060009, |
|
"step": 140 |
|
}, |
|
{ |
|
"epoch": 0.3, |
|
"learning_rate": 4.962019382530521e-06, |
|
"logits/chosen": -2.938009023666382, |
|
"logits/rejected": -2.3462884426116943, |
|
"logps/chosen": -113.1685791015625, |
|
"logps/rejected": -100.8033676147461, |
|
"loss": 0.001, |
|
"rewards/accuracies": 0.6499999761581421, |
|
"rewards/chosen": 0.00343899498693645, |
|
"rewards/margins": 0.007120263762772083, |
|
"rewards/rejected": -0.0036812685430049896, |
|
"step": 150 |
|
}, |
|
{ |
|
"epoch": 0.32, |
|
"learning_rate": 4.9453690018345144e-06, |
|
"logits/chosen": -3.1557974815368652, |
|
"logits/rejected": -2.805929183959961, |
|
"logps/chosen": -212.8404998779297, |
|
"logps/rejected": -204.03390502929688, |
|
"loss": 0.0013, |
|
"rewards/accuracies": 0.550000011920929, |
|
"rewards/chosen": -0.006425648927688599, |
|
"rewards/margins": -0.0002856804057955742, |
|
"rewards/rejected": -0.0061399685218930244, |
|
"step": 160 |
|
}, |
|
{ |
|
"epoch": 0.34, |
|
"learning_rate": 4.925739315689991e-06, |
|
"logits/chosen": -2.8449888229370117, |
|
"logits/rejected": -2.6668992042541504, |
|
"logps/chosen": -205.7440643310547, |
|
"logps/rejected": -196.88563537597656, |
|
"loss": 0.0012, |
|
"rewards/accuracies": 0.5, |
|
"rewards/chosen": -0.005873252637684345, |
|
"rewards/margins": 0.0015904292231425643, |
|
"rewards/rejected": -0.007463681511580944, |
|
"step": 170 |
|
}, |
|
{ |
|
"epoch": 0.36, |
|
"learning_rate": 4.903154239845798e-06, |
|
"logits/chosen": -3.059779167175293, |
|
"logits/rejected": -2.7152631282806396, |
|
"logps/chosen": -178.5559539794922, |
|
"logps/rejected": -174.60049438476562, |
|
"loss": 0.001, |
|
"rewards/accuracies": 0.4000000059604645, |
|
"rewards/chosen": 0.004108003806322813, |
|
"rewards/margins": 0.0049694306217134, |
|
"rewards/rejected": -0.0008614275720901787, |
|
"step": 180 |
|
}, |
|
{ |
|
"epoch": 0.38, |
|
"learning_rate": 4.8776412907378845e-06, |
|
"logits/chosen": -2.9436910152435303, |
|
"logits/rejected": -2.4861669540405273, |
|
"logps/chosen": -154.61508178710938, |
|
"logps/rejected": -129.8065643310547, |
|
"loss": 0.0008, |
|
"rewards/accuracies": 0.8500000238418579, |
|
"rewards/chosen": 0.02211841568350792, |
|
"rewards/margins": 0.012492652982473373, |
|
"rewards/rejected": 0.00962576363235712, |
|
"step": 190 |
|
}, |
|
{ |
|
"epoch": 0.4, |
|
"learning_rate": 4.849231551964771e-06, |
|
"logits/chosen": -2.934830904006958, |
|
"logits/rejected": -2.599217653274536, |
|
"logps/chosen": -148.74435424804688, |
|
"logps/rejected": -139.92453002929688, |
|
"loss": 0.0006, |
|
"rewards/accuracies": 0.8500000238418579, |
|
"rewards/chosen": 0.02799961529672146, |
|
"rewards/margins": 0.012480301782488823, |
|
"rewards/rejected": 0.015519311651587486, |
|
"step": 200 |
|
}, |
|
{ |
|
"epoch": 0.4, |
|
"eval_logits/chosen": -2.4034218788146973, |
|
"eval_logits/rejected": -2.205068349838257, |
|
"eval_logps/chosen": -270.9541015625, |
|
"eval_logps/rejected": -251.13922119140625, |
|
"eval_loss": 0.002935634693130851, |
|
"eval_rewards/accuracies": 0.4909999966621399, |
|
"eval_rewards/chosen": -0.0236628670245409, |
|
"eval_rewards/margins": -0.0006660502986051142, |
|
"eval_rewards/rejected": -0.022996816784143448, |
|
"eval_runtime": 1423.7196, |
|
"eval_samples_per_second": 1.405, |
|
"eval_steps_per_second": 0.702, |
|
"step": 200 |
|
}, |
|
{ |
|
"epoch": 0.42, |
|
"learning_rate": 4.817959636416969e-06, |
|
"logits/chosen": -2.7869691848754883, |
|
"logits/rejected": -2.3660902976989746, |
|
"logps/chosen": -188.44235229492188, |
|
"logps/rejected": -165.0704345703125, |
|
"loss": 0.0007, |
|
"rewards/accuracies": 0.6499999761581421, |
|
"rewards/chosen": 0.023742977529764175, |
|
"rewards/margins": 0.013938216492533684, |
|
"rewards/rejected": 0.009804759174585342, |
|
"step": 210 |
|
}, |
|
{ |
|
"epoch": 0.44, |
|
"learning_rate": 4.783863644106502e-06, |
|
"logits/chosen": -2.7817561626434326, |
|
"logits/rejected": -2.4513843059539795, |
|
"logps/chosen": -129.5194854736328, |
|
"logps/rejected": -119.9756088256836, |
|
"loss": 0.0011, |
|
"rewards/accuracies": 0.6499999761581421, |
|
"rewards/chosen": 0.014000030234456062, |
|
"rewards/margins": 0.006808738224208355, |
|
"rewards/rejected": 0.007191292010247707, |
|
"step": 220 |
|
}, |
|
{ |
|
"epoch": 0.46, |
|
"learning_rate": 4.746985115747918e-06, |
|
"logits/chosen": -2.997894525527954, |
|
"logits/rejected": -2.7682430744171143, |
|
"logps/chosen": -206.5100860595703, |
|
"logps/rejected": -196.3065185546875, |
|
"loss": 0.0007, |
|
"rewards/accuracies": 0.699999988079071, |
|
"rewards/chosen": 0.01694752275943756, |
|
"rewards/margins": 0.008601350709795952, |
|
"rewards/rejected": 0.00834617204964161, |
|
"step": 230 |
|
}, |
|
{ |
|
"epoch": 0.48, |
|
"learning_rate": 4.707368982147318e-06, |
|
"logits/chosen": -2.9324448108673096, |
|
"logits/rejected": -2.6252548694610596, |
|
"logps/chosen": -140.8667449951172, |
|
"logps/rejected": -124.31671142578125, |
|
"loss": 0.001, |
|
"rewards/accuracies": 0.6000000238418579, |
|
"rewards/chosen": 0.010156641714274883, |
|
"rewards/margins": 0.011887853033840656, |
|
"rewards/rejected": -0.0017312124837189913, |
|
"step": 240 |
|
}, |
|
{ |
|
"epoch": 0.5, |
|
"learning_rate": 4.665063509461098e-06, |
|
"logits/chosen": -3.0423521995544434, |
|
"logits/rejected": -2.578011989593506, |
|
"logps/chosen": -136.1217498779297, |
|
"logps/rejected": -116.2592544555664, |
|
"loss": 0.0006, |
|
"rewards/accuracies": 0.75, |
|
"rewards/chosen": 0.022909339517354965, |
|
"rewards/margins": 0.01127886213362217, |
|
"rewards/rejected": 0.011630477383732796, |
|
"step": 250 |
|
}, |
|
{ |
|
"epoch": 0.52, |
|
"learning_rate": 4.620120240391065e-06, |
|
"logits/chosen": -2.8443424701690674, |
|
"logits/rejected": -2.6343538761138916, |
|
"logps/chosen": -100.57234191894531, |
|
"logps/rejected": -98.87772369384766, |
|
"loss": 0.0011, |
|
"rewards/accuracies": 0.75, |
|
"rewards/chosen": 0.010612092912197113, |
|
"rewards/margins": 0.005750484298914671, |
|
"rewards/rejected": 0.004861608147621155, |
|
"step": 260 |
|
}, |
|
{ |
|
"epoch": 0.54, |
|
"learning_rate": 4.572593931387604e-06, |
|
"logits/chosen": -3.0539159774780273, |
|
"logits/rejected": -2.753909111022949, |
|
"logps/chosen": -126.12535095214844, |
|
"logps/rejected": -113.8405990600586, |
|
"loss": 0.0008, |
|
"rewards/accuracies": 0.6499999761581421, |
|
"rewards/chosen": 0.00659831753000617, |
|
"rewards/margins": 0.010872049257159233, |
|
"rewards/rejected": -0.004273730795830488, |
|
"step": 270 |
|
}, |
|
{ |
|
"epoch": 0.56, |
|
"learning_rate": 4.522542485937369e-06, |
|
"logits/chosen": -3.0358662605285645, |
|
"logits/rejected": -2.4368128776550293, |
|
"logps/chosen": -150.52206420898438, |
|
"logps/rejected": -120.9325942993164, |
|
"loss": 0.0008, |
|
"rewards/accuracies": 0.699999988079071, |
|
"rewards/chosen": 0.015789732336997986, |
|
"rewards/margins": 0.017957117408514023, |
|
"rewards/rejected": -0.002167386468499899, |
|
"step": 280 |
|
}, |
|
{ |
|
"epoch": 0.58, |
|
"learning_rate": 4.470026884016805e-06, |
|
"logits/chosen": -3.0055549144744873, |
|
"logits/rejected": -2.710265636444092, |
|
"logps/chosen": -143.09898376464844, |
|
"logps/rejected": -144.5852508544922, |
|
"loss": 0.0014, |
|
"rewards/accuracies": 0.75, |
|
"rewards/chosen": 0.013310904614627361, |
|
"rewards/margins": 0.008920473977923393, |
|
"rewards/rejected": 0.004390430636703968, |
|
"step": 290 |
|
}, |
|
{ |
|
"epoch": 0.6, |
|
"learning_rate": 4.415111107797445e-06, |
|
"logits/chosen": -2.88038969039917, |
|
"logits/rejected": -2.524245500564575, |
|
"logps/chosen": -116.92478942871094, |
|
"logps/rejected": -106.48399353027344, |
|
"loss": 0.001, |
|
"rewards/accuracies": 0.6499999761581421, |
|
"rewards/chosen": -0.016026504337787628, |
|
"rewards/margins": 0.011352911591529846, |
|
"rewards/rejected": -0.027379417791962624, |
|
"step": 300 |
|
}, |
|
{ |
|
"epoch": 0.6, |
|
"eval_logits/chosen": -2.3983988761901855, |
|
"eval_logits/rejected": -2.200815200805664, |
|
"eval_logps/chosen": -269.79119873046875, |
|
"eval_logps/rejected": -250.26023864746094, |
|
"eval_loss": 0.0018744752742350101, |
|
"eval_rewards/accuracies": 0.531000018119812, |
|
"eval_rewards/chosen": -0.012033737264573574, |
|
"eval_rewards/margins": 0.0021730433218181133, |
|
"eval_rewards/rejected": -0.01420677825808525, |
|
"eval_runtime": 1423.8246, |
|
"eval_samples_per_second": 1.405, |
|
"eval_steps_per_second": 0.702, |
|
"step": 300 |
|
}, |
|
{ |
|
"epoch": 0.62, |
|
"learning_rate": 4.357862063693486e-06, |
|
"logits/chosen": -3.0442090034484863, |
|
"logits/rejected": -2.7478137016296387, |
|
"logps/chosen": -200.55148315429688, |
|
"logps/rejected": -198.62619018554688, |
|
"loss": 0.001, |
|
"rewards/accuracies": 0.550000011920929, |
|
"rewards/chosen": -0.01667696237564087, |
|
"rewards/margins": 0.007582807447761297, |
|
"rewards/rejected": -0.024259772151708603, |
|
"step": 310 |
|
}, |
|
{ |
|
"epoch": 0.64, |
|
"learning_rate": 4.2983495008466285e-06, |
|
"logits/chosen": -2.8576464653015137, |
|
"logits/rejected": -2.5876190662384033, |
|
"logps/chosen": -129.97561645507812, |
|
"logps/rejected": -118.2795181274414, |
|
"loss": 0.0013, |
|
"rewards/accuracies": 0.6000000238418579, |
|
"rewards/chosen": -0.01680707558989525, |
|
"rewards/margins": 0.00610563438385725, |
|
"rewards/rejected": -0.022912709042429924, |
|
"step": 320 |
|
}, |
|
{ |
|
"epoch": 0.66, |
|
"learning_rate": 4.236645926147493e-06, |
|
"logits/chosen": -3.026426315307617, |
|
"logits/rejected": -2.7503247261047363, |
|
"logps/chosen": -146.84666442871094, |
|
"logps/rejected": -139.24612426757812, |
|
"loss": 0.001, |
|
"rewards/accuracies": 0.6499999761581421, |
|
"rewards/chosen": 0.014562124386429787, |
|
"rewards/margins": 0.005175650119781494, |
|
"rewards/rejected": 0.009386474266648293, |
|
"step": 330 |
|
}, |
|
{ |
|
"epoch": 0.68, |
|
"learning_rate": 4.172826515897146e-06, |
|
"logits/chosen": -3.144963502883911, |
|
"logits/rejected": -2.831359624862671, |
|
"logps/chosen": -169.22183227539062, |
|
"logps/rejected": -168.03089904785156, |
|
"loss": 0.0012, |
|
"rewards/accuracies": 0.44999998807907104, |
|
"rewards/chosen": 0.004123937338590622, |
|
"rewards/margins": 0.0042907195165753365, |
|
"rewards/rejected": -0.00016678198880981654, |
|
"step": 340 |
|
}, |
|
{ |
|
"epoch": 0.7, |
|
"learning_rate": 4.106969024216348e-06, |
|
"logits/chosen": -3.0851778984069824, |
|
"logits/rejected": -2.769751787185669, |
|
"logps/chosen": -130.99282836914062, |
|
"logps/rejected": -126.33982849121094, |
|
"loss": 0.0012, |
|
"rewards/accuracies": 0.699999988079071, |
|
"rewards/chosen": 0.007414456456899643, |
|
"rewards/margins": 0.006272236816585064, |
|
"rewards/rejected": 0.0011422175448387861, |
|
"step": 350 |
|
}, |
|
{ |
|
"epoch": 0.72, |
|
"learning_rate": 4.039153688314146e-06, |
|
"logits/chosen": -3.0202767848968506, |
|
"logits/rejected": -2.817288637161255, |
|
"logps/chosen": -201.23435974121094, |
|
"logps/rejected": -194.98385620117188, |
|
"loss": 0.0021, |
|
"rewards/accuracies": 0.4000000059604645, |
|
"rewards/chosen": 0.04826142266392708, |
|
"rewards/margins": -0.002473982283845544, |
|
"rewards/rejected": 0.05073540285229683, |
|
"step": 360 |
|
}, |
|
{ |
|
"epoch": 0.74, |
|
"learning_rate": 3.969463130731183e-06, |
|
"logits/chosen": -2.9788031578063965, |
|
"logits/rejected": -2.5181713104248047, |
|
"logps/chosen": -172.3706817626953, |
|
"logps/rejected": -149.59950256347656, |
|
"loss": 0.0009, |
|
"rewards/accuracies": 0.75, |
|
"rewards/chosen": 0.009602314792573452, |
|
"rewards/margins": 0.012693755328655243, |
|
"rewards/rejected": -0.00309143983758986, |
|
"step": 370 |
|
}, |
|
{ |
|
"epoch": 0.76, |
|
"learning_rate": 3.897982258676867e-06, |
|
"logits/chosen": -3.0599074363708496, |
|
"logits/rejected": -2.749642848968506, |
|
"logps/chosen": -216.9517822265625, |
|
"logps/rejected": -202.60659790039062, |
|
"loss": 0.0012, |
|
"rewards/accuracies": 0.699999988079071, |
|
"rewards/chosen": -0.008673595264554024, |
|
"rewards/margins": 0.00854658242315054, |
|
"rewards/rejected": -0.01722017675638199, |
|
"step": 380 |
|
}, |
|
{ |
|
"epoch": 0.78, |
|
"learning_rate": 3.824798160583012e-06, |
|
"logits/chosen": -2.6857285499572754, |
|
"logits/rejected": -2.262728214263916, |
|
"logps/chosen": -133.64395141601562, |
|
"logps/rejected": -119.55937194824219, |
|
"loss": 0.0009, |
|
"rewards/accuracies": 0.6000000238418579, |
|
"rewards/chosen": 0.020837822929024696, |
|
"rewards/margins": 0.01239033229649067, |
|
"rewards/rejected": 0.008447489701211452, |
|
"step": 390 |
|
}, |
|
{ |
|
"epoch": 0.8, |
|
"learning_rate": 3.7500000000000005e-06, |
|
"logits/chosen": -2.9022040367126465, |
|
"logits/rejected": -2.7616379261016846, |
|
"logps/chosen": -185.07240295410156, |
|
"logps/rejected": -179.7260284423828, |
|
"loss": 0.0011, |
|
"rewards/accuracies": 0.44999998807907104, |
|
"rewards/chosen": 0.003294271184131503, |
|
"rewards/margins": 0.0035428921692073345, |
|
"rewards/rejected": -0.0002486211305949837, |
|
"step": 400 |
|
}, |
|
{ |
|
"epoch": 0.8, |
|
"eval_logits/chosen": -2.396838426589966, |
|
"eval_logits/rejected": -2.1993439197540283, |
|
"eval_logps/chosen": -270.5950012207031, |
|
"eval_logps/rejected": -250.95413208007812, |
|
"eval_loss": 0.0023364874068647623, |
|
"eval_rewards/accuracies": 0.5009999871253967, |
|
"eval_rewards/chosen": -0.02007202059030533, |
|
"eval_rewards/margins": 0.0010739399585872889, |
|
"eval_rewards/rejected": -0.021145964041352272, |
|
"eval_runtime": 1423.743, |
|
"eval_samples_per_second": 1.405, |
|
"eval_steps_per_second": 0.702, |
|
"step": 400 |
|
}, |
|
{ |
|
"epoch": 0.82, |
|
"learning_rate": 3.6736789069647273e-06, |
|
"logits/chosen": -3.0739264488220215, |
|
"logits/rejected": -2.8628082275390625, |
|
"logps/chosen": -149.10903930664062, |
|
"logps/rejected": -134.99038696289062, |
|
"loss": 0.0011, |
|
"rewards/accuracies": 0.6499999761581421, |
|
"rewards/chosen": 0.008919402956962585, |
|
"rewards/margins": 0.0073748803697526455, |
|
"rewards/rejected": 0.0015445235185325146, |
|
"step": 410 |
|
}, |
|
{ |
|
"epoch": 0.84, |
|
"learning_rate": 3.595927866972694e-06, |
|
"logits/chosen": -3.0844318866729736, |
|
"logits/rejected": -2.8257293701171875, |
|
"logps/chosen": -252.5592498779297, |
|
"logps/rejected": -248.02474975585938, |
|
"loss": 0.001, |
|
"rewards/accuracies": 0.5, |
|
"rewards/chosen": -0.011812982149422169, |
|
"rewards/margins": 0.007170848548412323, |
|
"rewards/rejected": -0.018983829766511917, |
|
"step": 420 |
|
}, |
|
{ |
|
"epoch": 0.86, |
|
"learning_rate": 3.516841607689501e-06, |
|
"logits/chosen": -3.055979013442993, |
|
"logits/rejected": -2.7850348949432373, |
|
"logps/chosen": -147.57447814941406, |
|
"logps/rejected": -138.53738403320312, |
|
"loss": 0.0009, |
|
"rewards/accuracies": 0.550000011920929, |
|
"rewards/chosen": -0.005650115665048361, |
|
"rewards/margins": 0.012325585819780827, |
|
"rewards/rejected": -0.0179757010191679, |
|
"step": 430 |
|
}, |
|
{ |
|
"epoch": 0.88, |
|
"learning_rate": 3.436516483539781e-06, |
|
"logits/chosen": -2.7994961738586426, |
|
"logits/rejected": -2.700761079788208, |
|
"logps/chosen": -189.2479248046875, |
|
"logps/rejected": -180.1698760986328, |
|
"loss": 0.0016, |
|
"rewards/accuracies": 0.5, |
|
"rewards/chosen": -0.003194189164787531, |
|
"rewards/margins": -0.00013344390026759356, |
|
"rewards/rejected": -0.0030607457738369703, |
|
"step": 440 |
|
}, |
|
{ |
|
"epoch": 0.9, |
|
"learning_rate": 3.3550503583141726e-06, |
|
"logits/chosen": -3.072540521621704, |
|
"logits/rejected": -2.8204236030578613, |
|
"logps/chosen": -133.08212280273438, |
|
"logps/rejected": -119.2376480102539, |
|
"loss": 0.0009, |
|
"rewards/accuracies": 0.699999988079071, |
|
"rewards/chosen": -0.00826738215982914, |
|
"rewards/margins": 0.011625150218605995, |
|
"rewards/rejected": -0.019892532378435135, |
|
"step": 450 |
|
}, |
|
{ |
|
"epoch": 0.92, |
|
"learning_rate": 3.272542485937369e-06, |
|
"logits/chosen": -2.9859511852264404, |
|
"logits/rejected": -2.7228169441223145, |
|
"logps/chosen": -211.18002319335938, |
|
"logps/rejected": -202.37867736816406, |
|
"loss": 0.0009, |
|
"rewards/accuracies": 0.6000000238418579, |
|
"rewards/chosen": -0.00889207050204277, |
|
"rewards/margins": 0.014142923057079315, |
|
"rewards/rejected": -0.023034993559122086, |
|
"step": 460 |
|
}, |
|
{ |
|
"epoch": 0.94, |
|
"learning_rate": 3.189093389542498e-06, |
|
"logits/chosen": -2.9243226051330566, |
|
"logits/rejected": -2.708249092102051, |
|
"logps/chosen": -178.67819213867188, |
|
"logps/rejected": -162.6269073486328, |
|
"loss": 0.0013, |
|
"rewards/accuracies": 0.6000000238418579, |
|
"rewards/chosen": 0.008272857405245304, |
|
"rewards/margins": 0.005242692772299051, |
|
"rewards/rejected": 0.0030301641672849655, |
|
"step": 470 |
|
}, |
|
{ |
|
"epoch": 0.96, |
|
"learning_rate": 3.1048047389991693e-06, |
|
"logits/chosen": -3.1769165992736816, |
|
"logits/rejected": -2.587887763977051, |
|
"logps/chosen": -163.45852661132812, |
|
"logps/rejected": -134.19691467285156, |
|
"loss": 0.0012, |
|
"rewards/accuracies": 0.550000011920929, |
|
"rewards/chosen": -0.0025352207012474537, |
|
"rewards/margins": 0.014497722499072552, |
|
"rewards/rejected": -0.017032943665981293, |
|
"step": 480 |
|
}, |
|
{ |
|
"epoch": 0.98, |
|
"learning_rate": 3.019779227044398e-06, |
|
"logits/chosen": -3.0145325660705566, |
|
"logits/rejected": -2.7123026847839355, |
|
"logps/chosen": -167.49745178222656, |
|
"logps/rejected": -152.8248748779297, |
|
"loss": 0.0012, |
|
"rewards/accuracies": 0.6000000238418579, |
|
"rewards/chosen": 0.01072392426431179, |
|
"rewards/margins": 0.01687874272465706, |
|
"rewards/rejected": -0.006154821254312992, |
|
"step": 490 |
|
}, |
|
{ |
|
"epoch": 1.0, |
|
"learning_rate": 2.9341204441673267e-06, |
|
"logits/chosen": -3.132664918899536, |
|
"logits/rejected": -2.911961317062378, |
|
"logps/chosen": -108.4159927368164, |
|
"logps/rejected": -94.57585144042969, |
|
"loss": 0.0008, |
|
"rewards/accuracies": 0.6499999761581421, |
|
"rewards/chosen": 0.0020372807048261166, |
|
"rewards/margins": 0.008723934181034565, |
|
"rewards/rejected": -0.006686653010547161, |
|
"step": 500 |
|
}, |
|
{ |
|
"epoch": 1.0, |
|
"eval_logits/chosen": -2.395993947982788, |
|
"eval_logits/rejected": -2.1981606483459473, |
|
"eval_logps/chosen": -270.28497314453125, |
|
"eval_logps/rejected": -250.72601318359375, |
|
"eval_loss": 0.002102552680298686, |
|
"eval_rewards/accuracies": 0.5065000057220459, |
|
"eval_rewards/chosen": -0.016971532255411148, |
|
"eval_rewards/margins": 0.0018933486426249146, |
|
"eval_rewards/rejected": -0.018864883109927177, |
|
"eval_runtime": 1423.5769, |
|
"eval_samples_per_second": 1.405, |
|
"eval_steps_per_second": 0.702, |
|
"step": 500 |
|
}, |
|
{ |
|
"epoch": 1.02, |
|
"learning_rate": 2.847932752400164e-06, |
|
"logits/chosen": -3.008751392364502, |
|
"logits/rejected": -2.6613450050354004, |
|
"logps/chosen": -123.0440444946289, |
|
"logps/rejected": -122.49776458740234, |
|
"loss": 0.024, |
|
"rewards/accuracies": 0.8999999761581421, |
|
"rewards/chosen": 0.06933443248271942, |
|
"rewards/margins": 0.12685301899909973, |
|
"rewards/rejected": -0.05751859024167061, |
|
"step": 510 |
|
}, |
|
{ |
|
"epoch": 1.04, |
|
"learning_rate": 2.761321158169134e-06, |
|
"logits/chosen": -2.982330322265625, |
|
"logits/rejected": -2.6290860176086426, |
|
"logps/chosen": -207.28955078125, |
|
"logps/rejected": -207.44711303710938, |
|
"loss": 0.0169, |
|
"rewards/accuracies": 0.8500000238418579, |
|
"rewards/chosen": 0.023494327440857887, |
|
"rewards/margins": 0.059116560965776443, |
|
"rewards/rejected": -0.035622235387563705, |
|
"step": 520 |
|
}, |
|
{ |
|
"epoch": 1.06, |
|
"learning_rate": 2.6743911843603134e-06, |
|
"logits/chosen": -3.140193223953247, |
|
"logits/rejected": -2.7758877277374268, |
|
"logps/chosen": -161.1981658935547, |
|
"logps/rejected": -160.78762817382812, |
|
"loss": 0.0854, |
|
"rewards/accuracies": 0.8500000238418579, |
|
"rewards/chosen": 0.08391492813825607, |
|
"rewards/margins": 0.21972373127937317, |
|
"rewards/rejected": -0.1358087956905365, |
|
"step": 530 |
|
}, |
|
{ |
|
"epoch": 1.08, |
|
"learning_rate": 2.587248741756253e-06, |
|
"logits/chosen": -2.866288661956787, |
|
"logits/rejected": -2.557806968688965, |
|
"logps/chosen": -229.82077026367188, |
|
"logps/rejected": -227.31967163085938, |
|
"loss": 0.0241, |
|
"rewards/accuracies": 0.75, |
|
"rewards/chosen": -0.0034355625975877047, |
|
"rewards/margins": 0.05683975666761398, |
|
"rewards/rejected": -0.06027532368898392, |
|
"step": 540 |
|
}, |
|
{ |
|
"epoch": 1.1, |
|
"learning_rate": 2.5e-06, |
|
"logits/chosen": -2.7409634590148926, |
|
"logits/rejected": -2.412743330001831, |
|
"logps/chosen": -110.87519836425781, |
|
"logps/rejected": -106.71611022949219, |
|
"loss": 0.0099, |
|
"rewards/accuracies": 0.75, |
|
"rewards/chosen": 0.004647939465939999, |
|
"rewards/margins": 0.07160676270723343, |
|
"rewards/rejected": -0.06695882230997086, |
|
"step": 550 |
|
}, |
|
{ |
|
"epoch": 1.12, |
|
"learning_rate": 2.4127512582437486e-06, |
|
"logits/chosen": -3.080806255340576, |
|
"logits/rejected": -2.819638729095459, |
|
"logps/chosen": -138.67153930664062, |
|
"logps/rejected": -144.43199157714844, |
|
"loss": 0.0542, |
|
"rewards/accuracies": 0.8999999761581421, |
|
"rewards/chosen": 0.025073865428566933, |
|
"rewards/margins": 0.14305388927459717, |
|
"rewards/rejected": -0.11798002570867538, |
|
"step": 560 |
|
}, |
|
{ |
|
"epoch": 1.14, |
|
"learning_rate": 2.325608815639687e-06, |
|
"logits/chosen": -3.0271854400634766, |
|
"logits/rejected": -2.7436680793762207, |
|
"logps/chosen": -174.3506317138672, |
|
"logps/rejected": -171.3226776123047, |
|
"loss": 0.0444, |
|
"rewards/accuracies": 0.800000011920929, |
|
"rewards/chosen": 0.034181032329797745, |
|
"rewards/margins": 0.1548537313938141, |
|
"rewards/rejected": -0.12067268788814545, |
|
"step": 570 |
|
}, |
|
{ |
|
"epoch": 1.16, |
|
"learning_rate": 2.238678841830867e-06, |
|
"logits/chosen": -2.6729888916015625, |
|
"logits/rejected": -2.449924945831299, |
|
"logps/chosen": -143.39756774902344, |
|
"logps/rejected": -147.48681640625, |
|
"loss": 0.0234, |
|
"rewards/accuracies": 0.800000011920929, |
|
"rewards/chosen": 0.00795525498688221, |
|
"rewards/margins": 0.11490567773580551, |
|
"rewards/rejected": -0.10695041716098785, |
|
"step": 580 |
|
}, |
|
{ |
|
"epoch": 1.18, |
|
"learning_rate": 2.1520672475998374e-06, |
|
"logits/chosen": -2.940767526626587, |
|
"logits/rejected": -2.584683895111084, |
|
"logps/chosen": -128.87655639648438, |
|
"logps/rejected": -120.95220947265625, |
|
"loss": 0.0663, |
|
"rewards/accuracies": 0.6000000238418579, |
|
"rewards/chosen": -0.025388438254594803, |
|
"rewards/margins": 0.09438282996416092, |
|
"rewards/rejected": -0.11977125704288483, |
|
"step": 590 |
|
}, |
|
{ |
|
"epoch": 1.2, |
|
"learning_rate": 2.0658795558326745e-06, |
|
"logits/chosen": -2.8861563205718994, |
|
"logits/rejected": -2.5959839820861816, |
|
"logps/chosen": -124.5409927368164, |
|
"logps/rejected": -133.5659637451172, |
|
"loss": 0.044, |
|
"rewards/accuracies": 0.75, |
|
"rewards/chosen": -0.0006943264743313193, |
|
"rewards/margins": 0.09414149820804596, |
|
"rewards/rejected": -0.0948358103632927, |
|
"step": 600 |
|
}, |
|
{ |
|
"epoch": 1.2, |
|
"eval_logits/chosen": -2.389890670776367, |
|
"eval_logits/rejected": -2.1939876079559326, |
|
"eval_logps/chosen": -269.1193542480469, |
|
"eval_logps/rejected": -246.85476684570312, |
|
"eval_loss": 0.009110072627663612, |
|
"eval_rewards/accuracies": 0.36000001430511475, |
|
"eval_rewards/chosen": -0.005315269809216261, |
|
"eval_rewards/margins": -0.025163182988762856, |
|
"eval_rewards/rejected": 0.019847916439175606, |
|
"eval_runtime": 1422.5682, |
|
"eval_samples_per_second": 1.406, |
|
"eval_steps_per_second": 0.703, |
|
"step": 600 |
|
}, |
|
{ |
|
"epoch": 1.22, |
|
"learning_rate": 1.9802207729556023e-06, |
|
"logits/chosen": -2.90655779838562, |
|
"logits/rejected": -2.578254222869873, |
|
"logps/chosen": -182.38720703125, |
|
"logps/rejected": -175.3732147216797, |
|
"loss": 0.0322, |
|
"rewards/accuracies": 0.6499999761581421, |
|
"rewards/chosen": -0.02949387952685356, |
|
"rewards/margins": 0.04020578786730766, |
|
"rewards/rejected": -0.06969965994358063, |
|
"step": 610 |
|
}, |
|
{ |
|
"epoch": 1.24, |
|
"learning_rate": 1.895195261000831e-06, |
|
"logits/chosen": -3.0672218799591064, |
|
"logits/rejected": -2.651693105697632, |
|
"logps/chosen": -170.82894897460938, |
|
"logps/rejected": -157.90194702148438, |
|
"loss": 0.0163, |
|
"rewards/accuracies": 0.75, |
|
"rewards/chosen": -0.03491836413741112, |
|
"rewards/margins": 0.04328788444399834, |
|
"rewards/rejected": -0.07820625603199005, |
|
"step": 620 |
|
}, |
|
{ |
|
"epoch": 1.26, |
|
"learning_rate": 1.8109066104575023e-06, |
|
"logits/chosen": -3.076021432876587, |
|
"logits/rejected": -2.759376049041748, |
|
"logps/chosen": -130.21498107910156, |
|
"logps/rejected": -134.99252319335938, |
|
"loss": 0.0842, |
|
"rewards/accuracies": 0.75, |
|
"rewards/chosen": 0.011367540806531906, |
|
"rewards/margins": 0.15639443695545197, |
|
"rewards/rejected": -0.14502687752246857, |
|
"step": 630 |
|
}, |
|
{ |
|
"epoch": 1.28, |
|
"learning_rate": 1.7274575140626318e-06, |
|
"logits/chosen": -2.9677953720092773, |
|
"logits/rejected": -2.5879857540130615, |
|
"logps/chosen": -160.40704345703125, |
|
"logps/rejected": -148.29469299316406, |
|
"loss": 0.0238, |
|
"rewards/accuracies": 0.699999988079071, |
|
"rewards/chosen": -0.04639770835638046, |
|
"rewards/margins": 0.028317932039499283, |
|
"rewards/rejected": -0.07471564412117004, |
|
"step": 640 |
|
}, |
|
{ |
|
"epoch": 1.3, |
|
"learning_rate": 1.6449496416858285e-06, |
|
"logits/chosen": -2.873720169067383, |
|
"logits/rejected": -2.657524347305298, |
|
"logps/chosen": -211.8774871826172, |
|
"logps/rejected": -208.0607452392578, |
|
"loss": 0.052, |
|
"rewards/accuracies": 0.75, |
|
"rewards/chosen": -0.05117432028055191, |
|
"rewards/margins": 0.10337792336940765, |
|
"rewards/rejected": -0.15455225110054016, |
|
"step": 650 |
|
}, |
|
{ |
|
"epoch": 1.32, |
|
"learning_rate": 1.56348351646022e-06, |
|
"logits/chosen": -3.1277270317077637, |
|
"logits/rejected": -2.781031370162964, |
|
"logps/chosen": -159.2963104248047, |
|
"logps/rejected": -157.16714477539062, |
|
"loss": 0.0255, |
|
"rewards/accuracies": 0.699999988079071, |
|
"rewards/chosen": -0.08234690129756927, |
|
"rewards/margins": 0.07028084993362427, |
|
"rewards/rejected": -0.15262776613235474, |
|
"step": 660 |
|
}, |
|
{ |
|
"epoch": 1.34, |
|
"learning_rate": 1.4831583923105e-06, |
|
"logits/chosen": -2.9012064933776855, |
|
"logits/rejected": -2.5057435035705566, |
|
"logps/chosen": -133.7654571533203, |
|
"logps/rejected": -123.15315246582031, |
|
"loss": 0.0204, |
|
"rewards/accuracies": 0.5, |
|
"rewards/chosen": -0.06615938246250153, |
|
"rewards/margins": -0.006010846234858036, |
|
"rewards/rejected": -0.060148537158966064, |
|
"step": 670 |
|
}, |
|
{ |
|
"epoch": 1.36, |
|
"learning_rate": 1.4040721330273063e-06, |
|
"logits/chosen": -3.0350639820098877, |
|
"logits/rejected": -2.688361167907715, |
|
"logps/chosen": -161.60037231445312, |
|
"logps/rejected": -148.4673614501953, |
|
"loss": 0.018, |
|
"rewards/accuracies": 0.75, |
|
"rewards/chosen": -0.012395807541906834, |
|
"rewards/margins": 0.07525575160980225, |
|
"rewards/rejected": -0.08765155076980591, |
|
"step": 680 |
|
}, |
|
{ |
|
"epoch": 1.38, |
|
"learning_rate": 1.3263210930352737e-06, |
|
"logits/chosen": -2.987074613571167, |
|
"logits/rejected": -2.41349458694458, |
|
"logps/chosen": -155.8392333984375, |
|
"logps/rejected": -120.13862609863281, |
|
"loss": 0.1358, |
|
"rewards/accuracies": 0.6499999761581421, |
|
"rewards/chosen": -0.06855552643537521, |
|
"rewards/margins": 0.0165361650288105, |
|
"rewards/rejected": -0.08509168773889542, |
|
"step": 690 |
|
}, |
|
{ |
|
"epoch": 1.4, |
|
"learning_rate": 1.2500000000000007e-06, |
|
"logits/chosen": -3.142533302307129, |
|
"logits/rejected": -2.7605295181274414, |
|
"logps/chosen": -196.124267578125, |
|
"logps/rejected": -175.75848388671875, |
|
"loss": 0.0682, |
|
"rewards/accuracies": 0.550000011920929, |
|
"rewards/chosen": -0.08456405252218246, |
|
"rewards/margins": 0.07064096629619598, |
|
"rewards/rejected": -0.15520504117012024, |
|
"step": 700 |
|
}, |
|
{ |
|
"epoch": 1.4, |
|
"eval_logits/chosen": -2.399232864379883, |
|
"eval_logits/rejected": -2.203479528427124, |
|
"eval_logps/chosen": -272.0422668457031, |
|
"eval_logps/rejected": -247.98184204101562, |
|
"eval_loss": 0.019056277349591255, |
|
"eval_rewards/accuracies": 0.3449999988079071, |
|
"eval_rewards/chosen": -0.03454471752047539, |
|
"eval_rewards/margins": -0.043121837079524994, |
|
"eval_rewards/rejected": 0.008577119559049606, |
|
"eval_runtime": 1422.244, |
|
"eval_samples_per_second": 1.406, |
|
"eval_steps_per_second": 0.703, |
|
"step": 700 |
|
}, |
|
{ |
|
"epoch": 1.42, |
|
"learning_rate": 1.1752018394169882e-06, |
|
"logits/chosen": -2.923161029815674, |
|
"logits/rejected": -2.7423832416534424, |
|
"logps/chosen": -232.46353149414062, |
|
"logps/rejected": -236.95712280273438, |
|
"loss": 0.1005, |
|
"rewards/accuracies": 0.6499999761581421, |
|
"rewards/chosen": -0.058026909828186035, |
|
"rewards/margins": 0.11247865855693817, |
|
"rewards/rejected": -0.170505553483963, |
|
"step": 710 |
|
}, |
|
{ |
|
"epoch": 1.44, |
|
"learning_rate": 1.1020177413231334e-06, |
|
"logits/chosen": -2.9437034130096436, |
|
"logits/rejected": -2.7746634483337402, |
|
"logps/chosen": -182.21336364746094, |
|
"logps/rejected": -170.1109619140625, |
|
"loss": 0.0119, |
|
"rewards/accuracies": 0.550000011920929, |
|
"rewards/chosen": -0.009687254205346107, |
|
"rewards/margins": 0.010108504444360733, |
|
"rewards/rejected": -0.01979575864970684, |
|
"step": 720 |
|
}, |
|
{ |
|
"epoch": 1.46, |
|
"learning_rate": 1.0305368692688175e-06, |
|
"logits/chosen": -2.91925048828125, |
|
"logits/rejected": -2.4575612545013428, |
|
"logps/chosen": -141.11546325683594, |
|
"logps/rejected": -132.57675170898438, |
|
"loss": 0.0074, |
|
"rewards/accuracies": 0.550000011920929, |
|
"rewards/chosen": -0.058561988174915314, |
|
"rewards/margins": 0.03072230890393257, |
|
"rewards/rejected": -0.08928428590297699, |
|
"step": 730 |
|
}, |
|
{ |
|
"epoch": 1.48, |
|
"learning_rate": 9.608463116858544e-07, |
|
"logits/chosen": -2.8368327617645264, |
|
"logits/rejected": -2.523725986480713, |
|
"logps/chosen": -102.6483383178711, |
|
"logps/rejected": -94.72925567626953, |
|
"loss": 0.0083, |
|
"rewards/accuracies": 0.699999988079071, |
|
"rewards/chosen": -0.0365123450756073, |
|
"rewards/margins": 0.021143099293112755, |
|
"rewards/rejected": -0.057655446231365204, |
|
"step": 740 |
|
}, |
|
{ |
|
"epoch": 1.5, |
|
"learning_rate": 8.930309757836517e-07, |
|
"logits/chosen": -2.9483554363250732, |
|
"logits/rejected": -2.4691848754882812, |
|
"logps/chosen": -170.67236328125, |
|
"logps/rejected": -154.7274932861328, |
|
"loss": 0.0595, |
|
"rewards/accuracies": 0.550000011920929, |
|
"rewards/chosen": -0.09041354060173035, |
|
"rewards/margins": 0.019873833283782005, |
|
"rewards/rejected": -0.1102873831987381, |
|
"step": 750 |
|
}, |
|
{ |
|
"epoch": 1.52, |
|
"learning_rate": 8.271734841028553e-07, |
|
"logits/chosen": -2.998615264892578, |
|
"logits/rejected": -2.765198230743408, |
|
"logps/chosen": -178.57168579101562, |
|
"logps/rejected": -165.0284423828125, |
|
"loss": 0.0603, |
|
"rewards/accuracies": 0.6499999761581421, |
|
"rewards/chosen": -0.07312550395727158, |
|
"rewards/margins": 0.03924999386072159, |
|
"rewards/rejected": -0.11237549781799316, |
|
"step": 760 |
|
}, |
|
{ |
|
"epoch": 1.54, |
|
"learning_rate": 7.633540738525066e-07, |
|
"logits/chosen": -2.9713294506073, |
|
"logits/rejected": -2.581371784210205, |
|
"logps/chosen": -188.65562438964844, |
|
"logps/rejected": -198.20057678222656, |
|
"loss": 0.0376, |
|
"rewards/accuracies": 0.800000011920929, |
|
"rewards/chosen": -0.08031792938709259, |
|
"rewards/margins": 0.0691676214337349, |
|
"rewards/rejected": -0.1494855433702469, |
|
"step": 770 |
|
}, |
|
{ |
|
"epoch": 1.56, |
|
"learning_rate": 7.016504991533727e-07, |
|
"logits/chosen": -3.094494342803955, |
|
"logits/rejected": -2.8429107666015625, |
|
"logps/chosen": -172.08921813964844, |
|
"logps/rejected": -168.11550903320312, |
|
"loss": 0.0146, |
|
"rewards/accuracies": 0.800000011920929, |
|
"rewards/chosen": -0.03326564282178879, |
|
"rewards/margins": 0.049937326461076736, |
|
"rewards/rejected": -0.08320296555757523, |
|
"step": 780 |
|
}, |
|
{ |
|
"epoch": 1.58, |
|
"learning_rate": 6.421379363065142e-07, |
|
"logits/chosen": -2.8603293895721436, |
|
"logits/rejected": -2.629847288131714, |
|
"logps/chosen": -187.458251953125, |
|
"logps/rejected": -181.56585693359375, |
|
"loss": 0.0346, |
|
"rewards/accuracies": 0.699999988079071, |
|
"rewards/chosen": -0.026567768305540085, |
|
"rewards/margins": 0.08442084491252899, |
|
"rewards/rejected": -0.11098861694335938, |
|
"step": 790 |
|
}, |
|
{ |
|
"epoch": 1.6, |
|
"learning_rate": 5.848888922025553e-07, |
|
"logits/chosen": -3.0436525344848633, |
|
"logits/rejected": -2.855353355407715, |
|
"logps/chosen": -116.41280364990234, |
|
"logps/rejected": -118.14847564697266, |
|
"loss": 0.0505, |
|
"rewards/accuracies": 0.8500000238418579, |
|
"rewards/chosen": 0.013444220647215843, |
|
"rewards/margins": 0.11258585751056671, |
|
"rewards/rejected": -0.09914163500070572, |
|
"step": 800 |
|
}, |
|
{ |
|
"epoch": 1.6, |
|
"eval_logits/chosen": -2.4056026935577393, |
|
"eval_logits/rejected": -2.209423780441284, |
|
"eval_logps/chosen": -273.5587158203125, |
|
"eval_logps/rejected": -248.85421752929688, |
|
"eval_loss": 0.023677825927734375, |
|
"eval_rewards/accuracies": 0.34049999713897705, |
|
"eval_rewards/chosen": -0.049709003418684006, |
|
"eval_rewards/margins": -0.04956228658556938, |
|
"eval_rewards/rejected": -0.0001467149268137291, |
|
"eval_runtime": 1422.3272, |
|
"eval_samples_per_second": 1.406, |
|
"eval_steps_per_second": 0.703, |
|
"step": 800 |
|
}, |
|
{ |
|
"epoch": 1.62, |
|
"learning_rate": 5.299731159831953e-07, |
|
"logits/chosen": -2.867088794708252, |
|
"logits/rejected": -2.544645309448242, |
|
"logps/chosen": -144.49082946777344, |
|
"logps/rejected": -137.3140106201172, |
|
"loss": 0.0169, |
|
"rewards/accuracies": 0.6499999761581421, |
|
"rewards/chosen": -0.05922747775912285, |
|
"rewards/margins": 0.03063639998435974, |
|
"rewards/rejected": -0.08986388146877289, |
|
"step": 810 |
|
}, |
|
{ |
|
"epoch": 1.64, |
|
"learning_rate": 4.774575140626317e-07, |
|
"logits/chosen": -2.855769634246826, |
|
"logits/rejected": -2.6003212928771973, |
|
"logps/chosen": -149.01148986816406, |
|
"logps/rejected": -150.924072265625, |
|
"loss": 0.0045, |
|
"rewards/accuracies": 0.44999998807907104, |
|
"rewards/chosen": -0.0036942525766789913, |
|
"rewards/margins": 0.02296125330030918, |
|
"rewards/rejected": -0.02665550634264946, |
|
"step": 820 |
|
}, |
|
{ |
|
"epoch": 1.66, |
|
"learning_rate": 4.27406068612396e-07, |
|
"logits/chosen": -3.0886871814727783, |
|
"logits/rejected": -2.7754549980163574, |
|
"logps/chosen": -159.57974243164062, |
|
"logps/rejected": -160.63307189941406, |
|
"loss": 0.0227, |
|
"rewards/accuracies": 0.800000011920929, |
|
"rewards/chosen": -0.05405544117093086, |
|
"rewards/margins": 0.0963703840970993, |
|
"rewards/rejected": -0.15042582154273987, |
|
"step": 830 |
|
}, |
|
{ |
|
"epoch": 1.68, |
|
"learning_rate": 3.798797596089351e-07, |
|
"logits/chosen": -3.102863073348999, |
|
"logits/rejected": -2.912782669067383, |
|
"logps/chosen": -148.99937438964844, |
|
"logps/rejected": -143.40872192382812, |
|
"loss": 0.0379, |
|
"rewards/accuracies": 0.800000011920929, |
|
"rewards/chosen": -0.06271068751811981, |
|
"rewards/margins": 0.025210902094841003, |
|
"rewards/rejected": -0.08792158961296082, |
|
"step": 840 |
|
}, |
|
{ |
|
"epoch": 1.7, |
|
"learning_rate": 3.3493649053890325e-07, |
|
"logits/chosen": -2.9544389247894287, |
|
"logits/rejected": -2.6804440021514893, |
|
"logps/chosen": -142.18478393554688, |
|
"logps/rejected": -136.8746795654297, |
|
"loss": 0.0061, |
|
"rewards/accuracies": 0.699999988079071, |
|
"rewards/chosen": -0.008848746307194233, |
|
"rewards/margins": 0.052127204835414886, |
|
"rewards/rejected": -0.06097595766186714, |
|
"step": 850 |
|
}, |
|
{ |
|
"epoch": 1.72, |
|
"learning_rate": 2.9263101785268253e-07, |
|
"logits/chosen": -2.970134735107422, |
|
"logits/rejected": -2.784672737121582, |
|
"logps/chosen": -122.1519775390625, |
|
"logps/rejected": -113.82493591308594, |
|
"loss": 0.0044, |
|
"rewards/accuracies": 0.6000000238418579, |
|
"rewards/chosen": -0.058860231190919876, |
|
"rewards/margins": 0.03174171224236488, |
|
"rewards/rejected": -0.09060193598270416, |
|
"step": 860 |
|
}, |
|
{ |
|
"epoch": 1.74, |
|
"learning_rate": 2.53014884252083e-07, |
|
"logits/chosen": -2.949235200881958, |
|
"logits/rejected": -2.5656838417053223, |
|
"logps/chosen": -258.15875244140625, |
|
"logps/rejected": -249.7416534423828, |
|
"loss": 0.091, |
|
"rewards/accuracies": 0.5, |
|
"rewards/chosen": -0.05386580899357796, |
|
"rewards/margins": 0.09710750728845596, |
|
"rewards/rejected": -0.15097330510616302, |
|
"step": 870 |
|
}, |
|
{ |
|
"epoch": 1.76, |
|
"learning_rate": 2.1613635589349756e-07, |
|
"logits/chosen": -3.1699745655059814, |
|
"logits/rejected": -2.638118028640747, |
|
"logps/chosen": -151.49009704589844, |
|
"logps/rejected": -145.30239868164062, |
|
"loss": 0.0532, |
|
"rewards/accuracies": 0.75, |
|
"rewards/chosen": -0.06000872328877449, |
|
"rewards/margins": 0.11070142686367035, |
|
"rewards/rejected": -0.17071016132831573, |
|
"step": 880 |
|
}, |
|
{ |
|
"epoch": 1.78, |
|
"learning_rate": 1.8204036358303173e-07, |
|
"logits/chosen": -2.985215663909912, |
|
"logits/rejected": -2.570612668991089, |
|
"logps/chosen": -109.09466552734375, |
|
"logps/rejected": -98.76487731933594, |
|
"loss": 0.0108, |
|
"rewards/accuracies": 0.699999988079071, |
|
"rewards/chosen": -0.0210666935890913, |
|
"rewards/margins": 0.044059909880161285, |
|
"rewards/rejected": -0.06512660533189774, |
|
"step": 890 |
|
}, |
|
{ |
|
"epoch": 1.8, |
|
"learning_rate": 1.507684480352292e-07, |
|
"logits/chosen": -3.0888664722442627, |
|
"logits/rejected": -2.6895575523376465, |
|
"logps/chosen": -163.73812866210938, |
|
"logps/rejected": -161.14007568359375, |
|
"loss": 0.0243, |
|
"rewards/accuracies": 0.5, |
|
"rewards/chosen": -0.08492692559957504, |
|
"rewards/margins": 0.025812974199652672, |
|
"rewards/rejected": -0.11073990166187286, |
|
"step": 900 |
|
}, |
|
{ |
|
"epoch": 1.8, |
|
"eval_logits/chosen": -2.4080586433410645, |
|
"eval_logits/rejected": -2.2116518020629883, |
|
"eval_logps/chosen": -274.3966979980469, |
|
"eval_logps/rejected": -249.45701599121094, |
|
"eval_loss": 0.025878531858325005, |
|
"eval_rewards/accuracies": 0.33399999141693115, |
|
"eval_rewards/chosen": -0.058088988065719604, |
|
"eval_rewards/margins": -0.05191420391201973, |
|
"eval_rewards/rejected": -0.00617477810010314, |
|
"eval_runtime": 1421.9219, |
|
"eval_samples_per_second": 1.407, |
|
"eval_steps_per_second": 0.703, |
|
"step": 900 |
|
}, |
|
{ |
|
"epoch": 1.82, |
|
"learning_rate": 1.223587092621162e-07, |
|
"logits/chosen": -3.060760259628296, |
|
"logits/rejected": -2.724635124206543, |
|
"logps/chosen": -126.0656509399414, |
|
"logps/rejected": -112.5301742553711, |
|
"loss": 0.0271, |
|
"rewards/accuracies": 0.800000011920929, |
|
"rewards/chosen": -0.014755621552467346, |
|
"rewards/margins": 0.10490630567073822, |
|
"rewards/rejected": -0.11966194212436676, |
|
"step": 910 |
|
}, |
|
{ |
|
"epoch": 1.84, |
|
"learning_rate": 9.684576015420277e-08, |
|
"logits/chosen": -2.515842914581299, |
|
"logits/rejected": -1.9269781112670898, |
|
"logps/chosen": -214.9114532470703, |
|
"logps/rejected": -191.04974365234375, |
|
"loss": 0.0234, |
|
"rewards/accuracies": 0.44999998807907104, |
|
"rewards/chosen": -0.11193618923425674, |
|
"rewards/margins": 0.01120082102715969, |
|
"rewards/rejected": -0.12313701212406158, |
|
"step": 920 |
|
}, |
|
{ |
|
"epoch": 1.86, |
|
"learning_rate": 7.426068431000883e-08, |
|
"logits/chosen": -3.096656322479248, |
|
"logits/rejected": -2.6982362270355225, |
|
"logps/chosen": -156.7047576904297, |
|
"logps/rejected": -140.3162841796875, |
|
"loss": 0.0558, |
|
"rewards/accuracies": 0.6000000238418579, |
|
"rewards/chosen": -0.1117779016494751, |
|
"rewards/margins": 0.010117399506270885, |
|
"rewards/rejected": -0.12189529836177826, |
|
"step": 930 |
|
}, |
|
{ |
|
"epoch": 1.88, |
|
"learning_rate": 5.463099816548578e-08, |
|
"logits/chosen": -2.985032558441162, |
|
"logits/rejected": -2.7575299739837646, |
|
"logps/chosen": -217.82192993164062, |
|
"logps/rejected": -225.7892303466797, |
|
"loss": 0.0701, |
|
"rewards/accuracies": 0.6000000238418579, |
|
"rewards/chosen": -0.16152246296405792, |
|
"rewards/margins": 0.10506312549114227, |
|
"rewards/rejected": -0.2665855586528778, |
|
"step": 940 |
|
}, |
|
{ |
|
"epoch": 1.9, |
|
"learning_rate": 3.798061746947995e-08, |
|
"logits/chosen": -2.925285816192627, |
|
"logits/rejected": -2.7017900943756104, |
|
"logps/chosen": -161.64578247070312, |
|
"logps/rejected": -155.53384399414062, |
|
"loss": 0.0125, |
|
"rewards/accuracies": 0.6499999761581421, |
|
"rewards/chosen": -0.04954836145043373, |
|
"rewards/margins": 0.03911132365465164, |
|
"rewards/rejected": -0.08865968883037567, |
|
"step": 950 |
|
}, |
|
{ |
|
"epoch": 1.92, |
|
"learning_rate": 2.4329828146074096e-08, |
|
"logits/chosen": -3.1424098014831543, |
|
"logits/rejected": -2.879060745239258, |
|
"logps/chosen": -189.52865600585938, |
|
"logps/rejected": -195.4296875, |
|
"loss": 0.0599, |
|
"rewards/accuracies": 0.800000011920929, |
|
"rewards/chosen": -0.06265055388212204, |
|
"rewards/margins": 0.1324789822101593, |
|
"rewards/rejected": -0.19512954354286194, |
|
"step": 960 |
|
}, |
|
{ |
|
"epoch": 1.94, |
|
"learning_rate": 1.3695261579316776e-08, |
|
"logits/chosen": -3.0380070209503174, |
|
"logits/rejected": -2.844536304473877, |
|
"logps/chosen": -164.58413696289062, |
|
"logps/rejected": -158.5533447265625, |
|
"loss": 0.027, |
|
"rewards/accuracies": 0.75, |
|
"rewards/chosen": -0.0411117784678936, |
|
"rewards/margins": 0.07761015743017197, |
|
"rewards/rejected": -0.11872193962335587, |
|
"step": 970 |
|
}, |
|
{ |
|
"epoch": 1.96, |
|
"learning_rate": 6.089874350439507e-09, |
|
"logits/chosen": -3.0895726680755615, |
|
"logits/rejected": -2.8259506225585938, |
|
"logps/chosen": -196.0420684814453, |
|
"logps/rejected": -190.8524169921875, |
|
"loss": 0.022, |
|
"rewards/accuracies": 0.699999988079071, |
|
"rewards/chosen": -0.060807038098573685, |
|
"rewards/margins": 0.03220720216631889, |
|
"rewards/rejected": -0.09301424026489258, |
|
"step": 980 |
|
}, |
|
{ |
|
"epoch": 1.98, |
|
"learning_rate": 1.5229324522605949e-09, |
|
"logits/chosen": -3.0721163749694824, |
|
"logits/rejected": -2.770005941390991, |
|
"logps/chosen": -117.59437561035156, |
|
"logps/rejected": -115.3232421875, |
|
"loss": 0.0234, |
|
"rewards/accuracies": 0.8500000238418579, |
|
"rewards/chosen": -0.033795468509197235, |
|
"rewards/margins": 0.1025773137807846, |
|
"rewards/rejected": -0.13637278974056244, |
|
"step": 990 |
|
}, |
|
{ |
|
"epoch": 2.0, |
|
"learning_rate": 0.0, |
|
"logits/chosen": -2.9728710651397705, |
|
"logits/rejected": -2.4575047492980957, |
|
"logps/chosen": -203.40164184570312, |
|
"logps/rejected": -187.337890625, |
|
"loss": 0.0697, |
|
"rewards/accuracies": 0.5, |
|
"rewards/chosen": -0.11433009803295135, |
|
"rewards/margins": 0.06476924568414688, |
|
"rewards/rejected": -0.17909933626651764, |
|
"step": 1000 |
|
}, |
|
{ |
|
"epoch": 2.0, |
|
"eval_logits/chosen": -2.4070470333099365, |
|
"eval_logits/rejected": -2.210773468017578, |
|
"eval_logps/chosen": -274.3865966796875, |
|
"eval_logps/rejected": -249.44677734375, |
|
"eval_loss": 0.0258334930986166, |
|
"eval_rewards/accuracies": 0.33799999952316284, |
|
"eval_rewards/chosen": -0.05798804759979248, |
|
"eval_rewards/margins": -0.05191566422581673, |
|
"eval_rewards/rejected": -0.006072388496249914, |
|
"eval_runtime": 1422.3182, |
|
"eval_samples_per_second": 1.406, |
|
"eval_steps_per_second": 0.703, |
|
"step": 1000 |
|
}, |
|
{ |
|
"epoch": 2.0, |
|
"step": 1000, |
|
"total_flos": 0.0, |
|
"train_loss": 0.019690872263745406, |
|
"train_runtime": 18916.4989, |
|
"train_samples_per_second": 0.106, |
|
"train_steps_per_second": 0.053 |
|
} |
|
], |
|
"logging_steps": 10, |
|
"max_steps": 1000, |
|
"num_input_tokens_seen": 0, |
|
"num_train_epochs": 2, |
|
"save_steps": 100, |
|
"total_flos": 0.0, |
|
"train_batch_size": 1, |
|
"trial_name": null, |
|
"trial_params": null |
|
} |
|
|