| { | |
| "best_metric": null, | |
| "best_model_checkpoint": null, | |
| "epoch": 0.9998751404669747, | |
| "eval_steps": 1000, | |
| "global_step": 2002, | |
| "is_hyper_param_search": false, | |
| "is_local_process_zero": true, | |
| "is_world_process_zero": true, | |
| "log_history": [ | |
| { | |
| "epoch": 0.000499438132101386, | |
| "grad_norm": 0.22265625, | |
| "learning_rate": 2.4875621890547265e-08, | |
| "logits/chosen": -0.3009346127510071, | |
| "logits/rejected": -0.224898099899292, | |
| "logps/chosen": -43.235816955566406, | |
| "logps/rejected": -65.95542907714844, | |
| "loss": 0.5, | |
| "rewards/accuracies": 0.0, | |
| "rewards/chosen": 0.0, | |
| "rewards/margins": 0.0, | |
| "rewards/rejected": 0.0, | |
| "step": 1 | |
| }, | |
| { | |
| "epoch": 0.00499438132101386, | |
| "grad_norm": 0.1904296875, | |
| "learning_rate": 2.4875621890547267e-07, | |
| "logits/chosen": -0.4162670373916626, | |
| "logits/rejected": -0.31764352321624756, | |
| "logps/chosen": -43.73904037475586, | |
| "logps/rejected": -88.3354263305664, | |
| "loss": 0.4999, | |
| "rewards/accuracies": 0.5277777910232544, | |
| "rewards/chosen": 0.0004759904695674777, | |
| "rewards/margins": 0.0009994357824325562, | |
| "rewards/rejected": -0.0005234453710727394, | |
| "step": 10 | |
| }, | |
| { | |
| "epoch": 0.00998876264202772, | |
| "grad_norm": 0.16796875, | |
| "learning_rate": 4.975124378109453e-07, | |
| "logits/chosen": -0.41128048300743103, | |
| "logits/rejected": -0.3287343382835388, | |
| "logps/chosen": -43.18193054199219, | |
| "logps/rejected": -69.37371063232422, | |
| "loss": 0.4999, | |
| "rewards/accuracies": 0.625, | |
| "rewards/chosen": 8.649445953778923e-05, | |
| "rewards/margins": 0.000692047062329948, | |
| "rewards/rejected": -0.0006055526318959892, | |
| "step": 20 | |
| }, | |
| { | |
| "epoch": 0.014983143963041578, | |
| "grad_norm": 0.25, | |
| "learning_rate": 7.462686567164179e-07, | |
| "logits/chosen": -0.4024788439273834, | |
| "logits/rejected": -0.3096240162849426, | |
| "logps/chosen": -42.980751037597656, | |
| "logps/rejected": -73.10075378417969, | |
| "loss": 0.4999, | |
| "rewards/accuracies": 0.574999988079071, | |
| "rewards/chosen": 0.0004937038174830377, | |
| "rewards/margins": 0.0007297725533135235, | |
| "rewards/rejected": -0.00023606869217474014, | |
| "step": 30 | |
| }, | |
| { | |
| "epoch": 0.01997752528405544, | |
| "grad_norm": 0.208984375, | |
| "learning_rate": 9.950248756218907e-07, | |
| "logits/chosen": -0.41356319189071655, | |
| "logits/rejected": -0.34054869413375854, | |
| "logps/chosen": -43.257789611816406, | |
| "logps/rejected": -69.32649230957031, | |
| "loss": 0.4998, | |
| "rewards/accuracies": 0.675000011920929, | |
| "rewards/chosen": 0.0002575941034592688, | |
| "rewards/margins": 0.0018243074882775545, | |
| "rewards/rejected": -0.0015667133266106248, | |
| "step": 40 | |
| }, | |
| { | |
| "epoch": 0.024971906605069295, | |
| "grad_norm": 0.26953125, | |
| "learning_rate": 1.2437810945273632e-06, | |
| "logits/chosen": -0.4217616021633148, | |
| "logits/rejected": -0.3440130352973938, | |
| "logps/chosen": -44.67601776123047, | |
| "logps/rejected": -78.74809265136719, | |
| "loss": 0.4995, | |
| "rewards/accuracies": 0.7749999761581421, | |
| "rewards/chosen": 0.0004624166467692703, | |
| "rewards/margins": 0.002841049339622259, | |
| "rewards/rejected": -0.00237863278016448, | |
| "step": 50 | |
| }, | |
| { | |
| "epoch": 0.029966287926083156, | |
| "grad_norm": 0.158203125, | |
| "learning_rate": 1.4925373134328358e-06, | |
| "logits/chosen": -0.4335503578186035, | |
| "logits/rejected": -0.3408567011356354, | |
| "logps/chosen": -43.363746643066406, | |
| "logps/rejected": -77.2335433959961, | |
| "loss": 0.4991, | |
| "rewards/accuracies": 0.949999988079071, | |
| "rewards/chosen": 0.0020871085580438375, | |
| "rewards/margins": 0.0050177304074168205, | |
| "rewards/rejected": -0.0029306220822036266, | |
| "step": 60 | |
| }, | |
| { | |
| "epoch": 0.034960669247097016, | |
| "grad_norm": 0.1572265625, | |
| "learning_rate": 1.7412935323383088e-06, | |
| "logits/chosen": -0.4507155418395996, | |
| "logits/rejected": -0.35845330357551575, | |
| "logps/chosen": -42.748069763183594, | |
| "logps/rejected": -73.00779724121094, | |
| "loss": 0.4982, | |
| "rewards/accuracies": 0.9750000238418579, | |
| "rewards/chosen": 0.004088289104402065, | |
| "rewards/margins": 0.009887892752885818, | |
| "rewards/rejected": -0.005799603182822466, | |
| "step": 70 | |
| }, | |
| { | |
| "epoch": 0.03995505056811088, | |
| "grad_norm": 0.1943359375, | |
| "learning_rate": 1.9900497512437813e-06, | |
| "logits/chosen": -0.41265735030174255, | |
| "logits/rejected": -0.32930153608322144, | |
| "logps/chosen": -42.023521423339844, | |
| "logps/rejected": -82.02639770507812, | |
| "loss": 0.497, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.008388923481106758, | |
| "rewards/margins": 0.015186095610260963, | |
| "rewards/rejected": -0.006797172129154205, | |
| "step": 80 | |
| }, | |
| { | |
| "epoch": 0.04494943188912474, | |
| "grad_norm": 0.181640625, | |
| "learning_rate": 2.238805970149254e-06, | |
| "logits/chosen": -0.42406344413757324, | |
| "logits/rejected": -0.32654517889022827, | |
| "logps/chosen": -43.199241638183594, | |
| "logps/rejected": -79.2525405883789, | |
| "loss": 0.4952, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.015419301576912403, | |
| "rewards/margins": 0.025097712874412537, | |
| "rewards/rejected": -0.009678413160145283, | |
| "step": 90 | |
| }, | |
| { | |
| "epoch": 0.04994381321013859, | |
| "grad_norm": 0.232421875, | |
| "learning_rate": 2.4875621890547264e-06, | |
| "logits/chosen": -0.4186275601387024, | |
| "logits/rejected": -0.31876617670059204, | |
| "logps/chosen": -41.526851654052734, | |
| "logps/rejected": -73.03739929199219, | |
| "loss": 0.4934, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.020998705178499222, | |
| "rewards/margins": 0.03922630846500397, | |
| "rewards/rejected": -0.018227603286504745, | |
| "step": 100 | |
| }, | |
| { | |
| "epoch": 0.05493819453115245, | |
| "grad_norm": 0.1630859375, | |
| "learning_rate": 2.736318407960199e-06, | |
| "logits/chosen": -0.3820754289627075, | |
| "logits/rejected": -0.3049188256263733, | |
| "logps/chosen": -40.626625061035156, | |
| "logps/rejected": -77.20478057861328, | |
| "loss": 0.4909, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.026516741141676903, | |
| "rewards/margins": 0.0446377769112587, | |
| "rewards/rejected": -0.018121037632226944, | |
| "step": 110 | |
| }, | |
| { | |
| "epoch": 0.05993257585216631, | |
| "grad_norm": 0.1455078125, | |
| "learning_rate": 2.9850746268656716e-06, | |
| "logits/chosen": -0.4118029475212097, | |
| "logits/rejected": -0.3379635214805603, | |
| "logps/chosen": -40.367244720458984, | |
| "logps/rejected": -72.1778564453125, | |
| "loss": 0.4878, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.034462034702301025, | |
| "rewards/margins": 0.07104991376399994, | |
| "rewards/rejected": -0.036587879061698914, | |
| "step": 120 | |
| }, | |
| { | |
| "epoch": 0.06492695717318017, | |
| "grad_norm": 0.1416015625, | |
| "learning_rate": 3.233830845771145e-06, | |
| "logits/chosen": -0.38510891795158386, | |
| "logits/rejected": -0.2871672511100769, | |
| "logps/chosen": -39.84120559692383, | |
| "logps/rejected": -77.8514175415039, | |
| "loss": 0.4845, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.033992547541856766, | |
| "rewards/margins": 0.09460695832967758, | |
| "rewards/rejected": -0.06061442568898201, | |
| "step": 130 | |
| }, | |
| { | |
| "epoch": 0.06992133849419403, | |
| "grad_norm": 0.2197265625, | |
| "learning_rate": 3.4825870646766175e-06, | |
| "logits/chosen": -0.3648582696914673, | |
| "logits/rejected": -0.2659669816493988, | |
| "logps/chosen": -40.6865234375, | |
| "logps/rejected": -84.98823547363281, | |
| "loss": 0.4808, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.04041652753949165, | |
| "rewards/margins": 0.11247670650482178, | |
| "rewards/rejected": -0.07206018269062042, | |
| "step": 140 | |
| }, | |
| { | |
| "epoch": 0.07491571981520789, | |
| "grad_norm": 0.173828125, | |
| "learning_rate": 3.73134328358209e-06, | |
| "logits/chosen": -0.4185262620449066, | |
| "logits/rejected": -0.31970107555389404, | |
| "logps/chosen": -40.132545471191406, | |
| "logps/rejected": -80.09419250488281, | |
| "loss": 0.4784, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.04452138394117355, | |
| "rewards/margins": 0.12835349142551422, | |
| "rewards/rejected": -0.08383210748434067, | |
| "step": 150 | |
| }, | |
| { | |
| "epoch": 0.07991010113622175, | |
| "grad_norm": 0.130859375, | |
| "learning_rate": 3.980099502487563e-06, | |
| "logits/chosen": -0.37403732538223267, | |
| "logits/rejected": -0.27564138174057007, | |
| "logps/chosen": -39.31542205810547, | |
| "logps/rejected": -90.21852111816406, | |
| "loss": 0.4747, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.044510699808597565, | |
| "rewards/margins": 0.15536533296108246, | |
| "rewards/rejected": -0.11085464060306549, | |
| "step": 160 | |
| }, | |
| { | |
| "epoch": 0.08490448245723561, | |
| "grad_norm": 0.1572265625, | |
| "learning_rate": 4.228855721393035e-06, | |
| "logits/chosen": -0.35230112075805664, | |
| "logits/rejected": -0.2606234848499298, | |
| "logps/chosen": -38.46342086791992, | |
| "logps/rejected": -85.07556915283203, | |
| "loss": 0.4689, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.04909727722406387, | |
| "rewards/margins": 0.20509441196918488, | |
| "rewards/rejected": -0.1559971272945404, | |
| "step": 170 | |
| }, | |
| { | |
| "epoch": 0.08989886377824947, | |
| "grad_norm": 0.2421875, | |
| "learning_rate": 4.477611940298508e-06, | |
| "logits/chosen": -0.3421555161476135, | |
| "logits/rejected": -0.2503698766231537, | |
| "logps/chosen": -39.706153869628906, | |
| "logps/rejected": -85.4745101928711, | |
| "loss": 0.4643, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.0517905056476593, | |
| "rewards/margins": 0.2295013666152954, | |
| "rewards/rejected": -0.1777108609676361, | |
| "step": 180 | |
| }, | |
| { | |
| "epoch": 0.09489324509926333, | |
| "grad_norm": 0.2197265625, | |
| "learning_rate": 4.72636815920398e-06, | |
| "logits/chosen": -0.2977878451347351, | |
| "logits/rejected": -0.17351695895195007, | |
| "logps/chosen": -38.165069580078125, | |
| "logps/rejected": -104.08354187011719, | |
| "loss": 0.4516, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.0498540997505188, | |
| "rewards/margins": 0.33853739500045776, | |
| "rewards/rejected": -0.28868329524993896, | |
| "step": 190 | |
| }, | |
| { | |
| "epoch": 0.09988762642027718, | |
| "grad_norm": 0.484375, | |
| "learning_rate": 4.975124378109453e-06, | |
| "logits/chosen": -0.2946663498878479, | |
| "logits/rejected": -0.17343321442604065, | |
| "logps/chosen": -37.89108657836914, | |
| "logps/rejected": -119.83811950683594, | |
| "loss": 0.4244, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.06007291004061699, | |
| "rewards/margins": 0.5597599148750305, | |
| "rewards/rejected": -0.49968695640563965, | |
| "step": 200 | |
| }, | |
| { | |
| "epoch": 0.10488200774129104, | |
| "grad_norm": 0.271484375, | |
| "learning_rate": 4.999691923599309e-06, | |
| "logits/chosen": -0.24224761128425598, | |
| "logits/rejected": -0.10646134614944458, | |
| "logps/chosen": -38.592735290527344, | |
| "logps/rejected": -158.98190307617188, | |
| "loss": 0.3761, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.052448518574237823, | |
| "rewards/margins": 0.9627677202224731, | |
| "rewards/rejected": -0.9103191494941711, | |
| "step": 210 | |
| }, | |
| { | |
| "epoch": 0.1098763890623049, | |
| "grad_norm": 0.369140625, | |
| "learning_rate": 4.998627065620946e-06, | |
| "logits/chosen": -0.20557060837745667, | |
| "logits/rejected": -0.019889693707227707, | |
| "logps/chosen": -39.04503631591797, | |
| "logps/rejected": -309.40240478515625, | |
| "loss": 0.3162, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.05903823301196098, | |
| "rewards/margins": 2.2989630699157715, | |
| "rewards/rejected": -2.239924907684326, | |
| "step": 220 | |
| }, | |
| { | |
| "epoch": 0.11487077038331876, | |
| "grad_norm": 0.1435546875, | |
| "learning_rate": 4.996801946581365e-06, | |
| "logits/chosen": -0.08062759786844254, | |
| "logits/rejected": 0.12686052918434143, | |
| "logps/chosen": -38.807472229003906, | |
| "logps/rejected": -391.716552734375, | |
| "loss": 0.2979, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.03988034278154373, | |
| "rewards/margins": 3.2561073303222656, | |
| "rewards/rejected": -3.216226577758789, | |
| "step": 230 | |
| }, | |
| { | |
| "epoch": 0.11986515170433262, | |
| "grad_norm": 0.173828125, | |
| "learning_rate": 4.99421712181231e-06, | |
| "logits/chosen": -0.09656897932291031, | |
| "logits/rejected": 0.20183369517326355, | |
| "logps/chosen": -40.211158752441406, | |
| "logps/rejected": -563.9237060546875, | |
| "loss": 0.2787, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.03454852104187012, | |
| "rewards/margins": 4.958992958068848, | |
| "rewards/rejected": -4.924445152282715, | |
| "step": 240 | |
| }, | |
| { | |
| "epoch": 0.12485953302534648, | |
| "grad_norm": 0.19140625, | |
| "learning_rate": 4.990873377802351e-06, | |
| "logits/chosen": -0.04213310405611992, | |
| "logits/rejected": 0.26729267835617065, | |
| "logps/chosen": -38.64299011230469, | |
| "logps/rejected": -672.4085693359375, | |
| "loss": 0.2738, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.04478804022073746, | |
| "rewards/margins": 6.028790473937988, | |
| "rewards/rejected": -5.984002113342285, | |
| "step": 250 | |
| }, | |
| { | |
| "epoch": 0.12985391434636034, | |
| "grad_norm": 0.1591796875, | |
| "learning_rate": 4.986771731957569e-06, | |
| "logits/chosen": -0.013924488797783852, | |
| "logits/rejected": 0.32576116919517517, | |
| "logps/chosen": -38.04896926879883, | |
| "logps/rejected": -677.6256713867188, | |
| "loss": 0.269, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.04615269601345062, | |
| "rewards/margins": 6.120830535888672, | |
| "rewards/rejected": -6.07467794418335, | |
| "step": 260 | |
| }, | |
| { | |
| "epoch": 0.1348482956673742, | |
| "grad_norm": 0.1240234375, | |
| "learning_rate": 4.981913432291989e-06, | |
| "logits/chosen": -0.0022221256513148546, | |
| "logits/rejected": 0.3353291451931, | |
| "logps/chosen": -36.26408004760742, | |
| "logps/rejected": -657.3780517578125, | |
| "loss": 0.2603, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.07836371660232544, | |
| "rewards/margins": 5.838679790496826, | |
| "rewards/rejected": -5.760315895080566, | |
| "step": 270 | |
| }, | |
| { | |
| "epoch": 0.13984267698838806, | |
| "grad_norm": 0.138671875, | |
| "learning_rate": 4.976299957047846e-06, | |
| "logits/chosen": -0.008776476606726646, | |
| "logits/rejected": 0.35888582468032837, | |
| "logps/chosen": -34.863895416259766, | |
| "logps/rejected": -787.3309326171875, | |
| "loss": 0.2619, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.0779188945889473, | |
| "rewards/margins": 7.239710330963135, | |
| "rewards/rejected": -7.1617913246154785, | |
| "step": 280 | |
| }, | |
| { | |
| "epoch": 0.1448370583094019, | |
| "grad_norm": 0.2373046875, | |
| "learning_rate": 4.9699330142458e-06, | |
| "logits/chosen": -0.005239410791546106, | |
| "logits/rejected": 0.3836653232574463, | |
| "logps/chosen": -30.54348373413086, | |
| "logps/rejected": -689.7301635742188, | |
| "loss": 0.2517, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.1276303231716156, | |
| "rewards/margins": 6.348529815673828, | |
| "rewards/rejected": -6.2208991050720215, | |
| "step": 290 | |
| }, | |
| { | |
| "epoch": 0.14983143963041579, | |
| "grad_norm": 0.15625, | |
| "learning_rate": 4.96281454116523e-06, | |
| "logits/chosen": -0.018074408173561096, | |
| "logits/rejected": 0.34884509444236755, | |
| "logps/chosen": -19.059213638305664, | |
| "logps/rejected": -759.7054443359375, | |
| "loss": 0.2355, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.24109096825122833, | |
| "rewards/margins": 7.104301452636719, | |
| "rewards/rejected": -6.863211154937744, | |
| "step": 300 | |
| }, | |
| { | |
| "epoch": 0.15482582095142963, | |
| "grad_norm": 0.14453125, | |
| "learning_rate": 4.954946703754777e-06, | |
| "logits/chosen": -0.022661946713924408, | |
| "logits/rejected": 0.36587223410606384, | |
| "logps/chosen": -14.688285827636719, | |
| "logps/rejected": -652.6716918945312, | |
| "loss": 0.2303, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.2812207341194153, | |
| "rewards/margins": 6.1413116455078125, | |
| "rewards/rejected": -5.860090255737305, | |
| "step": 310 | |
| }, | |
| { | |
| "epoch": 0.1598202022724435, | |
| "grad_norm": 0.06591796875, | |
| "learning_rate": 4.946331895973308e-06, | |
| "logits/chosen": 0.027700275182724, | |
| "logits/rejected": 0.4779927134513855, | |
| "logps/chosen": -13.264841079711914, | |
| "logps/rejected": -853.7429809570312, | |
| "loss": 0.2306, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.29832005500793457, | |
| "rewards/margins": 7.9851837158203125, | |
| "rewards/rejected": -7.686862945556641, | |
| "step": 320 | |
| }, | |
| { | |
| "epoch": 0.16481458359345735, | |
| "grad_norm": 0.1298828125, | |
| "learning_rate": 4.936972739061503e-06, | |
| "logits/chosen": 0.028876056894659996, | |
| "logits/rejected": 0.4520367980003357, | |
| "logps/chosen": -14.747647285461426, | |
| "logps/rejected": -819.2418212890625, | |
| "loss": 0.2308, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.29899871349334717, | |
| "rewards/margins": 7.518294334411621, | |
| "rewards/rejected": -7.219296455383301, | |
| "step": 330 | |
| }, | |
| { | |
| "epoch": 0.16980896491447123, | |
| "grad_norm": 0.2353515625, | |
| "learning_rate": 4.926872080744284e-06, | |
| "logits/chosen": 0.09099732339382172, | |
| "logits/rejected": 0.6329769492149353, | |
| "logps/chosen": -14.6112699508667, | |
| "logps/rejected": -978.19482421875, | |
| "loss": 0.2205, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.290539413690567, | |
| "rewards/margins": 9.353235244750977, | |
| "rewards/rejected": -9.06269645690918, | |
| "step": 340 | |
| }, | |
| { | |
| "epoch": 0.17480334623548507, | |
| "grad_norm": 0.0673828125, | |
| "learning_rate": 4.9160329943643335e-06, | |
| "logits/chosen": 0.10238673537969589, | |
| "logits/rejected": 0.6276119947433472, | |
| "logps/chosen": -13.135534286499023, | |
| "logps/rejected": -862.1193237304688, | |
| "loss": 0.2233, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.2938804030418396, | |
| "rewards/margins": 8.220571517944336, | |
| "rewards/rejected": -7.92669153213501, | |
| "step": 350 | |
| }, | |
| { | |
| "epoch": 0.17979772755649895, | |
| "grad_norm": 0.064453125, | |
| "learning_rate": 4.904458777946967e-06, | |
| "logits/chosen": 0.023329418152570724, | |
| "logits/rejected": 0.6091981530189514, | |
| "logps/chosen": -13.777295112609863, | |
| "logps/rejected": -1096.3214111328125, | |
| "loss": 0.221, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.2997492849826813, | |
| "rewards/margins": 10.409059524536133, | |
| "rewards/rejected": -10.1093111038208, | |
| "step": 360 | |
| }, | |
| { | |
| "epoch": 0.1847921088775128, | |
| "grad_norm": 0.1201171875, | |
| "learning_rate": 4.892152953196633e-06, | |
| "logits/chosen": 0.029097210615873337, | |
| "logits/rejected": 0.650887131690979, | |
| "logps/chosen": -13.820713996887207, | |
| "logps/rejected": -1171.0504150390625, | |
| "loss": 0.2235, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.2969459295272827, | |
| "rewards/margins": 11.168619155883789, | |
| "rewards/rejected": -10.871672630310059, | |
| "step": 370 | |
| }, | |
| { | |
| "epoch": 0.18978649019852667, | |
| "grad_norm": 0.06640625, | |
| "learning_rate": 4.879119264425366e-06, | |
| "logits/chosen": 0.11170516163110733, | |
| "logits/rejected": 0.7552271485328674, | |
| "logps/chosen": -13.031651496887207, | |
| "logps/rejected": -990.927734375, | |
| "loss": 0.2191, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.30227065086364746, | |
| "rewards/margins": 9.518648147583008, | |
| "rewards/rejected": -9.216377258300781, | |
| "step": 380 | |
| }, | |
| { | |
| "epoch": 0.19478087151954052, | |
| "grad_norm": 0.0272216796875, | |
| "learning_rate": 4.865361677413489e-06, | |
| "logits/chosen": 0.10295484960079193, | |
| "logits/rejected": 0.6912266612052917, | |
| "logps/chosen": -14.148368835449219, | |
| "logps/rejected": -973.99365234375, | |
| "loss": 0.2236, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.30019253492355347, | |
| "rewards/margins": 9.242959022521973, | |
| "rewards/rejected": -8.942765235900879, | |
| "step": 390 | |
| }, | |
| { | |
| "epoch": 0.19977525284055436, | |
| "grad_norm": 0.048583984375, | |
| "learning_rate": 4.850884378202947e-06, | |
| "logits/chosen": 0.12218449264764786, | |
| "logits/rejected": 0.7848892211914062, | |
| "logps/chosen": -13.857281684875488, | |
| "logps/rejected": -1093.4356689453125, | |
| "loss": 0.2224, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.30381980538368225, | |
| "rewards/margins": 10.443190574645996, | |
| "rewards/rejected": -10.139370918273926, | |
| "step": 400 | |
| }, | |
| { | |
| "epoch": 0.20476963416156824, | |
| "grad_norm": 0.10888671875, | |
| "learning_rate": 4.8356917718236125e-06, | |
| "logits/chosen": 0.16129298508167267, | |
| "logits/rejected": 0.83033287525177, | |
| "logps/chosen": -13.215472221374512, | |
| "logps/rejected": -1056.891357421875, | |
| "loss": 0.2254, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.30162861943244934, | |
| "rewards/margins": 10.163104057312012, | |
| "rewards/rejected": -9.861475944519043, | |
| "step": 410 | |
| }, | |
| { | |
| "epoch": 0.20976401548258208, | |
| "grad_norm": 0.05712890625, | |
| "learning_rate": 4.8197884809529575e-06, | |
| "logits/chosen": 0.18466398119926453, | |
| "logits/rejected": 0.8971255421638489, | |
| "logps/chosen": -14.178210258483887, | |
| "logps/rejected": -1095.867431640625, | |
| "loss": 0.2192, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.29163575172424316, | |
| "rewards/margins": 10.56762409210205, | |
| "rewards/rejected": -10.275988578796387, | |
| "step": 420 | |
| }, | |
| { | |
| "epoch": 0.21475839680359596, | |
| "grad_norm": 0.078125, | |
| "learning_rate": 4.803179344509505e-06, | |
| "logits/chosen": 0.17180819809436798, | |
| "logits/rejected": 0.9859398603439331, | |
| "logps/chosen": -14.1710844039917, | |
| "logps/rejected": -1132.571044921875, | |
| "loss": 0.2206, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.2903401851654053, | |
| "rewards/margins": 10.91908073425293, | |
| "rewards/rejected": -10.628740310668945, | |
| "step": 430 | |
| }, | |
| { | |
| "epoch": 0.2197527781246098, | |
| "grad_norm": 0.0341796875, | |
| "learning_rate": 4.785869416180489e-06, | |
| "logits/chosen": 0.18128976225852966, | |
| "logits/rejected": 0.9951160550117493, | |
| "logps/chosen": -13.373617172241211, | |
| "logps/rejected": -1239.9248046875, | |
| "loss": 0.2185, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.3014771342277527, | |
| "rewards/margins": 11.977083206176758, | |
| "rewards/rejected": -11.675604820251465, | |
| "step": 440 | |
| }, | |
| { | |
| "epoch": 0.22474715944562368, | |
| "grad_norm": 0.035400390625, | |
| "learning_rate": 4.767863962884156e-06, | |
| "logits/chosen": 0.19665592908859253, | |
| "logits/rejected": 1.0053622722625732, | |
| "logps/chosen": -12.768911361694336, | |
| "logps/rejected": -1196.0384521484375, | |
| "loss": 0.217, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.30236831307411194, | |
| "rewards/margins": 11.53145694732666, | |
| "rewards/rejected": -11.229089736938477, | |
| "step": 450 | |
| }, | |
| { | |
| "epoch": 0.22974154076663753, | |
| "grad_norm": 0.06982421875, | |
| "learning_rate": 4.74916846316719e-06, | |
| "logits/chosen": 0.2026137411594391, | |
| "logits/rejected": 0.9967263340950012, | |
| "logps/chosen": -14.13359546661377, | |
| "logps/rejected": -1080.839111328125, | |
| "loss": 0.2196, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.29357805848121643, | |
| "rewards/margins": 10.319085121154785, | |
| "rewards/rejected": -10.025506973266602, | |
| "step": 460 | |
| }, | |
| { | |
| "epoch": 0.2347359220876514, | |
| "grad_norm": 0.047607421875, | |
| "learning_rate": 4.7297886055377525e-06, | |
| "logits/chosen": 0.22456324100494385, | |
| "logits/rejected": 0.9802171587944031, | |
| "logps/chosen": -13.012743949890137, | |
| "logps/rejected": -1017.5540161132812, | |
| "loss": 0.2208, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.29690033197402954, | |
| "rewards/margins": 9.80536937713623, | |
| "rewards/rejected": -9.508468627929688, | |
| "step": 470 | |
| }, | |
| { | |
| "epoch": 0.23973030340866525, | |
| "grad_norm": 0.02734375, | |
| "learning_rate": 4.709730286734631e-06, | |
| "logits/chosen": 0.2183937281370163, | |
| "logits/rejected": 1.0708736181259155, | |
| "logps/chosen": -12.663009643554688, | |
| "logps/rejected": -1249.983642578125, | |
| "loss": 0.2169, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.3058857023715973, | |
| "rewards/margins": 12.078222274780273, | |
| "rewards/rejected": -11.772336959838867, | |
| "step": 480 | |
| }, | |
| { | |
| "epoch": 0.24472468472967912, | |
| "grad_norm": 0.055908203125, | |
| "learning_rate": 4.688999609933023e-06, | |
| "logits/chosen": 0.22988371551036835, | |
| "logits/rejected": 1.0844902992248535, | |
| "logps/chosen": -12.956899642944336, | |
| "logps/rejected": -1156.6395263671875, | |
| "loss": 0.2176, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.30138763785362244, | |
| "rewards/margins": 11.153468132019043, | |
| "rewards/rejected": -10.852079391479492, | |
| "step": 490 | |
| }, | |
| { | |
| "epoch": 0.24971906605069297, | |
| "grad_norm": 0.049072265625, | |
| "learning_rate": 4.6676028828875195e-06, | |
| "logits/chosen": 0.19053277373313904, | |
| "logits/rejected": 1.1232895851135254, | |
| "logps/chosen": -13.526689529418945, | |
| "logps/rejected": -1328.9798583984375, | |
| "loss": 0.2173, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.2967537045478821, | |
| "rewards/margins": 12.862408638000488, | |
| "rewards/rejected": -12.565653800964355, | |
| "step": 500 | |
| }, | |
| { | |
| "epoch": 0.2547134473717068, | |
| "grad_norm": 0.00811767578125, | |
| "learning_rate": 4.645546616012835e-06, | |
| "logits/chosen": 0.19936171174049377, | |
| "logits/rejected": 1.1598930358886719, | |
| "logps/chosen": -13.963285446166992, | |
| "logps/rejected": -1301.6494140625, | |
| "loss": 0.217, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.29725441336631775, | |
| "rewards/margins": 12.456524848937988, | |
| "rewards/rejected": -12.159271240234375, | |
| "step": 510 | |
| }, | |
| { | |
| "epoch": 0.2597078286927207, | |
| "grad_norm": 0.022216796875, | |
| "learning_rate": 4.622837520402869e-06, | |
| "logits/chosen": 0.2132669985294342, | |
| "logits/rejected": 1.1716349124908447, | |
| "logps/chosen": -13.427679061889648, | |
| "logps/rejected": -1347.979248046875, | |
| "loss": 0.216, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.30607202649116516, | |
| "rewards/margins": 13.056879043579102, | |
| "rewards/rejected": -12.75080680847168, | |
| "step": 520 | |
| }, | |
| { | |
| "epoch": 0.26470221001373456, | |
| "grad_norm": 0.0654296875, | |
| "learning_rate": 4.599482505788715e-06, | |
| "logits/chosen": 0.1745399534702301, | |
| "logits/rejected": 1.1154874563217163, | |
| "logps/chosen": -13.649249076843262, | |
| "logps/rejected": -1377.6204833984375, | |
| "loss": 0.2163, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.2969611883163452, | |
| "rewards/margins": 13.256256103515625, | |
| "rewards/rejected": -12.959295272827148, | |
| "step": 530 | |
| }, | |
| { | |
| "epoch": 0.2696965913347484, | |
| "grad_norm": 0.040771484375, | |
| "learning_rate": 4.575488678436228e-06, | |
| "logits/chosen": 0.20975852012634277, | |
| "logits/rejected": 1.2858575582504272, | |
| "logps/chosen": -13.548286437988281, | |
| "logps/rejected": -1526.4365234375, | |
| "loss": 0.2161, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.30410271883010864, | |
| "rewards/margins": 14.82691764831543, | |
| "rewards/rejected": -14.522814750671387, | |
| "step": 540 | |
| }, | |
| { | |
| "epoch": 0.27469097265576226, | |
| "grad_norm": 0.0235595703125, | |
| "learning_rate": 4.550863338983784e-06, | |
| "logits/chosen": 0.23238572478294373, | |
| "logits/rejected": 1.2929937839508057, | |
| "logps/chosen": -12.818222045898438, | |
| "logps/rejected": -1398.010009765625, | |
| "loss": 0.2146, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.3006662130355835, | |
| "rewards/margins": 13.579294204711914, | |
| "rewards/rejected": -13.2786283493042, | |
| "step": 550 | |
| }, | |
| { | |
| "epoch": 0.27968535397677613, | |
| "grad_norm": 0.045654296875, | |
| "learning_rate": 4.525613980220909e-06, | |
| "logits/chosen": 0.21401552855968475, | |
| "logits/rejected": 1.2280786037445068, | |
| "logps/chosen": -13.640890121459961, | |
| "logps/rejected": -1425.2213134765625, | |
| "loss": 0.2165, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.30365556478500366, | |
| "rewards/margins": 13.8203706741333, | |
| "rewards/rejected": -13.516714096069336, | |
| "step": 560 | |
| }, | |
| { | |
| "epoch": 0.28467973529779, | |
| "grad_norm": 0.031494140625, | |
| "learning_rate": 4.499748284808433e-06, | |
| "logits/chosen": 0.2350139617919922, | |
| "logits/rejected": 1.237275242805481, | |
| "logps/chosen": -13.027705192565918, | |
| "logps/rejected": -1249.4954833984375, | |
| "loss": 0.2147, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.3124990463256836, | |
| "rewards/margins": 11.985966682434082, | |
| "rewards/rejected": -11.673466682434082, | |
| "step": 570 | |
| }, | |
| { | |
| "epoch": 0.2896741166188038, | |
| "grad_norm": 0.037109375, | |
| "learning_rate": 4.473274122940879e-06, | |
| "logits/chosen": 0.26041245460510254, | |
| "logits/rejected": 1.2588506937026978, | |
| "logps/chosen": -12.92595100402832, | |
| "logps/rejected": -1345.543212890625, | |
| "loss": 0.2145, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.3006027638912201, | |
| "rewards/margins": 13.026535034179688, | |
| "rewards/rejected": -12.725933074951172, | |
| "step": 580 | |
| }, | |
| { | |
| "epoch": 0.2946684979398177, | |
| "grad_norm": 0.023193359375, | |
| "learning_rate": 4.446199549951782e-06, | |
| "logits/chosen": 0.2726953327655792, | |
| "logits/rejected": 1.2741527557373047, | |
| "logps/chosen": -13.607648849487305, | |
| "logps/rejected": -1350.020263671875, | |
| "loss": 0.2164, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.2966735064983368, | |
| "rewards/margins": 13.099435806274414, | |
| "rewards/rejected": -12.802760124206543, | |
| "step": 590 | |
| }, | |
| { | |
| "epoch": 0.29966287926083157, | |
| "grad_norm": 0.060302734375, | |
| "learning_rate": 4.418532803862684e-06, | |
| "logits/chosen": 0.24927139282226562, | |
| "logits/rejected": 1.2164738178253174, | |
| "logps/chosen": -13.667009353637695, | |
| "logps/rejected": -1226.8724365234375, | |
| "loss": 0.2161, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.3081030249595642, | |
| "rewards/margins": 11.710563659667969, | |
| "rewards/rejected": -11.402461051940918, | |
| "step": 600 | |
| }, | |
| { | |
| "epoch": 0.30465726058184545, | |
| "grad_norm": 0.01220703125, | |
| "learning_rate": 4.39028230287654e-06, | |
| "logits/chosen": 0.24915924668312073, | |
| "logits/rejected": 1.2275068759918213, | |
| "logps/chosen": -14.152711868286133, | |
| "logps/rejected": -1383.346923828125, | |
| "loss": 0.2153, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.301180899143219, | |
| "rewards/margins": 13.357465744018555, | |
| "rewards/rejected": -13.05628490447998, | |
| "step": 610 | |
| }, | |
| { | |
| "epoch": 0.30965164190285926, | |
| "grad_norm": 0.056884765625, | |
| "learning_rate": 4.361456642816292e-06, | |
| "logits/chosen": 0.18370430171489716, | |
| "logits/rejected": 1.187785267829895, | |
| "logps/chosen": -14.063751220703125, | |
| "logps/rejected": -1465.3687744140625, | |
| "loss": 0.2156, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.3011600375175476, | |
| "rewards/margins": 14.13781452178955, | |
| "rewards/rejected": -13.836652755737305, | |
| "step": 620 | |
| }, | |
| { | |
| "epoch": 0.31464602322387314, | |
| "grad_norm": 0.036376953125, | |
| "learning_rate": 4.332064594509413e-06, | |
| "logits/chosen": 0.19446897506713867, | |
| "logits/rejected": 1.427197813987732, | |
| "logps/chosen": -14.258028030395508, | |
| "logps/rejected": -1825.0166015625, | |
| "loss": 0.2145, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.2957887053489685, | |
| "rewards/margins": 17.80067253112793, | |
| "rewards/rejected": -17.504884719848633, | |
| "step": 630 | |
| }, | |
| { | |
| "epoch": 0.319640404544887, | |
| "grad_norm": 0.025634765625, | |
| "learning_rate": 4.302115101119186e-06, | |
| "logits/chosen": 0.19377607107162476, | |
| "logits/rejected": 1.1977471113204956, | |
| "logps/chosen": -13.28663158416748, | |
| "logps/rejected": -1551.3272705078125, | |
| "loss": 0.2146, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.2949761748313904, | |
| "rewards/margins": 14.840034484863281, | |
| "rewards/rejected": -14.545059204101562, | |
| "step": 640 | |
| }, | |
| { | |
| "epoch": 0.3246347858659009, | |
| "grad_norm": 0.028076171875, | |
| "learning_rate": 4.271617275423564e-06, | |
| "logits/chosen": 0.18471740186214447, | |
| "logits/rejected": 1.2049682140350342, | |
| "logps/chosen": -14.22096061706543, | |
| "logps/rejected": -1509.560791015625, | |
| "loss": 0.214, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.2953301966190338, | |
| "rewards/margins": 14.56297779083252, | |
| "rewards/rejected": -14.267648696899414, | |
| "step": 650 | |
| }, | |
| { | |
| "epoch": 0.3296291671869147, | |
| "grad_norm": 0.031494140625, | |
| "learning_rate": 4.2405803970423995e-06, | |
| "logits/chosen": 0.21741405129432678, | |
| "logits/rejected": 1.3314052820205688, | |
| "logps/chosen": -13.835968017578125, | |
| "logps/rejected": -1617.0235595703125, | |
| "loss": 0.2147, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.29379233717918396, | |
| "rewards/margins": 15.649670600891113, | |
| "rewards/rejected": -15.355878829956055, | |
| "step": 660 | |
| }, | |
| { | |
| "epoch": 0.3346235485079286, | |
| "grad_norm": 0.11572265625, | |
| "learning_rate": 4.2090139096139306e-06, | |
| "logits/chosen": 0.16212065517902374, | |
| "logits/rejected": 1.2855770587921143, | |
| "logps/chosen": -14.017046928405762, | |
| "logps/rejected": -1740.139404296875, | |
| "loss": 0.2134, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.3004634976387024, | |
| "rewards/margins": 16.887771606445312, | |
| "rewards/rejected": -16.58730697631836, | |
| "step": 670 | |
| }, | |
| { | |
| "epoch": 0.33961792982894246, | |
| "grad_norm": 0.043701171875, | |
| "learning_rate": 4.176927417921343e-06, | |
| "logits/chosen": 0.326777845621109, | |
| "logits/rejected": 1.3592358827590942, | |
| "logps/chosen": -13.120327949523926, | |
| "logps/rejected": -1251.674560546875, | |
| "loss": 0.2143, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.30190396308898926, | |
| "rewards/margins": 12.16067123413086, | |
| "rewards/rejected": -11.858766555786133, | |
| "step": 680 | |
| }, | |
| { | |
| "epoch": 0.3446123111499563, | |
| "grad_norm": 0.03173828125, | |
| "learning_rate": 4.144330684970314e-06, | |
| "logits/chosen": 0.22485598921775818, | |
| "logits/rejected": 1.238599181175232, | |
| "logps/chosen": -14.03515625, | |
| "logps/rejected": -1432.62060546875, | |
| "loss": 0.214, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.30531880259513855, | |
| "rewards/margins": 13.757417678833008, | |
| "rewards/rejected": -13.45209789276123, | |
| "step": 690 | |
| }, | |
| { | |
| "epoch": 0.34960669247097015, | |
| "grad_norm": 0.064453125, | |
| "learning_rate": 4.111233629018404e-06, | |
| "logits/chosen": 0.2409452497959137, | |
| "logits/rejected": 1.3199043273925781, | |
| "logps/chosen": -13.525833129882812, | |
| "logps/rejected": -1432.090087890625, | |
| "loss": 0.2146, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.29940056800842285, | |
| "rewards/margins": 13.923855781555176, | |
| "rewards/rejected": -13.624455451965332, | |
| "step": 700 | |
| }, | |
| { | |
| "epoch": 0.354601073791984, | |
| "grad_norm": 0.044677734375, | |
| "learning_rate": 4.077646320557215e-06, | |
| "logits/chosen": 0.25844550132751465, | |
| "logits/rejected": 1.4347895383834839, | |
| "logps/chosen": -13.414407730102539, | |
| "logps/rejected": -1587.720947265625, | |
| "loss": 0.214, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.29702991247177124, | |
| "rewards/margins": 15.390164375305176, | |
| "rewards/rejected": -15.093134880065918, | |
| "step": 710 | |
| }, | |
| { | |
| "epoch": 0.3595954551129979, | |
| "grad_norm": 0.017822265625, | |
| "learning_rate": 4.043578979248228e-06, | |
| "logits/chosen": 0.24548295140266418, | |
| "logits/rejected": 1.3877404928207397, | |
| "logps/chosen": -12.786032676696777, | |
| "logps/rejected": -1540.37939453125, | |
| "loss": 0.2147, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.3044242262840271, | |
| "rewards/margins": 14.986343383789062, | |
| "rewards/rejected": -14.681918144226074, | |
| "step": 720 | |
| }, | |
| { | |
| "epoch": 0.3645898364340117, | |
| "grad_norm": 0.0203857421875, | |
| "learning_rate": 4.009041970813247e-06, | |
| "logits/chosen": 0.2618701457977295, | |
| "logits/rejected": 1.432408332824707, | |
| "logps/chosen": -12.812631607055664, | |
| "logps/rejected": -1661.7718505859375, | |
| "loss": 0.2146, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.29122787714004517, | |
| "rewards/margins": 16.198129653930664, | |
| "rewards/rejected": -15.906901359558105, | |
| "step": 730 | |
| }, | |
| { | |
| "epoch": 0.3695842177550256, | |
| "grad_norm": 0.030029296875, | |
| "learning_rate": 3.9740458038804075e-06, | |
| "logits/chosen": 0.25733712315559387, | |
| "logits/rejected": 1.3133214712142944, | |
| "logps/chosen": -14.166203498840332, | |
| "logps/rejected": -1473.091552734375, | |
| "loss": 0.2133, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.3017811179161072, | |
| "rewards/margins": 14.06616497039795, | |
| "rewards/rejected": -13.764383316040039, | |
| "step": 740 | |
| }, | |
| { | |
| "epoch": 0.37457859907603946, | |
| "grad_norm": 0.02880859375, | |
| "learning_rate": 3.938601126786702e-06, | |
| "logits/chosen": 0.28963789343833923, | |
| "logits/rejected": 1.4084501266479492, | |
| "logps/chosen": -12.976341247558594, | |
| "logps/rejected": -1537.215087890625, | |
| "loss": 0.2137, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.2965225577354431, | |
| "rewards/margins": 14.963714599609375, | |
| "rewards/rejected": -14.667192459106445, | |
| "step": 750 | |
| }, | |
| { | |
| "epoch": 0.37957298039705334, | |
| "grad_norm": 0.0306396484375, | |
| "learning_rate": 3.902718724337993e-06, | |
| "logits/chosen": 0.22370409965515137, | |
| "logits/rejected": 1.3502318859100342, | |
| "logps/chosen": -13.021102905273438, | |
| "logps/rejected": -1571.427978515625, | |
| "loss": 0.2141, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.3001527190208435, | |
| "rewards/margins": 15.25025463104248, | |
| "rewards/rejected": -14.950100898742676, | |
| "step": 760 | |
| }, | |
| { | |
| "epoch": 0.38456736171806716, | |
| "grad_norm": 0.014404296875, | |
| "learning_rate": 3.8664095145274995e-06, | |
| "logits/chosen": 0.26876306533813477, | |
| "logits/rejected": 1.432448387145996, | |
| "logps/chosen": -13.371549606323242, | |
| "logps/rejected": -1552.1263427734375, | |
| "loss": 0.2136, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.30356094241142273, | |
| "rewards/margins": 15.13947582244873, | |
| "rewards/rejected": -14.835916519165039, | |
| "step": 770 | |
| }, | |
| { | |
| "epoch": 0.38956174303908103, | |
| "grad_norm": 0.034912109375, | |
| "learning_rate": 3.829684545213768e-06, | |
| "logits/chosen": 0.23094145953655243, | |
| "logits/rejected": 1.379480242729187, | |
| "logps/chosen": -13.367365837097168, | |
| "logps/rejected": -1540.384765625, | |
| "loss": 0.2137, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.30820637941360474, | |
| "rewards/margins": 14.93891716003418, | |
| "rewards/rejected": -14.630711555480957, | |
| "step": 780 | |
| }, | |
| { | |
| "epoch": 0.3945561243600949, | |
| "grad_norm": 0.03857421875, | |
| "learning_rate": 3.7925549907591252e-06, | |
| "logits/chosen": 0.17974331974983215, | |
| "logits/rejected": 1.3995566368103027, | |
| "logps/chosen": -13.248870849609375, | |
| "logps/rejected": -1851.462646484375, | |
| "loss": 0.2139, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.3005455434322357, | |
| "rewards/margins": 18.06133460998535, | |
| "rewards/rejected": -17.760787963867188, | |
| "step": 790 | |
| }, | |
| { | |
| "epoch": 0.3995505056811087, | |
| "grad_norm": 0.0272216796875, | |
| "learning_rate": 3.7550321486296303e-06, | |
| "logits/chosen": 0.1997009515762329, | |
| "logits/rejected": 1.2776936292648315, | |
| "logps/chosen": -13.081275939941406, | |
| "logps/rejected": -1519.75537109375, | |
| "loss": 0.2139, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.29953616857528687, | |
| "rewards/margins": 14.774045944213867, | |
| "rewards/rejected": -14.474508285522461, | |
| "step": 800 | |
| }, | |
| { | |
| "epoch": 0.4045448870021226, | |
| "grad_norm": 0.0252685546875, | |
| "learning_rate": 3.717127435957583e-06, | |
| "logits/chosen": 0.22182372212409973, | |
| "logits/rejected": 1.330664873123169, | |
| "logps/chosen": -13.001907348632812, | |
| "logps/rejected": -1577.052978515625, | |
| "loss": 0.214, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.30547088384628296, | |
| "rewards/margins": 15.365854263305664, | |
| "rewards/rejected": -15.060381889343262, | |
| "step": 810 | |
| }, | |
| { | |
| "epoch": 0.4095392683231365, | |
| "grad_norm": 0.011474609375, | |
| "learning_rate": 3.6788523860676156e-06, | |
| "logits/chosen": 0.23856505751609802, | |
| "logits/rejected": 1.3909879922866821, | |
| "logps/chosen": -13.222851753234863, | |
| "logps/rejected": -1567.79931640625, | |
| "loss": 0.2142, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.3013755977153778, | |
| "rewards/margins": 15.270452499389648, | |
| "rewards/rejected": -14.969076156616211, | |
| "step": 820 | |
| }, | |
| { | |
| "epoch": 0.41453364964415035, | |
| "grad_norm": 0.0267333984375, | |
| "learning_rate": 3.640218644967429e-06, | |
| "logits/chosen": 0.2593843638896942, | |
| "logits/rejected": 1.4300034046173096, | |
| "logps/chosen": -12.888224601745605, | |
| "logps/rejected": -1659.7015380859375, | |
| "loss": 0.2143, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.295782208442688, | |
| "rewards/margins": 16.122600555419922, | |
| "rewards/rejected": -15.826817512512207, | |
| "step": 830 | |
| }, | |
| { | |
| "epoch": 0.41952803096516417, | |
| "grad_norm": 0.0194091796875, | |
| "learning_rate": 3.601237967804245e-06, | |
| "logits/chosen": 0.264489084482193, | |
| "logits/rejected": 1.4015864133834839, | |
| "logps/chosen": -12.973742485046387, | |
| "logps/rejected": -1561.237548828125, | |
| "loss": 0.2142, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.2937301695346832, | |
| "rewards/margins": 15.21537971496582, | |
| "rewards/rejected": -14.921648979187012, | |
| "step": 840 | |
| }, | |
| { | |
| "epoch": 0.42452241228617804, | |
| "grad_norm": 0.034912109375, | |
| "learning_rate": 3.5619222152880488e-06, | |
| "logits/chosen": 0.26485809683799744, | |
| "logits/rejected": 1.4641757011413574, | |
| "logps/chosen": -12.745534896850586, | |
| "logps/rejected": -1704.2015380859375, | |
| "loss": 0.2143, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.29673147201538086, | |
| "rewards/margins": 16.65966033935547, | |
| "rewards/rejected": -16.362926483154297, | |
| "step": 850 | |
| }, | |
| { | |
| "epoch": 0.4295167936071919, | |
| "grad_norm": 0.03662109375, | |
| "learning_rate": 3.522283350082713e-06, | |
| "logits/chosen": 0.27674156427383423, | |
| "logits/rejected": 1.3279974460601807, | |
| "logps/chosen": -13.220677375793457, | |
| "logps/rejected": -1268.2398681640625, | |
| "loss": 0.2143, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.30168816447257996, | |
| "rewards/margins": 12.249058723449707, | |
| "rewards/rejected": -11.947370529174805, | |
| "step": 860 | |
| }, | |
| { | |
| "epoch": 0.4345111749282058, | |
| "grad_norm": 0.0303955078125, | |
| "learning_rate": 3.482333433166101e-06, | |
| "logits/chosen": 0.2209288775920868, | |
| "logits/rejected": 1.239816427230835, | |
| "logps/chosen": -13.64061450958252, | |
| "logps/rejected": -1329.226806640625, | |
| "loss": 0.2133, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.32054653763771057, | |
| "rewards/margins": 12.724926948547363, | |
| "rewards/rejected": -12.404378890991211, | |
| "step": 870 | |
| }, | |
| { | |
| "epoch": 0.4395055562492196, | |
| "grad_norm": 0.017578125, | |
| "learning_rate": 3.442084620160255e-06, | |
| "logits/chosen": 0.2859000563621521, | |
| "logits/rejected": 1.3617407083511353, | |
| "logps/chosen": -13.033666610717773, | |
| "logps/rejected": -1480.4652099609375, | |
| "loss": 0.2148, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.3004864454269409, | |
| "rewards/margins": 14.408884048461914, | |
| "rewards/rejected": -14.1083984375, | |
| "step": 880 | |
| }, | |
| { | |
| "epoch": 0.4444999375702335, | |
| "grad_norm": 0.033203125, | |
| "learning_rate": 3.4015491576327813e-06, | |
| "logits/chosen": 0.2019362449645996, | |
| "logits/rejected": 1.4212459325790405, | |
| "logps/chosen": -13.03289794921875, | |
| "logps/rejected": -1749.9088134765625, | |
| "loss": 0.2134, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.2995554804801941, | |
| "rewards/margins": 17.093223571777344, | |
| "rewards/rejected": -16.793670654296875, | |
| "step": 890 | |
| }, | |
| { | |
| "epoch": 0.44949431889124736, | |
| "grad_norm": 0.043701171875, | |
| "learning_rate": 3.3607393793705774e-06, | |
| "logits/chosen": 0.18301896750926971, | |
| "logits/rejected": 1.3750990629196167, | |
| "logps/chosen": -13.328268051147461, | |
| "logps/rejected": -1911.580322265625, | |
| "loss": 0.2131, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.3058582544326782, | |
| "rewards/margins": 18.632659912109375, | |
| "rewards/rejected": -18.32680320739746, | |
| "step": 900 | |
| }, | |
| { | |
| "epoch": 0.45448870021226123, | |
| "grad_norm": 0.026611328125, | |
| "learning_rate": 3.319667702627004e-06, | |
| "logits/chosen": 0.251764714717865, | |
| "logits/rejected": 1.379320502281189, | |
| "logps/chosen": -13.333532333374023, | |
| "logps/rejected": -1562.4708251953125, | |
| "loss": 0.2145, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.29535120725631714, | |
| "rewards/margins": 15.228157043457031, | |
| "rewards/rejected": -14.932805061340332, | |
| "step": 910 | |
| }, | |
| { | |
| "epoch": 0.45948308153327505, | |
| "grad_norm": 0.0301513671875, | |
| "learning_rate": 3.2783466243436728e-06, | |
| "logits/chosen": 0.2565325200557709, | |
| "logits/rejected": 1.3139139413833618, | |
| "logps/chosen": -12.679740905761719, | |
| "logps/rejected": -1520.677490234375, | |
| "loss": 0.2148, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.29702773690223694, | |
| "rewards/margins": 14.843530654907227, | |
| "rewards/rejected": -14.546501159667969, | |
| "step": 920 | |
| }, | |
| { | |
| "epoch": 0.4644774628542889, | |
| "grad_norm": 0.032958984375, | |
| "learning_rate": 3.23678871734798e-06, | |
| "logits/chosen": 0.25534436106681824, | |
| "logits/rejected": 1.4121118783950806, | |
| "logps/chosen": -13.0289945602417, | |
| "logps/rejected": -1636.283447265625, | |
| "loss": 0.2134, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.2977290153503418, | |
| "rewards/margins": 15.95555591583252, | |
| "rewards/rejected": -15.657827377319336, | |
| "step": 930 | |
| }, | |
| { | |
| "epoch": 0.4694718441753028, | |
| "grad_norm": 0.017578125, | |
| "learning_rate": 3.1950066265275563e-06, | |
| "logits/chosen": 0.22841012477874756, | |
| "logits/rejected": 1.4126774072647095, | |
| "logps/chosen": -13.12025260925293, | |
| "logps/rejected": -1663.779541015625, | |
| "loss": 0.2137, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.2916621267795563, | |
| "rewards/margins": 16.221864700317383, | |
| "rewards/rejected": -15.930200576782227, | |
| "step": 940 | |
| }, | |
| { | |
| "epoch": 0.4744662254963166, | |
| "grad_norm": 0.0255126953125, | |
| "learning_rate": 3.1530130649827866e-06, | |
| "logits/chosen": 0.22560763359069824, | |
| "logits/rejected": 1.3270902633666992, | |
| "logps/chosen": -12.889676094055176, | |
| "logps/rejected": -1550.6907958984375, | |
| "loss": 0.214, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.2999538779258728, | |
| "rewards/margins": 15.000715255737305, | |
| "rewards/rejected": -14.700759887695312, | |
| "step": 950 | |
| }, | |
| { | |
| "epoch": 0.4794606068173305, | |
| "grad_norm": 0.033203125, | |
| "learning_rate": 3.1108208101585737e-06, | |
| "logits/chosen": 0.2439723014831543, | |
| "logits/rejected": 1.362210988998413, | |
| "logps/chosen": -13.089398384094238, | |
| "logps/rejected": -1693.647216796875, | |
| "loss": 0.2136, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.29442083835601807, | |
| "rewards/margins": 16.52579116821289, | |
| "rewards/rejected": -16.231369018554688, | |
| "step": 960 | |
| }, | |
| { | |
| "epoch": 0.48445498813834437, | |
| "grad_norm": 0.062255859375, | |
| "learning_rate": 3.068442699956526e-06, | |
| "logits/chosen": 0.2077961266040802, | |
| "logits/rejected": 1.3753139972686768, | |
| "logps/chosen": -14.569076538085938, | |
| "logps/rejected": -1670.797119140625, | |
| "loss": 0.2127, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.30582067370414734, | |
| "rewards/margins": 16.229455947875977, | |
| "rewards/rejected": -15.92363452911377, | |
| "step": 970 | |
| }, | |
| { | |
| "epoch": 0.48944936945935824, | |
| "grad_norm": 0.05224609375, | |
| "learning_rate": 3.025891628828754e-06, | |
| "logits/chosen": 0.1842622458934784, | |
| "logits/rejected": 1.2995259761810303, | |
| "logps/chosen": -14.32885456085205, | |
| "logps/rejected": -1653.052001953125, | |
| "loss": 0.2126, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.3057419955730438, | |
| "rewards/margins": 16.055688858032227, | |
| "rewards/rejected": -15.749944686889648, | |
| "step": 980 | |
| }, | |
| { | |
| "epoch": 0.49444375078037206, | |
| "grad_norm": 0.0269775390625, | |
| "learning_rate": 2.983180543854449e-06, | |
| "logits/chosen": 0.19390757381916046, | |
| "logits/rejected": 1.3017512559890747, | |
| "logps/chosen": -13.34800910949707, | |
| "logps/rejected": -1712.0159912109375, | |
| "loss": 0.2135, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.30465009808540344, | |
| "rewards/margins": 16.635692596435547, | |
| "rewards/rejected": -16.331043243408203, | |
| "step": 990 | |
| }, | |
| { | |
| "epoch": 0.49943813210138593, | |
| "grad_norm": 0.0223388671875, | |
| "learning_rate": 2.9403224408004607e-06, | |
| "logits/chosen": 0.23906031250953674, | |
| "logits/rejected": 1.400268316268921, | |
| "logps/chosen": -13.12585735321045, | |
| "logps/rejected": -1704.7308349609375, | |
| "loss": 0.2129, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.3049536347389221, | |
| "rewards/margins": 16.624217987060547, | |
| "rewards/rejected": -16.319265365600586, | |
| "step": 1000 | |
| }, | |
| { | |
| "epoch": 0.49943813210138593, | |
| "eval_logits/chosen": 0.20361633598804474, | |
| "eval_logits/rejected": 1.1080797910690308, | |
| "eval_logps/chosen": -12.131524085998535, | |
| "eval_logps/rejected": -998.1762084960938, | |
| "eval_loss": 0.21242494881153107, | |
| "eval_rewards/accuracies": 1.0, | |
| "eval_rewards/chosen": 0.32524025440216064, | |
| "eval_rewards/margins": 9.676619529724121, | |
| "eval_rewards/rejected": -9.351378440856934, | |
| "eval_runtime": 0.4258, | |
| "eval_samples_per_second": 11.742, | |
| "eval_steps_per_second": 7.045, | |
| "step": 1000 | |
| }, | |
| { | |
| "epoch": 0.5044325134223998, | |
| "grad_norm": 0.0205078125, | |
| "learning_rate": 2.8973303601670537e-06, | |
| "logits/chosen": 0.23553326725959778, | |
| "logits/rejected": 1.356740117073059, | |
| "logps/chosen": -13.091280937194824, | |
| "logps/rejected": -1667.8570556640625, | |
| "loss": 0.2138, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.3002287745475769, | |
| "rewards/margins": 16.279354095458984, | |
| "rewards/rejected": -15.979124069213867, | |
| "step": 1010 | |
| }, | |
| { | |
| "epoch": 0.5094268947434136, | |
| "grad_norm": 0.04833984375, | |
| "learning_rate": 2.8542173832200547e-06, | |
| "logits/chosen": 0.1925448775291443, | |
| "logits/rejected": 1.3025437593460083, | |
| "logps/chosen": -14.501489639282227, | |
| "logps/rejected": -1580.2572021484375, | |
| "loss": 0.2144, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.2990257143974304, | |
| "rewards/margins": 15.303033828735352, | |
| "rewards/rejected": -15.004008293151855, | |
| "step": 1020 | |
| }, | |
| { | |
| "epoch": 0.5144212760644276, | |
| "grad_norm": 0.02587890625, | |
| "learning_rate": 2.810996628010594e-06, | |
| "logits/chosen": 0.2747485637664795, | |
| "logits/rejected": 1.341671109199524, | |
| "logps/chosen": -13.159135818481445, | |
| "logps/rejected": -1436.7039794921875, | |
| "loss": 0.2139, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.30046582221984863, | |
| "rewards/margins": 14.00048542022705, | |
| "rewards/rejected": -13.700021743774414, | |
| "step": 1030 | |
| }, | |
| { | |
| "epoch": 0.5194156573854414, | |
| "grad_norm": 0.0380859375, | |
| "learning_rate": 2.7676812453836617e-06, | |
| "logits/chosen": 0.2172623872756958, | |
| "logits/rejected": 1.389795184135437, | |
| "logps/chosen": -13.409383773803711, | |
| "logps/rejected": -1729.2418212890625, | |
| "loss": 0.2143, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.29702064394950867, | |
| "rewards/margins": 16.85504722595215, | |
| "rewards/rejected": -16.558027267456055, | |
| "step": 1040 | |
| }, | |
| { | |
| "epoch": 0.5244100387064552, | |
| "grad_norm": 0.059814453125, | |
| "learning_rate": 2.724284414976672e-06, | |
| "logits/chosen": 0.194356769323349, | |
| "logits/rejected": 1.3346575498580933, | |
| "logps/chosen": -13.100080490112305, | |
| "logps/rejected": -1775.0599365234375, | |
| "loss": 0.2141, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.3071358799934387, | |
| "rewards/margins": 17.33095359802246, | |
| "rewards/rejected": -17.023818969726562, | |
| "step": 1050 | |
| }, | |
| { | |
| "epoch": 0.5294044200274691, | |
| "grad_norm": 0.0322265625, | |
| "learning_rate": 2.6808193412092823e-06, | |
| "logits/chosen": 0.27043357491493225, | |
| "logits/rejected": 1.2958372831344604, | |
| "logps/chosen": -13.240577697753906, | |
| "logps/rejected": -1308.380615234375, | |
| "loss": 0.212, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.3128414452075958, | |
| "rewards/margins": 12.567632675170898, | |
| "rewards/rejected": -12.254792213439941, | |
| "step": 1060 | |
| }, | |
| { | |
| "epoch": 0.5343988013484829, | |
| "grad_norm": 0.018310546875, | |
| "learning_rate": 2.637299249265659e-06, | |
| "logits/chosen": 0.24779090285301208, | |
| "logits/rejected": 1.3237859010696411, | |
| "logps/chosen": -13.30639934539795, | |
| "logps/rejected": -1560.58740234375, | |
| "loss": 0.2129, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.3085622787475586, | |
| "rewards/margins": 15.089482307434082, | |
| "rewards/rejected": -14.780920028686523, | |
| "step": 1070 | |
| }, | |
| { | |
| "epoch": 0.5393931826694968, | |
| "grad_norm": 0.049560546875, | |
| "learning_rate": 2.5937373810704352e-06, | |
| "logits/chosen": 0.20865114033222198, | |
| "logits/rejected": 1.3283964395523071, | |
| "logps/chosen": -12.944803237915039, | |
| "logps/rejected": -1628.0531005859375, | |
| "loss": 0.213, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.3075375556945801, | |
| "rewards/margins": 15.755208969116211, | |
| "rewards/rejected": -15.447671890258789, | |
| "step": 1080 | |
| }, | |
| { | |
| "epoch": 0.5443875639905107, | |
| "grad_norm": 0.0247802734375, | |
| "learning_rate": 2.550146991259565e-06, | |
| "logits/chosen": 0.2642405331134796, | |
| "logits/rejected": 1.330570936203003, | |
| "logps/chosen": -12.811630249023438, | |
| "logps/rejected": -1518.474609375, | |
| "loss": 0.213, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.309398889541626, | |
| "rewards/margins": 14.794235229492188, | |
| "rewards/rejected": -14.484835624694824, | |
| "step": 1090 | |
| }, | |
| { | |
| "epoch": 0.5493819453115245, | |
| "grad_norm": 0.0252685546875, | |
| "learning_rate": 2.5065413431473196e-06, | |
| "logits/chosen": 0.22386522591114044, | |
| "logits/rejected": 1.3922302722930908, | |
| "logps/chosen": -13.141294479370117, | |
| "logps/rejected": -1650.009765625, | |
| "loss": 0.2129, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.30252230167388916, | |
| "rewards/margins": 16.12454605102539, | |
| "rewards/rejected": -15.822023391723633, | |
| "step": 1100 | |
| }, | |
| { | |
| "epoch": 0.5543763266325384, | |
| "grad_norm": 0.017822265625, | |
| "learning_rate": 2.462933704690635e-06, | |
| "logits/chosen": 0.23435406386852264, | |
| "logits/rejected": 1.2522070407867432, | |
| "logps/chosen": -13.335357666015625, | |
| "logps/rejected": -1444.448974609375, | |
| "loss": 0.2132, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.3048686385154724, | |
| "rewards/margins": 13.933290481567383, | |
| "rewards/rejected": -13.62842082977295, | |
| "step": 1110 | |
| }, | |
| { | |
| "epoch": 0.5593707079535523, | |
| "grad_norm": 0.0159912109375, | |
| "learning_rate": 2.4193373444520558e-06, | |
| "logits/chosen": 0.23952054977416992, | |
| "logits/rejected": 1.466230869293213, | |
| "logps/chosen": -13.09483528137207, | |
| "logps/rejected": -1737.9251708984375, | |
| "loss": 0.213, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.3013126254081726, | |
| "rewards/margins": 16.979856491088867, | |
| "rewards/rejected": -16.67854118347168, | |
| "step": 1120 | |
| }, | |
| { | |
| "epoch": 0.5643650892745661, | |
| "grad_norm": 0.0277099609375, | |
| "learning_rate": 2.3757655275624826e-06, | |
| "logits/chosen": 0.20145916938781738, | |
| "logits/rejected": 1.4154694080352783, | |
| "logps/chosen": -12.834500312805176, | |
| "logps/rejected": -1683.4241943359375, | |
| "loss": 0.2142, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.29736918210983276, | |
| "rewards/margins": 16.437068939208984, | |
| "rewards/rejected": -16.13970184326172, | |
| "step": 1130 | |
| }, | |
| { | |
| "epoch": 0.56935947059558, | |
| "grad_norm": 0.0235595703125, | |
| "learning_rate": 2.3322315116849747e-06, | |
| "logits/chosen": 0.18402531743049622, | |
| "logits/rejected": 1.3126466274261475, | |
| "logps/chosen": -13.314038276672363, | |
| "logps/rejected": -1694.0882568359375, | |
| "loss": 0.2131, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.3090851306915283, | |
| "rewards/margins": 16.457656860351562, | |
| "rewards/rejected": -16.14857292175293, | |
| "step": 1140 | |
| }, | |
| { | |
| "epoch": 0.5743538519165938, | |
| "grad_norm": 0.0208740234375, | |
| "learning_rate": 2.2887485429808213e-06, | |
| "logits/chosen": 0.24247586727142334, | |
| "logits/rejected": 1.3415312767028809, | |
| "logps/chosen": -13.364187240600586, | |
| "logps/rejected": -1531.287353515625, | |
| "loss": 0.2132, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.31205669045448303, | |
| "rewards/margins": 14.862896919250488, | |
| "rewards/rejected": -14.5508394241333, | |
| "step": 1150 | |
| }, | |
| { | |
| "epoch": 0.5793482332376076, | |
| "grad_norm": 0.0257568359375, | |
| "learning_rate": 2.245329852079109e-06, | |
| "logits/chosen": 0.2564612329006195, | |
| "logits/rejected": 1.3275741338729858, | |
| "logps/chosen": -12.749259948730469, | |
| "logps/rejected": -1410.6912841796875, | |
| "loss": 0.2134, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.3032976984977722, | |
| "rewards/margins": 13.754470825195312, | |
| "rewards/rejected": -13.451173782348633, | |
| "step": 1160 | |
| }, | |
| { | |
| "epoch": 0.5843426145586216, | |
| "grad_norm": 0.02099609375, | |
| "learning_rate": 2.2019886500510197e-06, | |
| "logits/chosen": 0.234290212392807, | |
| "logits/rejected": 1.3947325944900513, | |
| "logps/chosen": -12.72685432434082, | |
| "logps/rejected": -1736.8687744140625, | |
| "loss": 0.2138, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.2989721894264221, | |
| "rewards/margins": 16.979671478271484, | |
| "rewards/rejected": -16.68069839477539, | |
| "step": 1170 | |
| }, | |
| { | |
| "epoch": 0.5893369958796354, | |
| "grad_norm": 0.037353515625, | |
| "learning_rate": 2.1587381243900777e-06, | |
| "logits/chosen": 0.26597389578819275, | |
| "logits/rejected": 1.3139584064483643, | |
| "logps/chosen": -14.154109001159668, | |
| "logps/rejected": -1483.2686767578125, | |
| "loss": 0.2125, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.3113531470298767, | |
| "rewards/margins": 14.383076667785645, | |
| "rewards/rejected": -14.071722030639648, | |
| "step": 1180 | |
| }, | |
| { | |
| "epoch": 0.5943313772006493, | |
| "grad_norm": 0.0233154296875, | |
| "learning_rate": 2.115591434999573e-06, | |
| "logits/chosen": 0.277686208486557, | |
| "logits/rejected": 1.401039719581604, | |
| "logps/chosen": -12.76582145690918, | |
| "logps/rejected": -1506.6636962890625, | |
| "loss": 0.2132, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.30175262689590454, | |
| "rewards/margins": 14.693206787109375, | |
| "rewards/rejected": -14.391454696655273, | |
| "step": 1190 | |
| }, | |
| { | |
| "epoch": 0.5993257585216631, | |
| "grad_norm": 0.0294189453125, | |
| "learning_rate": 2.0725617101883726e-06, | |
| "logits/chosen": 0.25775861740112305, | |
| "logits/rejected": 1.345365285873413, | |
| "logps/chosen": -12.69609260559082, | |
| "logps/rejected": -1601.963623046875, | |
| "loss": 0.2147, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.2992851138114929, | |
| "rewards/margins": 15.617083549499512, | |
| "rewards/rejected": -15.317797660827637, | |
| "step": 1200 | |
| }, | |
| { | |
| "epoch": 0.604320139842677, | |
| "grad_norm": 0.09033203125, | |
| "learning_rate": 2.0296620426763545e-06, | |
| "logits/chosen": 0.14509257674217224, | |
| "logits/rejected": 1.3586305379867554, | |
| "logps/chosen": -13.326802253723145, | |
| "logps/rejected": -1893.167724609375, | |
| "loss": 0.2128, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.31738612055778503, | |
| "rewards/margins": 18.294166564941406, | |
| "rewards/rejected": -17.97677993774414, | |
| "step": 1210 | |
| }, | |
| { | |
| "epoch": 0.6093145211636909, | |
| "grad_norm": 0.0206298828125, | |
| "learning_rate": 1.9869054856106628e-06, | |
| "logits/chosen": 0.2093639373779297, | |
| "logits/rejected": 1.3707678318023682, | |
| "logps/chosen": -13.017126083374023, | |
| "logps/rejected": -1714.967529296875, | |
| "loss": 0.213, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.305789053440094, | |
| "rewards/margins": 16.755239486694336, | |
| "rewards/rejected": -16.449451446533203, | |
| "step": 1220 | |
| }, | |
| { | |
| "epoch": 0.6143089024847047, | |
| "grad_norm": 0.027587890625, | |
| "learning_rate": 1.9443050485940118e-06, | |
| "logits/chosen": 0.29796817898750305, | |
| "logits/rejected": 1.4095687866210938, | |
| "logps/chosen": -13.055410385131836, | |
| "logps/rejected": -1501.1917724609375, | |
| "loss": 0.2132, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.30659452080726624, | |
| "rewards/margins": 14.630342483520508, | |
| "rewards/rejected": -14.323748588562012, | |
| "step": 1230 | |
| }, | |
| { | |
| "epoch": 0.6193032838057185, | |
| "grad_norm": 0.0205078125, | |
| "learning_rate": 1.9018736937262271e-06, | |
| "logits/chosen": 0.20551720261573792, | |
| "logits/rejected": 1.3722885847091675, | |
| "logps/chosen": -13.263589859008789, | |
| "logps/rejected": -1644.015380859375, | |
| "loss": 0.213, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.3152340054512024, | |
| "rewards/margins": 15.940661430358887, | |
| "rewards/rejected": -15.62542724609375, | |
| "step": 1240 | |
| }, | |
| { | |
| "epoch": 0.6242976651267325, | |
| "grad_norm": 0.025146484375, | |
| "learning_rate": 1.859624331660253e-06, | |
| "logits/chosen": 0.19998934864997864, | |
| "logits/rejected": 1.380974531173706, | |
| "logps/chosen": -12.843725204467773, | |
| "logps/rejected": -1903.5989990234375, | |
| "loss": 0.2149, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.293190598487854, | |
| "rewards/margins": 18.591859817504883, | |
| "rewards/rejected": -18.298667907714844, | |
| "step": 1250 | |
| }, | |
| { | |
| "epoch": 0.6292920464477463, | |
| "grad_norm": 0.0184326171875, | |
| "learning_rate": 1.817569817673806e-06, | |
| "logits/chosen": 0.20200982689857483, | |
| "logits/rejected": 1.3313050270080566, | |
| "logps/chosen": -13.815347671508789, | |
| "logps/rejected": -1682.6148681640625, | |
| "loss": 0.2137, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.3007175624370575, | |
| "rewards/margins": 16.28915786743164, | |
| "rewards/rejected": -15.988439559936523, | |
| "step": 1260 | |
| }, | |
| { | |
| "epoch": 0.6342864277687601, | |
| "grad_norm": 0.0264892578125, | |
| "learning_rate": 1.7757229477578824e-06, | |
| "logits/chosen": 0.2238602340221405, | |
| "logits/rejected": 1.3182499408721924, | |
| "logps/chosen": -13.298685073852539, | |
| "logps/rejected": -1689.486328125, | |
| "loss": 0.2124, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.31393638253211975, | |
| "rewards/margins": 16.413013458251953, | |
| "rewards/rejected": -16.099077224731445, | |
| "step": 1270 | |
| }, | |
| { | |
| "epoch": 0.639280809089774, | |
| "grad_norm": 0.0120849609375, | |
| "learning_rate": 1.7340964547232993e-06, | |
| "logits/chosen": 0.23566928505897522, | |
| "logits/rejected": 1.3794082403182983, | |
| "logps/chosen": -12.66304874420166, | |
| "logps/rejected": -1540.5882568359375, | |
| "loss": 0.2122, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.30794182419776917, | |
| "rewards/margins": 15.042073249816895, | |
| "rewards/rejected": -14.734130859375, | |
| "step": 1280 | |
| }, | |
| { | |
| "epoch": 0.6442751904107878, | |
| "grad_norm": 0.0233154296875, | |
| "learning_rate": 1.6927030043264656e-06, | |
| "logits/chosen": 0.29966339468955994, | |
| "logits/rejected": 1.3575140237808228, | |
| "logps/chosen": -12.536134719848633, | |
| "logps/rejected": -1450.3671875, | |
| "loss": 0.2131, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.3003261685371399, | |
| "rewards/margins": 14.137969970703125, | |
| "rewards/rejected": -13.83764362335205, | |
| "step": 1290 | |
| }, | |
| { | |
| "epoch": 0.6492695717318018, | |
| "grad_norm": 0.01904296875, | |
| "learning_rate": 1.6515551914155522e-06, | |
| "logits/chosen": 0.21864613890647888, | |
| "logits/rejected": 1.2960518598556519, | |
| "logps/chosen": -13.931121826171875, | |
| "logps/rejected": -1731.0433349609375, | |
| "loss": 0.2135, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.29954832792282104, | |
| "rewards/margins": 16.782882690429688, | |
| "rewards/rejected": -16.483333587646484, | |
| "step": 1300 | |
| }, | |
| { | |
| "epoch": 0.6542639530528156, | |
| "grad_norm": 0.027587890625, | |
| "learning_rate": 1.6106655360982376e-06, | |
| "logits/chosen": 0.11391136795282364, | |
| "logits/rejected": 1.1829859018325806, | |
| "logps/chosen": -13.162788391113281, | |
| "logps/rejected": -1799.5191650390625, | |
| "loss": 0.2121, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.3114808201789856, | |
| "rewards/margins": 17.426467895507812, | |
| "rewards/rejected": -17.114986419677734, | |
| "step": 1310 | |
| }, | |
| { | |
| "epoch": 0.6592583343738294, | |
| "grad_norm": 0.126953125, | |
| "learning_rate": 1.570046479932196e-06, | |
| "logits/chosen": 0.25235381722450256, | |
| "logits/rejected": 1.2799979448318481, | |
| "logps/chosen": -13.60442066192627, | |
| "logps/rejected": -1395.318115234375, | |
| "loss": 0.213, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.3034665286540985, | |
| "rewards/margins": 13.48466968536377, | |
| "rewards/rejected": -13.18120288848877, | |
| "step": 1320 | |
| }, | |
| { | |
| "epoch": 0.6642527156948433, | |
| "grad_norm": 0.0306396484375, | |
| "learning_rate": 1.5297103821394876e-06, | |
| "logits/chosen": 0.2604614198207855, | |
| "logits/rejected": 1.4426438808441162, | |
| "logps/chosen": -12.831866264343262, | |
| "logps/rejected": -1725.9541015625, | |
| "loss": 0.2136, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.30009177327156067, | |
| "rewards/margins": 16.85196876525879, | |
| "rewards/rejected": -16.551877975463867, | |
| "step": 1330 | |
| }, | |
| { | |
| "epoch": 0.6692470970158572, | |
| "grad_norm": 0.0291748046875, | |
| "learning_rate": 1.489669515845995e-06, | |
| "logits/chosen": 0.19801196455955505, | |
| "logits/rejected": 1.246242642402649, | |
| "logps/chosen": -12.573989868164062, | |
| "logps/rejected": -1484.0341796875, | |
| "loss": 0.2136, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.3015730679035187, | |
| "rewards/margins": 14.43646240234375, | |
| "rewards/rejected": -14.134889602661133, | |
| "step": 1340 | |
| }, | |
| { | |
| "epoch": 0.674241478336871, | |
| "grad_norm": 0.020751953125, | |
| "learning_rate": 1.449936064347065e-06, | |
| "logits/chosen": 0.24275144934654236, | |
| "logits/rejected": 1.3506710529327393, | |
| "logps/chosen": -12.805200576782227, | |
| "logps/rejected": -1675.633544921875, | |
| "loss": 0.2131, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.3000423312187195, | |
| "rewards/margins": 16.352081298828125, | |
| "rewards/rejected": -16.052040100097656, | |
| "step": 1350 | |
| }, | |
| { | |
| "epoch": 0.6792358596578849, | |
| "grad_norm": 0.01483154296875, | |
| "learning_rate": 1.4105221174004771e-06, | |
| "logits/chosen": 0.18348607420921326, | |
| "logits/rejected": 1.3507370948791504, | |
| "logps/chosen": -13.73768424987793, | |
| "logps/rejected": -1898.157470703125, | |
| "loss": 0.2124, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.309729665517807, | |
| "rewards/margins": 18.553356170654297, | |
| "rewards/rejected": -18.24362564086914, | |
| "step": 1360 | |
| }, | |
| { | |
| "epoch": 0.6842302409788987, | |
| "grad_norm": 0.03515625, | |
| "learning_rate": 1.3714396675478714e-06, | |
| "logits/chosen": 0.29044079780578613, | |
| "logits/rejected": 1.34650719165802, | |
| "logps/chosen": -12.877673149108887, | |
| "logps/rejected": -1523.441162109375, | |
| "loss": 0.2132, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.2951946556568146, | |
| "rewards/margins": 14.83436107635498, | |
| "rewards/rejected": -14.539166450500488, | |
| "step": 1370 | |
| }, | |
| { | |
| "epoch": 0.6892246222999125, | |
| "grad_norm": 0.0115966796875, | |
| "learning_rate": 1.332700606465766e-06, | |
| "logits/chosen": 0.18918542563915253, | |
| "logits/rejected": 1.3702523708343506, | |
| "logps/chosen": -13.452165603637695, | |
| "logps/rejected": -1564.6370849609375, | |
| "loss": 0.2125, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.3073347210884094, | |
| "rewards/margins": 15.239773750305176, | |
| "rewards/rejected": -14.9324369430542, | |
| "step": 1380 | |
| }, | |
| { | |
| "epoch": 0.6942190036209265, | |
| "grad_norm": 0.034423828125, | |
| "learning_rate": 1.294316721347254e-06, | |
| "logits/chosen": 0.23732297122478485, | |
| "logits/rejected": 1.3234728574752808, | |
| "logps/chosen": -13.713842391967773, | |
| "logps/rejected": -1537.1632080078125, | |
| "loss": 0.2131, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.30026909708976746, | |
| "rewards/margins": 14.980550765991211, | |
| "rewards/rejected": -14.680282592773438, | |
| "step": 1390 | |
| }, | |
| { | |
| "epoch": 0.6992133849419403, | |
| "grad_norm": 0.07080078125, | |
| "learning_rate": 1.2562996913154952e-06, | |
| "logits/chosen": 0.150472030043602, | |
| "logits/rejected": 1.4298745393753052, | |
| "logps/chosen": -12.643513679504395, | |
| "logps/rejected": -2120.697509765625, | |
| "loss": 0.2133, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.30036434531211853, | |
| "rewards/margins": 20.737525939941406, | |
| "rewards/rejected": -20.43716049194336, | |
| "step": 1400 | |
| }, | |
| { | |
| "epoch": 0.7042077662629542, | |
| "grad_norm": 0.034423828125, | |
| "learning_rate": 1.2186610838700958e-06, | |
| "logits/chosen": 0.30068179965019226, | |
| "logits/rejected": 1.335126280784607, | |
| "logps/chosen": -12.845601081848145, | |
| "logps/rejected": -1345.099365234375, | |
| "loss": 0.2143, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.2946023643016815, | |
| "rewards/margins": 13.083051681518555, | |
| "rewards/rejected": -12.788450241088867, | |
| "step": 1410 | |
| }, | |
| { | |
| "epoch": 0.709202147583968, | |
| "grad_norm": 0.033447265625, | |
| "learning_rate": 1.1814123513674465e-06, | |
| "logits/chosen": 0.18157488107681274, | |
| "logits/rejected": 1.380326509475708, | |
| "logps/chosen": -13.374841690063477, | |
| "logps/rejected": -1741.3206787109375, | |
| "loss": 0.2124, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.30905359983444214, | |
| "rewards/margins": 16.993389129638672, | |
| "rewards/rejected": -16.684337615966797, | |
| "step": 1420 | |
| }, | |
| { | |
| "epoch": 0.7141965289049819, | |
| "grad_norm": 0.036865234375, | |
| "learning_rate": 1.1445648275360925e-06, | |
| "logits/chosen": 0.19126050174236298, | |
| "logits/rejected": 1.4904680252075195, | |
| "logps/chosen": -13.341898918151855, | |
| "logps/rejected": -1982.956298828125, | |
| "loss": 0.2139, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.30447131395339966, | |
| "rewards/margins": 19.409332275390625, | |
| "rewards/rejected": -19.104862213134766, | |
| "step": 1430 | |
| }, | |
| { | |
| "epoch": 0.7191909102259958, | |
| "grad_norm": 0.03955078125, | |
| "learning_rate": 1.1081297240282077e-06, | |
| "logits/chosen": 0.2438248097896576, | |
| "logits/rejected": 1.3988474607467651, | |
| "logps/chosen": -13.187724113464355, | |
| "logps/rejected": -1571.8028564453125, | |
| "loss": 0.2139, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.29910221695899963, | |
| "rewards/margins": 15.328478813171387, | |
| "rewards/rejected": -15.029376029968262, | |
| "step": 1440 | |
| }, | |
| { | |
| "epoch": 0.7241852915470096, | |
| "grad_norm": 0.0235595703125, | |
| "learning_rate": 1.0721181270082061e-06, | |
| "logits/chosen": 0.20241305232048035, | |
| "logits/rejected": 1.308318018913269, | |
| "logps/chosen": -12.698432922363281, | |
| "logps/rejected": -1740.4573974609375, | |
| "loss": 0.2125, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.313620924949646, | |
| "rewards/margins": 16.896198272705078, | |
| "rewards/rejected": -16.582576751708984, | |
| "step": 1450 | |
| }, | |
| { | |
| "epoch": 0.7291796728680234, | |
| "grad_norm": 0.0164794921875, | |
| "learning_rate": 1.0365409937795385e-06, | |
| "logits/chosen": 0.20683518052101135, | |
| "logits/rejected": 1.2756832838058472, | |
| "logps/chosen": -13.767419815063477, | |
| "logps/rejected": -1547.2147216796875, | |
| "loss": 0.2129, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.3099841773509979, | |
| "rewards/margins": 14.946782112121582, | |
| "rewards/rejected": -14.636796951293945, | |
| "step": 1460 | |
| }, | |
| { | |
| "epoch": 0.7341740541890374, | |
| "grad_norm": 0.0263671875, | |
| "learning_rate": 1.0014091494506962e-06, | |
| "logits/chosen": 0.17677463591098785, | |
| "logits/rejected": 1.3411346673965454, | |
| "logps/chosen": -13.297311782836914, | |
| "logps/rejected": -1913.2984619140625, | |
| "loss": 0.2128, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.3021236062049866, | |
| "rewards/margins": 18.699064254760742, | |
| "rewards/rejected": -18.396942138671875, | |
| "step": 1470 | |
| }, | |
| { | |
| "epoch": 0.7391684355100512, | |
| "grad_norm": 0.033203125, | |
| "learning_rate": 9.667332836414368e-07, | |
| "logits/chosen": 0.15931569039821625, | |
| "logits/rejected": 1.2274577617645264, | |
| "logps/chosen": -13.343734741210938, | |
| "logps/rejected": -1609.459228515625, | |
| "loss": 0.2133, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.3135462999343872, | |
| "rewards/margins": 15.410505294799805, | |
| "rewards/rejected": -15.096961975097656, | |
| "step": 1480 | |
| }, | |
| { | |
| "epoch": 0.744162816831065, | |
| "grad_norm": 0.013427734375, | |
| "learning_rate": 9.325239472302422e-07, | |
| "logits/chosen": 0.25666847825050354, | |
| "logits/rejected": 1.4118614196777344, | |
| "logps/chosen": -13.085573196411133, | |
| "logps/rejected": -1693.067626953125, | |
| "loss": 0.2127, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.30241408944129944, | |
| "rewards/margins": 16.529926300048828, | |
| "rewards/rejected": -16.227514266967773, | |
| "step": 1490 | |
| }, | |
| { | |
| "epoch": 0.7491571981520789, | |
| "grad_norm": 0.032958984375, | |
| "learning_rate": 8.987915491439844e-07, | |
| "logits/chosen": 0.2501397132873535, | |
| "logits/rejected": 1.383455514907837, | |
| "logps/chosen": -12.903341293334961, | |
| "logps/rejected": -1736.0384521484375, | |
| "loss": 0.2143, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.3011986017227173, | |
| "rewards/margins": 16.931396484375, | |
| "rewards/rejected": -16.630199432373047, | |
| "step": 1500 | |
| }, | |
| { | |
| "epoch": 0.7541515794730927, | |
| "grad_norm": 0.021240234375, | |
| "learning_rate": 8.655463531907823e-07, | |
| "logits/chosen": 0.2224547117948532, | |
| "logits/rejected": 1.3173682689666748, | |
| "logps/chosen": -13.032022476196289, | |
| "logps/rejected": -1771.166259765625, | |
| "loss": 0.2132, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.29998722672462463, | |
| "rewards/margins": 17.284591674804688, | |
| "rewards/rejected": -16.984607696533203, | |
| "step": 1510 | |
| }, | |
| { | |
| "epoch": 0.7591459607941067, | |
| "grad_norm": 0.014404296875, | |
| "learning_rate": 8.327984749370227e-07, | |
| "logits/chosen": 0.2447456419467926, | |
| "logits/rejected": 1.344362497329712, | |
| "logps/chosen": -12.957304000854492, | |
| "logps/rejected": -1557.052978515625, | |
| "loss": 0.2142, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.29349425435066223, | |
| "rewards/margins": 15.1736478805542, | |
| "rewards/rejected": -14.880154609680176, | |
| "step": 1520 | |
| }, | |
| { | |
| "epoch": 0.7641403421151205, | |
| "grad_norm": 0.0218505859375, | |
| "learning_rate": 8.005578786294782e-07, | |
| "logits/chosen": 0.1744759976863861, | |
| "logits/rejected": 1.3996227979660034, | |
| "logps/chosen": -13.042366027832031, | |
| "logps/rejected": -1864.2431640625, | |
| "loss": 0.2122, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.30568575859069824, | |
| "rewards/margins": 18.201250076293945, | |
| "rewards/rejected": -17.89556312561035, | |
| "step": 1530 | |
| }, | |
| { | |
| "epoch": 0.7691347234361343, | |
| "grad_norm": 0.012939453125, | |
| "learning_rate": 7.688343741634702e-07, | |
| "logits/chosen": 0.22156497836112976, | |
| "logits/rejected": 1.3012608289718628, | |
| "logps/chosen": -12.66821575164795, | |
| "logps/rejected": -1628.963134765625, | |
| "loss": 0.2137, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.2943740785121918, | |
| "rewards/margins": 15.8633394241333, | |
| "rewards/rejected": -15.568964958190918, | |
| "step": 1540 | |
| }, | |
| { | |
| "epoch": 0.7741291047571482, | |
| "grad_norm": 0.03662109375, | |
| "learning_rate": 7.376376140980001e-07, | |
| "logits/chosen": 0.1970866173505783, | |
| "logits/rejected": 1.2925320863723755, | |
| "logps/chosen": -12.667881965637207, | |
| "logps/rejected": -1566.008056640625, | |
| "loss": 0.2128, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.3171369433403015, | |
| "rewards/margins": 15.188334465026855, | |
| "rewards/rejected": -14.871198654174805, | |
| "step": 1550 | |
| }, | |
| { | |
| "epoch": 0.7791234860781621, | |
| "grad_norm": 0.0625, | |
| "learning_rate": 7.069770907187465e-07, | |
| "logits/chosen": 0.20419040322303772, | |
| "logits/rejected": 1.306980013847351, | |
| "logps/chosen": -13.543539047241211, | |
| "logps/rejected": -1506.096923828125, | |
| "loss": 0.2137, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.3025529384613037, | |
| "rewards/margins": 14.58825397491455, | |
| "rewards/rejected": -14.285697937011719, | |
| "step": 1560 | |
| }, | |
| { | |
| "epoch": 0.7841178673991759, | |
| "grad_norm": 0.01483154296875, | |
| "learning_rate": 6.768621331498371e-07, | |
| "logits/chosen": 0.22659845650196075, | |
| "logits/rejected": 1.3488702774047852, | |
| "logps/chosen": -12.837379455566406, | |
| "logps/rejected": -1573.134033203125, | |
| "loss": 0.2131, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.29855865240097046, | |
| "rewards/margins": 15.335705757141113, | |
| "rewards/rejected": -15.037145614624023, | |
| "step": 1570 | |
| }, | |
| { | |
| "epoch": 0.7891122487201898, | |
| "grad_norm": 0.049560546875, | |
| "learning_rate": 6.473019045152593e-07, | |
| "logits/chosen": 0.22067594528198242, | |
| "logits/rejected": 1.39100980758667, | |
| "logps/chosen": -12.964533805847168, | |
| "logps/rejected": -1819.2757568359375, | |
| "loss": 0.2124, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.3059755563735962, | |
| "rewards/margins": 17.78072738647461, | |
| "rewards/rejected": -17.474750518798828, | |
| "step": 1580 | |
| }, | |
| { | |
| "epoch": 0.7941066300412036, | |
| "grad_norm": 0.0159912109375, | |
| "learning_rate": 6.183053991507818e-07, | |
| "logits/chosen": 0.18515101075172424, | |
| "logits/rejected": 1.2657784223556519, | |
| "logps/chosen": -13.701101303100586, | |
| "logps/rejected": -1632.0228271484375, | |
| "loss": 0.2135, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.30029112100601196, | |
| "rewards/margins": 15.897903442382812, | |
| "rewards/rejected": -15.597612380981445, | |
| "step": 1590 | |
| }, | |
| { | |
| "epoch": 0.7991010113622175, | |
| "grad_norm": 0.035400390625, | |
| "learning_rate": 5.898814398672376e-07, | |
| "logits/chosen": 0.2673841416835785, | |
| "logits/rejected": 1.404524326324463, | |
| "logps/chosen": -12.89787769317627, | |
| "logps/rejected": -1488.13818359375, | |
| "loss": 0.2137, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.2999451160430908, | |
| "rewards/margins": 14.503445625305176, | |
| "rewards/rejected": -14.203500747680664, | |
| "step": 1600 | |
| }, | |
| { | |
| "epoch": 0.8040953926832314, | |
| "grad_norm": 0.028564453125, | |
| "learning_rate": 5.620386752659912e-07, | |
| "logits/chosen": 0.20275497436523438, | |
| "logits/rejected": 1.2962656021118164, | |
| "logps/chosen": -14.060361862182617, | |
| "logps/rejected": -1563.873046875, | |
| "loss": 0.2123, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.3067697286605835, | |
| "rewards/margins": 15.22687816619873, | |
| "rewards/rejected": -14.920109748840332, | |
| "step": 1610 | |
| }, | |
| { | |
| "epoch": 0.8090897740042452, | |
| "grad_norm": 0.031494140625, | |
| "learning_rate": 5.347855771074157e-07, | |
| "logits/chosen": 0.22789278626441956, | |
| "logits/rejected": 1.4333293437957764, | |
| "logps/chosen": -12.939372062683105, | |
| "logps/rejected": -1751.370361328125, | |
| "loss": 0.2136, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.2999444007873535, | |
| "rewards/margins": 17.115280151367188, | |
| "rewards/rejected": -16.81533432006836, | |
| "step": 1620 | |
| }, | |
| { | |
| "epoch": 0.8140841553252591, | |
| "grad_norm": 0.031005859375, | |
| "learning_rate": 5.081304377331786e-07, | |
| "logits/chosen": 0.27506810426712036, | |
| "logits/rejected": 1.2999309301376343, | |
| "logps/chosen": -13.067828178405762, | |
| "logps/rejected": -1445.669677734375, | |
| "loss": 0.2139, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.30137357115745544, | |
| "rewards/margins": 14.080915451049805, | |
| "rewards/rejected": -13.779541015625, | |
| "step": 1630 | |
| }, | |
| { | |
| "epoch": 0.819078536646273, | |
| "grad_norm": 0.03369140625, | |
| "learning_rate": 4.820813675431186e-07, | |
| "logits/chosen": 0.15463611483573914, | |
| "logits/rejected": 1.3889650106430054, | |
| "logps/chosen": -13.349421501159668, | |
| "logps/rejected": -1812.4769287109375, | |
| "loss": 0.2136, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.3092433512210846, | |
| "rewards/margins": 17.648366928100586, | |
| "rewards/rejected": -17.33912467956543, | |
| "step": 1640 | |
| }, | |
| { | |
| "epoch": 0.8240729179672868, | |
| "grad_norm": 0.0242919921875, | |
| "learning_rate": 4.5664629252747865e-07, | |
| "logits/chosen": 0.21123354136943817, | |
| "logits/rejected": 1.3822557926177979, | |
| "logps/chosen": -12.89905834197998, | |
| "logps/rejected": -1771.5474853515625, | |
| "loss": 0.213, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.29551658034324646, | |
| "rewards/margins": 17.291709899902344, | |
| "rewards/rejected": -16.99619483947754, | |
| "step": 1650 | |
| }, | |
| { | |
| "epoch": 0.8290672992883007, | |
| "grad_norm": 0.021484375, | |
| "learning_rate": 4.3183295185525746e-07, | |
| "logits/chosen": 0.17760224640369415, | |
| "logits/rejected": 1.320516586303711, | |
| "logps/chosen": -12.863115310668945, | |
| "logps/rejected": -1791.6168212890625, | |
| "loss": 0.2129, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.3116030693054199, | |
| "rewards/margins": 17.404626846313477, | |
| "rewards/rejected": -17.09302520751953, | |
| "step": 1660 | |
| }, | |
| { | |
| "epoch": 0.8340616806093145, | |
| "grad_norm": 0.0274658203125, | |
| "learning_rate": 4.0764889551939773e-07, | |
| "logits/chosen": 0.19689543545246124, | |
| "logits/rejected": 1.3633372783660889, | |
| "logps/chosen": -12.999624252319336, | |
| "logps/rejected": -1783.937255859375, | |
| "loss": 0.2131, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.30732038617134094, | |
| "rewards/margins": 17.336694717407227, | |
| "rewards/rejected": -17.029375076293945, | |
| "step": 1670 | |
| }, | |
| { | |
| "epoch": 0.8390560619303283, | |
| "grad_norm": 0.02392578125, | |
| "learning_rate": 3.8410148203953916e-07, | |
| "logits/chosen": 0.20565366744995117, | |
| "logits/rejected": 1.2732315063476562, | |
| "logps/chosen": -13.205337524414062, | |
| "logps/rejected": -1707.686279296875, | |
| "loss": 0.2141, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.29469722509384155, | |
| "rewards/margins": 16.513946533203125, | |
| "rewards/rejected": -16.219249725341797, | |
| "step": 1680 | |
| }, | |
| { | |
| "epoch": 0.8440504432513423, | |
| "grad_norm": 0.01904296875, | |
| "learning_rate": 3.611978762230306e-07, | |
| "logits/chosen": 0.2300119698047638, | |
| "logits/rejected": 1.3706471920013428, | |
| "logps/chosen": -12.709399223327637, | |
| "logps/rejected": -1650.5601806640625, | |
| "loss": 0.2126, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.30553561449050903, | |
| "rewards/margins": 16.097576141357422, | |
| "rewards/rejected": -15.79203987121582, | |
| "step": 1690 | |
| }, | |
| { | |
| "epoch": 0.8490448245723561, | |
| "grad_norm": 0.031982421875, | |
| "learning_rate": 3.389450469848821e-07, | |
| "logits/chosen": 0.26923322677612305, | |
| "logits/rejected": 1.3886299133300781, | |
| "logps/chosen": -12.681255340576172, | |
| "logps/rejected": -1633.20361328125, | |
| "loss": 0.2138, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.3012009263038635, | |
| "rewards/margins": 15.944944381713867, | |
| "rewards/rejected": -15.643745422363281, | |
| "step": 1700 | |
| }, | |
| { | |
| "epoch": 0.85403920589337, | |
| "grad_norm": 0.015869140625, | |
| "learning_rate": 3.173497652273241e-07, | |
| "logits/chosen": 0.22611021995544434, | |
| "logits/rejected": 1.4598513841629028, | |
| "logps/chosen": -13.163076400756836, | |
| "logps/rejected": -1702.979736328125, | |
| "loss": 0.2126, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.3090182840824127, | |
| "rewards/margins": 16.64162826538086, | |
| "rewards/rejected": -16.332609176635742, | |
| "step": 1710 | |
| }, | |
| { | |
| "epoch": 0.8590335872143838, | |
| "grad_norm": 0.0257568359375, | |
| "learning_rate": 2.964186017796153e-07, | |
| "logits/chosen": 0.23432429134845734, | |
| "logits/rejected": 1.3954203128814697, | |
| "logps/chosen": -12.975980758666992, | |
| "logps/rejected": -1669.3160400390625, | |
| "loss": 0.2133, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.29741281270980835, | |
| "rewards/margins": 16.287899017333984, | |
| "rewards/rejected": -15.990484237670898, | |
| "step": 1720 | |
| }, | |
| { | |
| "epoch": 0.8640279685353977, | |
| "grad_norm": 0.020263671875, | |
| "learning_rate": 2.761579253987226e-07, | |
| "logits/chosen": 0.24720034003257751, | |
| "logits/rejected": 1.284588098526001, | |
| "logps/chosen": -13.461567878723145, | |
| "logps/rejected": -1476.7037353515625, | |
| "loss": 0.2131, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.3023452162742615, | |
| "rewards/margins": 14.363668441772461, | |
| "rewards/rejected": -14.061323165893555, | |
| "step": 1730 | |
| }, | |
| { | |
| "epoch": 0.8690223498564116, | |
| "grad_norm": 0.04541015625, | |
| "learning_rate": 2.565739008314944e-07, | |
| "logits/chosen": 0.25941091775894165, | |
| "logits/rejected": 1.3293951749801636, | |
| "logps/chosen": -12.768040657043457, | |
| "logps/rejected": -1509.4613037109375, | |
| "loss": 0.2144, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.2932388186454773, | |
| "rewards/margins": 14.711868286132812, | |
| "rewards/rejected": -14.418627738952637, | |
| "step": 1740 | |
| }, | |
| { | |
| "epoch": 0.8740167311774254, | |
| "grad_norm": 0.0240478515625, | |
| "learning_rate": 2.3767248693890106e-07, | |
| "logits/chosen": 0.2348676472902298, | |
| "logits/rejected": 1.3176116943359375, | |
| "logps/chosen": -13.70958137512207, | |
| "logps/rejected": -1628.524658203125, | |
| "loss": 0.2137, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.3072062134742737, | |
| "rewards/margins": 15.790201187133789, | |
| "rewards/rejected": -15.482992172241211, | |
| "step": 1750 | |
| }, | |
| { | |
| "epoch": 0.8790111124984392, | |
| "grad_norm": 0.0849609375, | |
| "learning_rate": 2.1945943488292265e-07, | |
| "logits/chosen": 0.14879265427589417, | |
| "logits/rejected": 1.2918832302093506, | |
| "logps/chosen": -14.206727981567383, | |
| "logps/rejected": -1858.029052734375, | |
| "loss": 0.2125, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.3139779269695282, | |
| "rewards/margins": 18.043270111083984, | |
| "rewards/rejected": -17.729291915893555, | |
| "step": 1760 | |
| }, | |
| { | |
| "epoch": 0.8840054938194531, | |
| "grad_norm": 0.019775390625, | |
| "learning_rate": 2.0194028637663733e-07, | |
| "logits/chosen": 0.2688780426979065, | |
| "logits/rejected": 1.2982268333435059, | |
| "logps/chosen": -13.01366901397705, | |
| "logps/rejected": -1392.9710693359375, | |
| "loss": 0.213, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.3066830039024353, | |
| "rewards/margins": 13.560384750366211, | |
| "rewards/rejected": -13.253702163696289, | |
| "step": 1770 | |
| }, | |
| { | |
| "epoch": 0.888999875140467, | |
| "grad_norm": 0.0380859375, | |
| "learning_rate": 1.851203719980324e-07, | |
| "logits/chosen": 0.10393796861171722, | |
| "logits/rejected": 1.3042396306991577, | |
| "logps/chosen": -13.014623641967773, | |
| "logps/rejected": -1894.3724365234375, | |
| "loss": 0.2117, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.3259292542934418, | |
| "rewards/margins": 18.341753005981445, | |
| "rewards/rejected": -18.015825271606445, | |
| "step": 1780 | |
| }, | |
| { | |
| "epoch": 0.8939942564614808, | |
| "grad_norm": 0.0250244140625, | |
| "learning_rate": 1.6900480956806214e-07, | |
| "logits/chosen": 0.14930710196495056, | |
| "logits/rejected": 1.24697744846344, | |
| "logps/chosen": -13.011159896850586, | |
| "logps/rejected": -1752.00390625, | |
| "loss": 0.2127, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.30487823486328125, | |
| "rewards/margins": 17.018848419189453, | |
| "rewards/rejected": -16.713970184326172, | |
| "step": 1790 | |
| }, | |
| { | |
| "epoch": 0.8989886377824947, | |
| "grad_norm": 0.0159912109375, | |
| "learning_rate": 1.5359850259344223e-07, | |
| "logits/chosen": 0.19253353774547577, | |
| "logits/rejected": 1.2840015888214111, | |
| "logps/chosen": -13.470372200012207, | |
| "logps/rejected": -1626.4345703125, | |
| "loss": 0.2143, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.3020302951335907, | |
| "rewards/margins": 15.777534484863281, | |
| "rewards/rejected": -15.475504875183105, | |
| "step": 1800 | |
| }, | |
| { | |
| "epoch": 0.9039830191035085, | |
| "grad_norm": 0.0238037109375, | |
| "learning_rate": 1.3890613877465127e-07, | |
| "logits/chosen": 0.236587792634964, | |
| "logits/rejected": 1.3434031009674072, | |
| "logps/chosen": -13.017538070678711, | |
| "logps/rejected": -1619.8177490234375, | |
| "loss": 0.2148, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.29720932245254517, | |
| "rewards/margins": 15.795916557312012, | |
| "rewards/rejected": -15.498708724975586, | |
| "step": 1810 | |
| }, | |
| { | |
| "epoch": 0.9089774004245225, | |
| "grad_norm": 0.064453125, | |
| "learning_rate": 1.249321885795954e-07, | |
| "logits/chosen": 0.23312029242515564, | |
| "logits/rejected": 1.218126893043518, | |
| "logps/chosen": -13.364558219909668, | |
| "logps/rejected": -1396.6041259765625, | |
| "loss": 0.214, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.30306655168533325, | |
| "rewards/margins": 13.558195114135742, | |
| "rewards/rejected": -13.255128860473633, | |
| "step": 1820 | |
| }, | |
| { | |
| "epoch": 0.9139717817455363, | |
| "grad_norm": 0.02001953125, | |
| "learning_rate": 1.1168090388337577e-07, | |
| "logits/chosen": 0.289134681224823, | |
| "logits/rejected": 1.3270288705825806, | |
| "logps/chosen": -12.729695320129395, | |
| "logps/rejected": -1422.671142578125, | |
| "loss": 0.2137, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.3039693832397461, | |
| "rewards/margins": 13.834306716918945, | |
| "rewards/rejected": -13.5303373336792, | |
| "step": 1830 | |
| }, | |
| { | |
| "epoch": 0.9189661630665501, | |
| "grad_norm": 0.0257568359375, | |
| "learning_rate": 9.915631667455989e-08, | |
| "logits/chosen": 0.23302344977855682, | |
| "logits/rejected": 1.3944863080978394, | |
| "logps/chosen": -13.118586540222168, | |
| "logps/rejected": -1715.78515625, | |
| "loss": 0.2139, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.29517239332199097, | |
| "rewards/margins": 16.75180435180664, | |
| "rewards/rejected": -16.45663070678711, | |
| "step": 1840 | |
| }, | |
| { | |
| "epoch": 0.923960544387564, | |
| "grad_norm": 0.025634765625, | |
| "learning_rate": 8.736223782836589e-08, | |
| "logits/chosen": 0.1992538869380951, | |
| "logits/rejected": 1.3024797439575195, | |
| "logps/chosen": -12.67003059387207, | |
| "logps/rejected": -1692.8782958984375, | |
| "loss": 0.214, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.3054552972316742, | |
| "rewards/margins": 16.512258529663086, | |
| "rewards/rejected": -16.206802368164062, | |
| "step": 1850 | |
| }, | |
| { | |
| "epoch": 0.9289549257085778, | |
| "grad_norm": 0.01226806640625, | |
| "learning_rate": 7.63022559471202e-08, | |
| "logits/chosen": 0.23122599720954895, | |
| "logits/rejected": 1.37257981300354, | |
| "logps/chosen": -12.650789260864258, | |
| "logps/rejected": -1592.7677001953125, | |
| "loss": 0.2138, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.2996312975883484, | |
| "rewards/margins": 15.537762641906738, | |
| "rewards/rejected": -15.238128662109375, | |
| "step": 1860 | |
| }, | |
| { | |
| "epoch": 0.9339493070295917, | |
| "grad_norm": 0.0159912109375, | |
| "learning_rate": 6.597973626834759e-08, | |
| "logits/chosen": 0.21128106117248535, | |
| "logits/rejected": 1.4774492979049683, | |
| "logps/chosen": -13.167986869812012, | |
| "logps/rejected": -1897.197998046875, | |
| "loss": 0.2133, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.30018025636672974, | |
| "rewards/margins": 18.570720672607422, | |
| "rewards/rejected": -18.27054214477539, | |
| "step": 1870 | |
| }, | |
| { | |
| "epoch": 0.9389436883506056, | |
| "grad_norm": 0.0264892578125, | |
| "learning_rate": 5.639781964082547e-08, | |
| "logits/chosen": 0.27233806252479553, | |
| "logits/rejected": 1.4588285684585571, | |
| "logps/chosen": -13.405789375305176, | |
| "logps/rejected": -1711.1881103515625, | |
| "loss": 0.214, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.29316192865371704, | |
| "rewards/margins": 16.696842193603516, | |
| "rewards/rejected": -16.40367889404297, | |
| "step": 1880 | |
| }, | |
| { | |
| "epoch": 0.9439380696716194, | |
| "grad_norm": 0.033203125, | |
| "learning_rate": 4.755942156891458e-08, | |
| "logits/chosen": 0.23750165104866028, | |
| "logits/rejected": 1.4071403741836548, | |
| "logps/chosen": -12.829435348510742, | |
| "logps/rejected": -1591.2479248046875, | |
| "loss": 0.2133, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.3001479208469391, | |
| "rewards/margins": 15.529951095581055, | |
| "rewards/rejected": -15.229803085327148, | |
| "step": 1890 | |
| }, | |
| { | |
| "epoch": 0.9489324509926332, | |
| "grad_norm": 0.0203857421875, | |
| "learning_rate": 3.946723132545155e-08, | |
| "logits/chosen": 0.18308812379837036, | |
| "logits/rejected": 1.3526315689086914, | |
| "logps/chosen": -13.234288215637207, | |
| "logps/rejected": -1674.6246337890625, | |
| "loss": 0.2124, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.3094561696052551, | |
| "rewards/margins": 16.279918670654297, | |
| "rewards/rejected": -15.970464706420898, | |
| "step": 1900 | |
| }, | |
| { | |
| "epoch": 0.9539268323136472, | |
| "grad_norm": 0.02197265625, | |
| "learning_rate": 3.212371113348156e-08, | |
| "logits/chosen": 0.2626166343688965, | |
| "logits/rejected": 1.3081789016723633, | |
| "logps/chosen": -12.9055757522583, | |
| "logps/rejected": -1487.3057861328125, | |
| "loss": 0.2143, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.29553088545799255, | |
| "rewards/margins": 14.46813678741455, | |
| "rewards/rejected": -14.172607421875, | |
| "step": 1910 | |
| }, | |
| { | |
| "epoch": 0.958921213634661, | |
| "grad_norm": 0.033935546875, | |
| "learning_rate": 2.5531095417073437e-08, | |
| "logits/chosen": 0.2499731481075287, | |
| "logits/rejected": 1.3821344375610352, | |
| "logps/chosen": -13.182432174682617, | |
| "logps/rejected": -1497.8023681640625, | |
| "loss": 0.214, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.29157382249832153, | |
| "rewards/margins": 14.601869583129883, | |
| "rewards/rejected": -14.310295104980469, | |
| "step": 1920 | |
| }, | |
| { | |
| "epoch": 0.9639155949556749, | |
| "grad_norm": 0.0218505859375, | |
| "learning_rate": 1.969139012144822e-08, | |
| "logits/chosen": 0.27068477869033813, | |
| "logits/rejected": 1.3521279096603394, | |
| "logps/chosen": -13.608545303344727, | |
| "logps/rejected": -1491.3599853515625, | |
| "loss": 0.214, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.3026901185512543, | |
| "rewards/margins": 14.520078659057617, | |
| "rewards/rejected": -14.217389106750488, | |
| "step": 1930 | |
| }, | |
| { | |
| "epoch": 0.9689099762766887, | |
| "grad_norm": 0.049072265625, | |
| "learning_rate": 1.4606372102626277e-08, | |
| "logits/chosen": 0.19163861870765686, | |
| "logits/rejected": 1.3109387159347534, | |
| "logps/chosen": -13.46105670928955, | |
| "logps/rejected": -1731.31640625, | |
| "loss": 0.2127, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.3083009421825409, | |
| "rewards/margins": 16.883403778076172, | |
| "rewards/rejected": -16.575105667114258, | |
| "step": 1940 | |
| }, | |
| { | |
| "epoch": 0.9739043575977026, | |
| "grad_norm": 0.0322265625, | |
| "learning_rate": 1.0277588586781463e-08, | |
| "logits/chosen": 0.19613580405712128, | |
| "logits/rejected": 1.228542685508728, | |
| "logps/chosen": -13.699869155883789, | |
| "logps/rejected": -1520.361572265625, | |
| "loss": 0.2129, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.3030748963356018, | |
| "rewards/margins": 14.732551574707031, | |
| "rewards/rejected": -14.429475784301758, | |
| "step": 1950 | |
| }, | |
| { | |
| "epoch": 0.9788987389187165, | |
| "grad_norm": 0.025146484375, | |
| "learning_rate": 6.7063566994651775e-09, | |
| "logits/chosen": 0.21389658749103546, | |
| "logits/rejected": 1.3907114267349243, | |
| "logps/chosen": -13.105180740356445, | |
| "logps/rejected": -1654.1494140625, | |
| "loss": 0.2134, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.30368101596832275, | |
| "rewards/margins": 16.11886978149414, | |
| "rewards/rejected": -15.81518840789795, | |
| "step": 1960 | |
| }, | |
| { | |
| "epoch": 0.9838931202397303, | |
| "grad_norm": 0.02490234375, | |
| "learning_rate": 3.893763064840295e-09, | |
| "logits/chosen": 0.18713845312595367, | |
| "logits/rejected": 1.294327974319458, | |
| "logps/chosen": -12.946528434753418, | |
| "logps/rejected": -1592.420654296875, | |
| "loss": 0.2118, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.31735336780548096, | |
| "rewards/margins": 15.468029975891113, | |
| "rewards/rejected": -15.150675773620605, | |
| "step": 1970 | |
| }, | |
| { | |
| "epoch": 0.9888875015607441, | |
| "grad_norm": 0.0172119140625, | |
| "learning_rate": 1.840663475053961e-09, | |
| "logits/chosen": 0.23091156780719757, | |
| "logits/rejected": 1.4303152561187744, | |
| "logps/chosen": -13.524391174316406, | |
| "logps/rejected": -1685.2825927734375, | |
| "loss": 0.2139, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.30492401123046875, | |
| "rewards/margins": 16.39162826538086, | |
| "rewards/rejected": -16.08670425415039, | |
| "step": 1980 | |
| }, | |
| { | |
| "epoch": 0.993881882881758, | |
| "grad_norm": 0.01446533203125, | |
| "learning_rate": 5.476826298439486e-10, | |
| "logits/chosen": 0.18041366338729858, | |
| "logits/rejected": 1.3766809701919556, | |
| "logps/chosen": -12.73656940460205, | |
| "logps/rejected": -1975.449462890625, | |
| "loss": 0.2138, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.2982867360115051, | |
| "rewards/margins": 19.314851760864258, | |
| "rewards/rejected": -19.016565322875977, | |
| "step": 1990 | |
| }, | |
| { | |
| "epoch": 0.9988762642027719, | |
| "grad_norm": 0.029052734375, | |
| "learning_rate": 1.521394646070151e-11, | |
| "logits/chosen": 0.212058424949646, | |
| "logits/rejected": 1.3283421993255615, | |
| "logps/chosen": -12.978428840637207, | |
| "logps/rejected": -1706.1982421875, | |
| "loss": 0.2118, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.3162993788719177, | |
| "rewards/margins": 16.5406551361084, | |
| "rewards/rejected": -16.224355697631836, | |
| "step": 2000 | |
| }, | |
| { | |
| "epoch": 0.9988762642027719, | |
| "eval_logits/chosen": 0.19315297901630402, | |
| "eval_logits/rejected": 1.0988490581512451, | |
| "eval_logps/chosen": -12.085772514343262, | |
| "eval_logps/rejected": -1018.5144653320312, | |
| "eval_loss": 0.2121797353029251, | |
| "eval_rewards/accuracies": 1.0, | |
| "eval_rewards/chosen": 0.32569777965545654, | |
| "eval_rewards/margins": 9.880459785461426, | |
| "eval_rewards/rejected": -9.554760932922363, | |
| "eval_runtime": 0.4236, | |
| "eval_samples_per_second": 11.805, | |
| "eval_steps_per_second": 7.083, | |
| "step": 2000 | |
| }, | |
| { | |
| "epoch": 0.9998751404669747, | |
| "step": 2002, | |
| "total_flos": 0.0, | |
| "train_loss": 0.2449444185841929, | |
| "train_runtime": 3711.3289, | |
| "train_samples_per_second": 4.316, | |
| "train_steps_per_second": 0.539 | |
| } | |
| ], | |
| "logging_steps": 10, | |
| "max_steps": 2002, | |
| "num_input_tokens_seen": 0, | |
| "num_train_epochs": 1, | |
| "save_steps": 1000, | |
| "total_flos": 0.0, | |
| "train_batch_size": 1, | |
| "trial_name": null, | |
| "trial_params": null | |
| } | |