| { | |
| "best_metric": null, | |
| "best_model_checkpoint": null, | |
| "epoch": 1.0, | |
| "eval_steps": 500, | |
| "global_step": 1904, | |
| "is_hyper_param_search": false, | |
| "is_local_process_zero": true, | |
| "is_world_process_zero": true, | |
| "log_history": [ | |
| { | |
| "epoch": 0.005252100840336135, | |
| "grad_norm": 69.5775218105222, | |
| "kl": 0.15183432400226593, | |
| "learning_rate": 4.999659696812289e-07, | |
| "logps/chosen": -298.5141715116279, | |
| "logps/rejected": -259.0493032094595, | |
| "loss": 0.4994, | |
| "rewards/chosen": -0.04585062625796296, | |
| "rewards/margins": 0.01403944421908452, | |
| "rewards/rejected": -0.05989007047704748, | |
| "step": 10 | |
| }, | |
| { | |
| "epoch": 0.01050420168067227, | |
| "grad_norm": 55.49240989732535, | |
| "kl": 2.105862855911255, | |
| "learning_rate": 4.998638879894165e-07, | |
| "logps/chosen": -273.6553505345395, | |
| "logps/rejected": -227.15778459821428, | |
| "loss": 0.4776, | |
| "rewards/chosen": 0.49514409115439967, | |
| "rewards/margins": 0.21992081329039764, | |
| "rewards/rejected": 0.275223277864002, | |
| "step": 20 | |
| }, | |
| { | |
| "epoch": 0.015756302521008403, | |
| "grad_norm": 51.28980044595933, | |
| "kl": 0.28644976019859314, | |
| "learning_rate": 4.996937827155428e-07, | |
| "logps/chosen": -311.9908854166667, | |
| "logps/rejected": -247.46916946308724, | |
| "loss": 0.4486, | |
| "rewards/chosen": 0.30754662675467154, | |
| "rewards/margins": 0.5134327379181245, | |
| "rewards/rejected": -0.20588611116345296, | |
| "step": 30 | |
| }, | |
| { | |
| "epoch": 0.02100840336134454, | |
| "grad_norm": 53.39172095489608, | |
| "kl": 0.0, | |
| "learning_rate": 4.994557001695013e-07, | |
| "logps/chosen": -298.2471417682927, | |
| "logps/rejected": -238.9893078926282, | |
| "loss": 0.4172, | |
| "rewards/chosen": 0.01273148815806319, | |
| "rewards/margins": 0.8681685468269335, | |
| "rewards/rejected": -0.8554370586688702, | |
| "step": 40 | |
| }, | |
| { | |
| "epoch": 0.026260504201680673, | |
| "grad_norm": 48.3236105129346, | |
| "kl": 0.0, | |
| "learning_rate": 4.991497051674917e-07, | |
| "logps/chosen": -289.8418828616352, | |
| "logps/rejected": -264.62730493012424, | |
| "loss": 0.3922, | |
| "rewards/chosen": 0.3131725623172784, | |
| "rewards/margins": 1.3170881950509257, | |
| "rewards/rejected": -1.0039156327336471, | |
| "step": 50 | |
| }, | |
| { | |
| "epoch": 0.031512605042016806, | |
| "grad_norm": 51.661072159206874, | |
| "kl": 0.0, | |
| "learning_rate": 4.987758810143735e-07, | |
| "logps/chosen": -331.279736328125, | |
| "logps/rejected": -255.901513671875, | |
| "loss": 0.3954, | |
| "rewards/chosen": 0.22145886421203614, | |
| "rewards/margins": 1.2238109111785889, | |
| "rewards/rejected": -1.0023520469665528, | |
| "step": 60 | |
| }, | |
| { | |
| "epoch": 0.03676470588235294, | |
| "grad_norm": 48.98498102999305, | |
| "kl": 0.0, | |
| "learning_rate": 4.983343294809874e-07, | |
| "logps/chosen": -309.5669285973837, | |
| "logps/rejected": -255.4453125, | |
| "loss": 0.3732, | |
| "rewards/chosen": 0.5395132552745731, | |
| "rewards/margins": 1.6830449514910533, | |
| "rewards/rejected": -1.14353169621648, | |
| "step": 70 | |
| }, | |
| { | |
| "epoch": 0.04201680672268908, | |
| "grad_norm": 49.52520073364695, | |
| "kl": 0.0, | |
| "learning_rate": 4.978251707764491e-07, | |
| "logps/chosen": -298.83478282232704, | |
| "logps/rejected": -262.3364712732919, | |
| "loss": 0.3542, | |
| "rewards/chosen": 0.41468177651459315, | |
| "rewards/margins": 2.0195703134298686, | |
| "rewards/rejected": -1.6048885369152757, | |
| "step": 80 | |
| }, | |
| { | |
| "epoch": 0.04726890756302521, | |
| "grad_norm": 45.84438443510472, | |
| "kl": 0.0, | |
| "learning_rate": 4.972485435154228e-07, | |
| "logps/chosen": -284.00230493012424, | |
| "logps/rejected": -292.5305866745283, | |
| "loss": 0.3424, | |
| "rewards/chosen": 0.20849199472747235, | |
| "rewards/margins": 2.4833206356472837, | |
| "rewards/rejected": -2.2748286409198113, | |
| "step": 90 | |
| }, | |
| { | |
| "epoch": 0.052521008403361345, | |
| "grad_norm": 51.90503609381182, | |
| "kl": 0.0, | |
| "learning_rate": 4.966046046803842e-07, | |
| "logps/chosen": -305.3333748009554, | |
| "logps/rejected": -265.70988305214723, | |
| "loss": 0.3825, | |
| "rewards/chosen": 0.518976345183743, | |
| "rewards/margins": 2.160451905658724, | |
| "rewards/rejected": -1.6414755604749809, | |
| "step": 100 | |
| }, | |
| { | |
| "epoch": 0.05777310924369748, | |
| "grad_norm": 46.615982508495186, | |
| "kl": 0.0, | |
| "learning_rate": 4.958935295788841e-07, | |
| "logps/chosen": -346.7928911423841, | |
| "logps/rejected": -282.3527181952663, | |
| "loss": 0.3829, | |
| "rewards/chosen": 0.18162511358197952, | |
| "rewards/margins": 1.8821555777091985, | |
| "rewards/rejected": -1.700530464127219, | |
| "step": 110 | |
| }, | |
| { | |
| "epoch": 0.06302521008403361, | |
| "grad_norm": 43.56170699177104, | |
| "kl": 0.0, | |
| "learning_rate": 4.951155117958216e-07, | |
| "logps/chosen": -299.7170380015432, | |
| "logps/rejected": -281.8764092167722, | |
| "loss": 0.3787, | |
| "rewards/chosen": -0.607381797131197, | |
| "rewards/margins": 2.2764308295746973, | |
| "rewards/rejected": -2.883812626705894, | |
| "step": 120 | |
| }, | |
| { | |
| "epoch": 0.06827731092436974, | |
| "grad_norm": 52.35577489618577, | |
| "kl": 0.0, | |
| "learning_rate": 4.942707631407419e-07, | |
| "logps/chosen": -335.1082099780702, | |
| "logps/rejected": -281.61443477348996, | |
| "loss": 0.3757, | |
| "rewards/chosen": -0.22571238021404422, | |
| "rewards/margins": 2.2212163225308172, | |
| "rewards/rejected": -2.4469287027448616, | |
| "step": 130 | |
| }, | |
| { | |
| "epoch": 0.07352941176470588, | |
| "grad_norm": 28.41591231440036, | |
| "kl": 0.0, | |
| "learning_rate": 4.933595135901732e-07, | |
| "logps/chosen": -318.5261247783688, | |
| "logps/rejected": -269.2608676675978, | |
| "loss": 0.3369, | |
| "rewards/chosen": 0.26553474588597076, | |
| "rewards/margins": 2.99381714157382, | |
| "rewards/rejected": -2.7282823956878492, | |
| "step": 140 | |
| }, | |
| { | |
| "epoch": 0.07878151260504201, | |
| "grad_norm": 77.73773489117062, | |
| "kl": 0.0, | |
| "learning_rate": 4.923820112250169e-07, | |
| "logps/chosen": -301.63650901845637, | |
| "logps/rejected": -266.3384959795322, | |
| "loss": 0.3375, | |
| "rewards/chosen": 0.025881773673447985, | |
| "rewards/margins": 3.2468575937575124, | |
| "rewards/rejected": -3.2209758200840644, | |
| "step": 150 | |
| }, | |
| { | |
| "epoch": 0.08403361344537816, | |
| "grad_norm": 37.575572316994865, | |
| "kl": 0.0, | |
| "learning_rate": 4.913385221630096e-07, | |
| "logps/chosen": -250.38438482704402, | |
| "logps/rejected": -265.8534064440994, | |
| "loss": 0.3758, | |
| "rewards/chosen": 0.7617762343688581, | |
| "rewards/margins": 1.6022024720556787, | |
| "rewards/rejected": -0.8404262376868207, | |
| "step": 160 | |
| }, | |
| { | |
| "epoch": 0.08928571428571429, | |
| "grad_norm": 40.501974364358205, | |
| "kl": 0.0, | |
| "learning_rate": 4.902293304862749e-07, | |
| "logps/chosen": -270.815234375, | |
| "logps/rejected": -255.6860107421875, | |
| "loss": 0.3672, | |
| "rewards/chosen": 0.9947892189025879, | |
| "rewards/margins": 1.629128646850586, | |
| "rewards/rejected": -0.634339427947998, | |
| "step": 170 | |
| }, | |
| { | |
| "epoch": 0.09453781512605042, | |
| "grad_norm": 36.96138019330735, | |
| "kl": 0.0, | |
| "learning_rate": 4.890547381639833e-07, | |
| "logps/chosen": -299.6106373856707, | |
| "logps/rejected": -230.88887219551282, | |
| "loss": 0.3625, | |
| "rewards/chosen": 0.3851209733544326, | |
| "rewards/margins": 2.033356000961103, | |
| "rewards/rejected": -1.6482350276066706, | |
| "step": 180 | |
| }, | |
| { | |
| "epoch": 0.09978991596638656, | |
| "grad_norm": 44.46735174580667, | |
| "kl": 0.0, | |
| "learning_rate": 4.878150649701439e-07, | |
| "logps/chosen": -344.7996875, | |
| "logps/rejected": -265.18083639705884, | |
| "loss": 0.3185, | |
| "rewards/chosen": 0.36289586385091144, | |
| "rewards/margins": 2.9504150719736133, | |
| "rewards/rejected": -2.587519208122702, | |
| "step": 190 | |
| }, | |
| { | |
| "epoch": 0.10504201680672269, | |
| "grad_norm": 52.28178000802996, | |
| "kl": 0.0, | |
| "learning_rate": 4.865106483965486e-07, | |
| "logps/chosen": -292.6435797275641, | |
| "logps/rejected": -240.90084317835365, | |
| "loss": 0.3407, | |
| "rewards/chosen": -0.03864570764394907, | |
| "rewards/margins": 2.6535366936875104, | |
| "rewards/rejected": -2.6921824013314595, | |
| "step": 200 | |
| }, | |
| { | |
| "epoch": 0.11029411764705882, | |
| "grad_norm": 42.51233566429091, | |
| "kl": 0.0, | |
| "learning_rate": 4.851418435608919e-07, | |
| "logps/chosen": -300.3898746468927, | |
| "logps/rejected": -269.88991477272725, | |
| "loss": 0.3554, | |
| "rewards/chosen": 0.8636472023139565, | |
| "rewards/margins": 2.70990092417803, | |
| "rewards/rejected": -1.8462537218640733, | |
| "step": 210 | |
| }, | |
| { | |
| "epoch": 0.11554621848739496, | |
| "grad_norm": 45.269264573206264, | |
| "kl": 0.0, | |
| "learning_rate": 4.837090231100927e-07, | |
| "logps/chosen": -315.7438054733728, | |
| "logps/rejected": -242.41240687086093, | |
| "loss": 0.327, | |
| "rewards/chosen": 1.163497202495146, | |
| "rewards/margins": 2.8938327665084946, | |
| "rewards/rejected": -1.7303355640133484, | |
| "step": 220 | |
| }, | |
| { | |
| "epoch": 0.1207983193277311, | |
| "grad_norm": 46.30035764458271, | |
| "kl": 0.0, | |
| "learning_rate": 4.822125771188448e-07, | |
| "logps/chosen": -285.65898944805195, | |
| "logps/rejected": -271.8729762801205, | |
| "loss": 0.3828, | |
| "rewards/chosen": 0.5068141392299107, | |
| "rewards/margins": 1.911893440811236, | |
| "rewards/rejected": -1.4050793015813252, | |
| "step": 230 | |
| }, | |
| { | |
| "epoch": 0.12605042016806722, | |
| "grad_norm": 44.08666008078037, | |
| "kl": 0.0, | |
| "learning_rate": 4.806529129834207e-07, | |
| "logps/chosen": -280.8894210188356, | |
| "logps/rejected": -270.21331716954023, | |
| "loss": 0.329, | |
| "rewards/chosen": 0.5693619087950824, | |
| "rewards/margins": 2.748881386727015, | |
| "rewards/rejected": -2.1795194779319327, | |
| "step": 240 | |
| }, | |
| { | |
| "epoch": 0.13130252100840337, | |
| "grad_norm": 38.010572045662606, | |
| "kl": 0.0, | |
| "learning_rate": 4.790304553107622e-07, | |
| "logps/chosen": -304.2262851331361, | |
| "logps/rejected": -254.57090749172184, | |
| "loss": 0.3588, | |
| "rewards/chosen": 0.49599551590236685, | |
| "rewards/margins": 2.326870937893778, | |
| "rewards/rejected": -1.8308754219914114, | |
| "step": 250 | |
| }, | |
| { | |
| "epoch": 0.13655462184873948, | |
| "grad_norm": 44.14238619732289, | |
| "kl": 0.0, | |
| "learning_rate": 4.773456458028837e-07, | |
| "logps/chosen": -278.3834842754777, | |
| "logps/rejected": -268.8938362730061, | |
| "loss": 0.3378, | |
| "rewards/chosen": 0.5815278071506768, | |
| "rewards/margins": 2.6120607376769422, | |
| "rewards/rejected": -2.0305329305262654, | |
| "step": 260 | |
| }, | |
| { | |
| "epoch": 0.14180672268907563, | |
| "grad_norm": 39.61832047489729, | |
| "kl": 1.3655751943588257, | |
| "learning_rate": 4.755989431366221e-07, | |
| "logps/chosen": -301.3936798878205, | |
| "logps/rejected": -299.8501572027439, | |
| "loss": 0.3287, | |
| "rewards/chosen": 0.6345734229454627, | |
| "rewards/margins": 2.3345022648852494, | |
| "rewards/rejected": -1.6999288419397867, | |
| "step": 270 | |
| }, | |
| { | |
| "epoch": 0.14705882352941177, | |
| "grad_norm": 43.29173967453139, | |
| "kl": 0.0, | |
| "learning_rate": 4.737908228387656e-07, | |
| "logps/chosen": -294.7026832217262, | |
| "logps/rejected": -264.49424342105266, | |
| "loss": 0.3553, | |
| "rewards/chosen": 0.5891142799740746, | |
| "rewards/margins": 2.708873086704646, | |
| "rewards/rejected": -2.1197588067305717, | |
| "step": 280 | |
| }, | |
| { | |
| "epoch": 0.15231092436974789, | |
| "grad_norm": 49.48779153565427, | |
| "kl": 0.0, | |
| "learning_rate": 4.7192177715659516e-07, | |
| "logps/chosen": -290.1478293413174, | |
| "logps/rejected": -243.8566176470588, | |
| "loss": 0.3763, | |
| "rewards/chosen": 0.392075178865901, | |
| "rewards/margins": 2.094171158045333, | |
| "rewards/rejected": -1.7020959791794321, | |
| "step": 290 | |
| }, | |
| { | |
| "epoch": 0.15756302521008403, | |
| "grad_norm": 44.86612305681867, | |
| "kl": 0.0, | |
| "learning_rate": 4.699923149238736e-07, | |
| "logps/chosen": -283.2191996402878, | |
| "logps/rejected": -272.7527192679558, | |
| "loss": 0.3369, | |
| "rewards/chosen": 0.6118597675570481, | |
| "rewards/margins": 2.6730571222057016, | |
| "rewards/rejected": -2.0611973546486535, | |
| "step": 300 | |
| }, | |
| { | |
| "epoch": 0.16281512605042017, | |
| "grad_norm": 43.00617930560404, | |
| "kl": 0.0, | |
| "learning_rate": 4.680029614223198e-07, | |
| "logps/chosen": -272.5205015923567, | |
| "logps/rejected": -266.77614551380367, | |
| "loss": 0.3531, | |
| "rewards/chosen": 0.13305815921467581, | |
| "rewards/margins": 2.627990556226275, | |
| "rewards/rejected": -2.494932397011599, | |
| "step": 310 | |
| }, | |
| { | |
| "epoch": 0.16806722689075632, | |
| "grad_norm": 39.410413100307075, | |
| "kl": 0.0, | |
| "learning_rate": 4.65954258238604e-07, | |
| "logps/chosen": -288.9263426272455, | |
| "logps/rejected": -254.3640216503268, | |
| "loss": 0.3736, | |
| "rewards/chosen": 0.09242550056137724, | |
| "rewards/margins": 2.2175613337032276, | |
| "rewards/rejected": -2.1251358331418504, | |
| "step": 320 | |
| }, | |
| { | |
| "epoch": 0.17331932773109243, | |
| "grad_norm": 41.100451094010424, | |
| "kl": 0.0, | |
| "learning_rate": 4.638467631169056e-07, | |
| "logps/chosen": -326.0165550595238, | |
| "logps/rejected": -306.81566097861844, | |
| "loss": 0.318, | |
| "rewards/chosen": 0.5781731832595098, | |
| "rewards/margins": 3.290003924740287, | |
| "rewards/rejected": -2.711830741480777, | |
| "step": 330 | |
| }, | |
| { | |
| "epoch": 0.17857142857142858, | |
| "grad_norm": 59.056271025186184, | |
| "kl": 0.0, | |
| "learning_rate": 4.6168104980707103e-07, | |
| "logps/chosen": -285.7653391768293, | |
| "logps/rejected": -274.3047876602564, | |
| "loss": 0.347, | |
| "rewards/chosen": -0.003299643353718083, | |
| "rewards/margins": 3.289772396910705, | |
| "rewards/rejected": -3.293072040264423, | |
| "step": 340 | |
| }, | |
| { | |
| "epoch": 0.18382352941176472, | |
| "grad_norm": 40.5420682235125, | |
| "kl": 0.0, | |
| "learning_rate": 4.594577079084145e-07, | |
| "logps/chosen": -290.82744140625, | |
| "logps/rejected": -292.7601318359375, | |
| "loss": 0.3356, | |
| "rewards/chosen": 0.4554912567138672, | |
| "rewards/margins": 3.3385635375976563, | |
| "rewards/rejected": -2.883072280883789, | |
| "step": 350 | |
| }, | |
| { | |
| "epoch": 0.18907563025210083, | |
| "grad_norm": 73.25856842000852, | |
| "kl": 0.0, | |
| "learning_rate": 4.5717734270920466e-07, | |
| "logps/chosen": -281.4008819018405, | |
| "logps/rejected": -228.66883957006368, | |
| "loss": 0.3378, | |
| "rewards/chosen": 0.49251336407807705, | |
| "rewards/margins": 2.78153659942342, | |
| "rewards/rejected": -2.2890232353453426, | |
| "step": 360 | |
| }, | |
| { | |
| "epoch": 0.19432773109243698, | |
| "grad_norm": 36.80227868403581, | |
| "kl": 0.0, | |
| "learning_rate": 4.548405750218785e-07, | |
| "logps/chosen": -288.27572185672517, | |
| "logps/rejected": -269.6975146812081, | |
| "loss": 0.3552, | |
| "rewards/chosen": -0.02239043949640285, | |
| "rewards/margins": 3.2017408950516257, | |
| "rewards/rejected": -3.2241313345480287, | |
| "step": 370 | |
| }, | |
| { | |
| "epoch": 0.19957983193277312, | |
| "grad_norm": 52.176625363111675, | |
| "kl": 0.0, | |
| "learning_rate": 4.5244804101403025e-07, | |
| "logps/chosen": -262.20579637096773, | |
| "logps/rejected": -253.31008522727274, | |
| "loss": 0.3773, | |
| "rewards/chosen": -0.5523546772618447, | |
| "rewards/margins": 2.2483803339834205, | |
| "rewards/rejected": -2.800735011245265, | |
| "step": 380 | |
| }, | |
| { | |
| "epoch": 0.20483193277310924, | |
| "grad_norm": 40.91805541297444, | |
| "kl": 0.0, | |
| "learning_rate": 4.5000039203521976e-07, | |
| "logps/chosen": -305.74300986842104, | |
| "logps/rejected": -303.6615048363095, | |
| "loss": 0.3428, | |
| "rewards/chosen": 0.08902386615150853, | |
| "rewards/margins": 3.217569718086032, | |
| "rewards/rejected": -3.1285458519345237, | |
| "step": 390 | |
| }, | |
| { | |
| "epoch": 0.21008403361344538, | |
| "grad_norm": 48.39763770048436, | |
| "kl": 0.0, | |
| "learning_rate": 4.47498294439647e-07, | |
| "logps/chosen": -293.79254518072287, | |
| "logps/rejected": -266.05382508116884, | |
| "loss": 0.3273, | |
| "rewards/chosen": 0.20805818488798947, | |
| "rewards/margins": 2.784955744115871, | |
| "rewards/rejected": -2.5768975592278816, | |
| "step": 400 | |
| }, | |
| { | |
| "epoch": 0.21533613445378152, | |
| "grad_norm": 64.69719531344766, | |
| "kl": 0.0, | |
| "learning_rate": 4.449424294047419e-07, | |
| "logps/chosen": -298.2714004984663, | |
| "logps/rejected": -277.8911972531847, | |
| "loss": 0.3478, | |
| "rewards/chosen": 0.7857334628426955, | |
| "rewards/margins": 2.803303423904598, | |
| "rewards/rejected": -2.017569961061903, | |
| "step": 410 | |
| }, | |
| { | |
| "epoch": 0.22058823529411764, | |
| "grad_norm": 41.5548039641909, | |
| "kl": 0.0, | |
| "learning_rate": 4.4233349274571974e-07, | |
| "logps/chosen": -294.40205536912754, | |
| "logps/rejected": -262.53922240497076, | |
| "loss": 0.3273, | |
| "rewards/chosen": 0.6274464754450241, | |
| "rewards/margins": 3.27400759055376, | |
| "rewards/rejected": -2.6465611151087356, | |
| "step": 420 | |
| }, | |
| { | |
| "epoch": 0.22584033613445378, | |
| "grad_norm": 51.90314639757878, | |
| "kl": 0.0, | |
| "learning_rate": 4.396721947261496e-07, | |
| "logps/chosen": -290.494140625, | |
| "logps/rejected": -266.31001420454544, | |
| "loss": 0.3513, | |
| "rewards/chosen": 0.22074366190347328, | |
| "rewards/margins": 2.942957208854569, | |
| "rewards/rejected": -2.722213546951096, | |
| "step": 430 | |
| }, | |
| { | |
| "epoch": 0.23109243697478993, | |
| "grad_norm": 45.08774574606674, | |
| "kl": 0.0, | |
| "learning_rate": 4.3695925986459107e-07, | |
| "logps/chosen": -274.8046160442073, | |
| "logps/rejected": -247.5850110176282, | |
| "loss": 0.3506, | |
| "rewards/chosen": 0.5506789044636052, | |
| "rewards/margins": 3.319194662488946, | |
| "rewards/rejected": -2.7685157580253406, | |
| "step": 440 | |
| }, | |
| { | |
| "epoch": 0.23634453781512604, | |
| "grad_norm": 38.43146082353361, | |
| "kl": 0.0, | |
| "learning_rate": 4.341954267373494e-07, | |
| "logps/chosen": -293.00198951863354, | |
| "logps/rejected": -258.1283411949685, | |
| "loss": 0.3646, | |
| "rewards/chosen": 0.5552752477041683, | |
| "rewards/margins": 2.609661794480898, | |
| "rewards/rejected": -2.0543865467767297, | |
| "step": 450 | |
| }, | |
| { | |
| "epoch": 0.2415966386554622, | |
| "grad_norm": 42.36936446062343, | |
| "kl": 0.0, | |
| "learning_rate": 4.313814477774035e-07, | |
| "logps/chosen": -307.16541371855345, | |
| "logps/rejected": -266.2528872282609, | |
| "loss": 0.3188, | |
| "rewards/chosen": 0.6136390638051543, | |
| "rewards/margins": 3.2912566758975457, | |
| "rewards/rejected": -2.677617612092391, | |
| "step": 460 | |
| }, | |
| { | |
| "epoch": 0.24684873949579833, | |
| "grad_norm": 36.14877170239455, | |
| "kl": 0.0, | |
| "learning_rate": 4.2851808906956134e-07, | |
| "logps/chosen": -299.0251017011834, | |
| "logps/rejected": -283.08899006622516, | |
| "loss": 0.3248, | |
| "rewards/chosen": 0.8636552257650703, | |
| "rewards/margins": 4.432308458769002, | |
| "rewards/rejected": -3.568653233003932, | |
| "step": 470 | |
| }, | |
| { | |
| "epoch": 0.25210084033613445, | |
| "grad_norm": 48.52314483845558, | |
| "kl": 0.0, | |
| "learning_rate": 4.256061301418996e-07, | |
| "logps/chosen": -288.11904978197674, | |
| "logps/rejected": -273.0204022381757, | |
| "loss": 0.3566, | |
| "rewards/chosen": 0.09835364097772642, | |
| "rewards/margins": 3.3749882696860247, | |
| "rewards/rejected": -3.2766346287082984, | |
| "step": 480 | |
| }, | |
| { | |
| "epoch": 0.25735294117647056, | |
| "grad_norm": 40.328590442957, | |
| "kl": 0.0, | |
| "learning_rate": 4.2264636375354283e-07, | |
| "logps/chosen": -290.32015931372547, | |
| "logps/rejected": -256.6384496631737, | |
| "loss": 0.3385, | |
| "rewards/chosen": 0.6565687890146293, | |
| "rewards/margins": 2.809185004337889, | |
| "rewards/rejected": -2.1526162153232598, | |
| "step": 490 | |
| }, | |
| { | |
| "epoch": 0.26260504201680673, | |
| "grad_norm": 51.22796460390333, | |
| "kl": 0.0, | |
| "learning_rate": 4.1963959567884045e-07, | |
| "logps/chosen": -302.9179159628378, | |
| "logps/rejected": -269.88603742732556, | |
| "loss": 0.3213, | |
| "rewards/chosen": 1.2393865843076963, | |
| "rewards/margins": 3.752552790135276, | |
| "rewards/rejected": -2.51316620582758, | |
| "step": 500 | |
| }, | |
| { | |
| "epoch": 0.26785714285714285, | |
| "grad_norm": 40.763747675132, | |
| "kl": 0.0, | |
| "learning_rate": 4.1658664448800094e-07, | |
| "logps/chosen": -278.07726258116884, | |
| "logps/rejected": -242.03510918674698, | |
| "loss": 0.3613, | |
| "rewards/chosen": 1.3944342476981026, | |
| "rewards/margins": 2.644719200987824, | |
| "rewards/rejected": -1.2502849532897213, | |
| "step": 510 | |
| }, | |
| { | |
| "epoch": 0.27310924369747897, | |
| "grad_norm": 41.37886905858638, | |
| "kl": 0.0, | |
| "learning_rate": 4.1348834132424204e-07, | |
| "logps/chosen": -303.84667354559747, | |
| "logps/rejected": -279.02011354813664, | |
| "loss": 0.355, | |
| "rewards/chosen": 0.7681676516742826, | |
| "rewards/margins": 3.026760719293654, | |
| "rewards/rejected": -2.2585930676193713, | |
| "step": 520 | |
| }, | |
| { | |
| "epoch": 0.27836134453781514, | |
| "grad_norm": 37.134601685804, | |
| "kl": 0.0, | |
| "learning_rate": 4.103455296775181e-07, | |
| "logps/chosen": -311.4682323042169, | |
| "logps/rejected": -256.0505783279221, | |
| "loss": 0.3388, | |
| "rewards/chosen": 1.0605972473879894, | |
| "rewards/margins": 2.909687229061589, | |
| "rewards/rejected": -1.8490899816736, | |
| "step": 530 | |
| }, | |
| { | |
| "epoch": 0.28361344537815125, | |
| "grad_norm": 50.21606157716232, | |
| "kl": 0.0, | |
| "learning_rate": 4.071590651548867e-07, | |
| "logps/chosen": -282.52527573529414, | |
| "logps/rejected": -283.90199288922156, | |
| "loss": 0.3322, | |
| "rewards/chosen": 0.6519086251851001, | |
| "rewards/margins": 3.288977687743577, | |
| "rewards/rejected": -2.637069062558477, | |
| "step": 540 | |
| }, | |
| { | |
| "epoch": 0.28886554621848737, | |
| "grad_norm": 40.94800472377613, | |
| "kl": 0.0, | |
| "learning_rate": 4.039298152475754e-07, | |
| "logps/chosen": -296.5549363057325, | |
| "logps/rejected": -296.0996932515337, | |
| "loss": 0.3261, | |
| "rewards/chosen": 0.4167486786083051, | |
| "rewards/margins": 3.556985000742124, | |
| "rewards/rejected": -3.140236322133819, | |
| "step": 550 | |
| }, | |
| { | |
| "epoch": 0.29411764705882354, | |
| "grad_norm": 47.7284966576563, | |
| "kl": 0.0, | |
| "learning_rate": 4.006586590948141e-07, | |
| "logps/chosen": -287.6368415880503, | |
| "logps/rejected": -276.68611704192546, | |
| "loss": 0.3348, | |
| "rewards/chosen": -0.10697765470300831, | |
| "rewards/margins": 3.6730512630716423, | |
| "rewards/rejected": -3.7800289177746507, | |
| "step": 560 | |
| }, | |
| { | |
| "epoch": 0.29936974789915966, | |
| "grad_norm": 45.50328767344306, | |
| "kl": 0.0, | |
| "learning_rate": 3.973464872444958e-07, | |
| "logps/chosen": -296.46134996118013, | |
| "logps/rejected": -281.4715507075472, | |
| "loss": 0.3322, | |
| "rewards/chosen": 0.1290158692354001, | |
| "rewards/margins": 3.9525375132998652, | |
| "rewards/rejected": -3.8235216440644653, | |
| "step": 570 | |
| }, | |
| { | |
| "epoch": 0.30462184873949577, | |
| "grad_norm": 38.47971512593867, | |
| "kl": 0.0, | |
| "learning_rate": 3.939942014107318e-07, | |
| "logps/chosen": -262.15194610778445, | |
| "logps/rejected": -292.0669424019608, | |
| "loss": 0.3594, | |
| "rewards/chosen": 0.3589071856287425, | |
| "rewards/margins": 3.160983936739343, | |
| "rewards/rejected": -2.8020767511106004, | |
| "step": 580 | |
| }, | |
| { | |
| "epoch": 0.30987394957983194, | |
| "grad_norm": 34.207797203706725, | |
| "kl": 0.0, | |
| "learning_rate": 3.9060271422836624e-07, | |
| "logps/chosen": -292.61703330592104, | |
| "logps/rejected": -271.3793247767857, | |
| "loss": 0.3159, | |
| "rewards/chosen": 1.2860569201017682, | |
| "rewards/margins": 4.367648571654968, | |
| "rewards/rejected": -3.0815916515531994, | |
| "step": 590 | |
| }, | |
| { | |
| "epoch": 0.31512605042016806, | |
| "grad_norm": 57.39149614591673, | |
| "kl": 0.0, | |
| "learning_rate": 3.871729490045185e-07, | |
| "logps/chosen": -290.49703584558824, | |
| "logps/rejected": -252.63739583333333, | |
| "loss": 0.3164, | |
| "rewards/chosen": 0.8580891328699448, | |
| "rewards/margins": 3.7738939831303617, | |
| "rewards/rejected": -2.915804850260417, | |
| "step": 600 | |
| }, | |
| { | |
| "epoch": 0.32037815126050423, | |
| "grad_norm": 40.37544580915199, | |
| "kl": 0.0, | |
| "learning_rate": 3.837058394672196e-07, | |
| "logps/chosen": -273.4779296875, | |
| "logps/rejected": -287.0096923828125, | |
| "loss": 0.3289, | |
| "rewards/chosen": 0.3099039077758789, | |
| "rewards/margins": 3.173082160949707, | |
| "rewards/rejected": -2.863178253173828, | |
| "step": 610 | |
| }, | |
| { | |
| "epoch": 0.32563025210084034, | |
| "grad_norm": 57.45015573264363, | |
| "kl": 0.0, | |
| "learning_rate": 3.8020232951121166e-07, | |
| "logps/chosen": -290.50161637931035, | |
| "logps/rejected": -294.233125, | |
| "loss": 0.3188, | |
| "rewards/chosen": 0.1405291984821188, | |
| "rewards/margins": 3.6642485065178327, | |
| "rewards/rejected": -3.523719308035714, | |
| "step": 620 | |
| }, | |
| { | |
| "epoch": 0.33088235294117646, | |
| "grad_norm": 39.287989744489565, | |
| "kl": 0.0, | |
| "learning_rate": 3.7666337294097985e-07, | |
| "logps/chosen": -302.1064724392361, | |
| "logps/rejected": -260.4077814275568, | |
| "loss": 0.3435, | |
| "rewards/chosen": -0.013078765736685859, | |
| "rewards/margins": 3.530285672406958, | |
| "rewards/rejected": -3.5433644381436435, | |
| "step": 630 | |
| }, | |
| { | |
| "epoch": 0.33613445378151263, | |
| "grad_norm": 44.59203672342995, | |
| "kl": 0.0, | |
| "learning_rate": 3.730899332110855e-07, | |
| "logps/chosen": -281.5672392003676, | |
| "logps/rejected": -294.1050866168478, | |
| "loss": 0.2851, | |
| "rewards/chosen": 0.49381469277774587, | |
| "rewards/margins": 5.133490330727814, | |
| "rewards/rejected": -4.639675637950068, | |
| "step": 640 | |
| }, | |
| { | |
| "epoch": 0.34138655462184875, | |
| "grad_norm": 41.26338033254354, | |
| "kl": 0.0, | |
| "learning_rate": 3.694829831638738e-07, | |
| "logps/chosen": -272.1574797453704, | |
| "logps/rejected": -303.05520668512656, | |
| "loss": 0.3318, | |
| "rewards/chosen": -0.2948492309193552, | |
| "rewards/margins": 3.4522689825893473, | |
| "rewards/rejected": -3.7471182135087027, | |
| "step": 650 | |
| }, | |
| { | |
| "epoch": 0.34663865546218486, | |
| "grad_norm": 53.39190760647335, | |
| "kl": 0.0, | |
| "learning_rate": 3.658435047646238e-07, | |
| "logps/chosen": -281.7225378787879, | |
| "logps/rejected": -267.5343245967742, | |
| "loss": 0.3109, | |
| "rewards/chosen": 0.5971468838778409, | |
| "rewards/margins": 4.032071987834494, | |
| "rewards/rejected": -3.4349251039566533, | |
| "step": 660 | |
| }, | |
| { | |
| "epoch": 0.35189075630252103, | |
| "grad_norm": 50.14394406023911, | |
| "kl": 0.0, | |
| "learning_rate": 3.621724888342161e-07, | |
| "logps/chosen": -307.17963248239437, | |
| "logps/rejected": -244.93414238061797, | |
| "loss": 0.3456, | |
| "rewards/chosen": 0.5407970052369884, | |
| "rewards/margins": 2.9319001711099695, | |
| "rewards/rejected": -2.391103165872981, | |
| "step": 670 | |
| }, | |
| { | |
| "epoch": 0.35714285714285715, | |
| "grad_norm": 44.42377974125605, | |
| "kl": 0.0, | |
| "learning_rate": 3.584709347793895e-07, | |
| "logps/chosen": -328.5807938664596, | |
| "logps/rejected": -273.77439563679246, | |
| "loss": 0.3028, | |
| "rewards/chosen": 0.6922078103012179, | |
| "rewards/margins": 4.292771099535692, | |
| "rewards/rejected": -3.6005632892344734, | |
| "step": 680 | |
| }, | |
| { | |
| "epoch": 0.36239495798319327, | |
| "grad_norm": 64.33577601812173, | |
| "kl": 0.0, | |
| "learning_rate": 3.5473985032065946e-07, | |
| "logps/chosen": -339.54715008802816, | |
| "logps/rejected": -288.18354985955057, | |
| "loss": 0.3051, | |
| "rewards/chosen": 0.7634544372558594, | |
| "rewards/margins": 4.167792459552208, | |
| "rewards/rejected": -3.4043380222963484, | |
| "step": 690 | |
| }, | |
| { | |
| "epoch": 0.36764705882352944, | |
| "grad_norm": 50.09740849242821, | |
| "kl": 0.0, | |
| "learning_rate": 3.509802512179737e-07, | |
| "logps/chosen": -320.1032510080645, | |
| "logps/rejected": -247.99145359848484, | |
| "loss": 0.3144, | |
| "rewards/chosen": -0.7038151402627268, | |
| "rewards/margins": 4.308948235684243, | |
| "rewards/rejected": -5.01276337594697, | |
| "step": 700 | |
| }, | |
| { | |
| "epoch": 0.37289915966386555, | |
| "grad_norm": 48.91753923946882, | |
| "kl": 0.0, | |
| "learning_rate": 3.4719316099417983e-07, | |
| "logps/chosen": -319.73927600931677, | |
| "logps/rejected": -308.56738895440253, | |
| "loss": 0.3676, | |
| "rewards/chosen": -1.7911930439635093, | |
| "rewards/margins": 3.521101202302214, | |
| "rewards/rejected": -5.312294246265723, | |
| "step": 710 | |
| }, | |
| { | |
| "epoch": 0.37815126050420167, | |
| "grad_norm": 58.85183781985146, | |
| "kl": 0.0, | |
| "learning_rate": 3.4337961065637786e-07, | |
| "logps/chosen": -351.19349449685535, | |
| "logps/rejected": -328.6528289984472, | |
| "loss": 0.3281, | |
| "rewards/chosen": -0.32832849850444673, | |
| "rewards/margins": 4.381978936714885, | |
| "rewards/rejected": -4.710307435219332, | |
| "step": 720 | |
| }, | |
| { | |
| "epoch": 0.38340336134453784, | |
| "grad_norm": 34.12584073424692, | |
| "kl": 0.0, | |
| "learning_rate": 3.395406384152371e-07, | |
| "logps/chosen": -318.7544778963415, | |
| "logps/rejected": -281.5553635817308, | |
| "loss": 0.3211, | |
| "rewards/chosen": 0.22548980247683642, | |
| "rewards/margins": 3.761472765843223, | |
| "rewards/rejected": -3.5359829633663864, | |
| "step": 730 | |
| }, | |
| { | |
| "epoch": 0.38865546218487396, | |
| "grad_norm": 66.4865558017911, | |
| "kl": 0.0, | |
| "learning_rate": 3.356772894023505e-07, | |
| "logps/chosen": -274.7813126671123, | |
| "logps/rejected": -285.46543115601503, | |
| "loss": 0.376, | |
| "rewards/chosen": -0.13362450013186206, | |
| "rewards/margins": 3.3327875780371934, | |
| "rewards/rejected": -3.4664120781690553, | |
| "step": 740 | |
| }, | |
| { | |
| "epoch": 0.3939075630252101, | |
| "grad_norm": 23.48397277478164, | |
| "kl": 0.0, | |
| "learning_rate": 3.317906153857054e-07, | |
| "logps/chosen": -302.44775079617835, | |
| "logps/rejected": -287.7147958205521, | |
| "loss": 0.3351, | |
| "rewards/chosen": 0.14863140567852434, | |
| "rewards/margins": 4.358767210773425, | |
| "rewards/rejected": -4.2101358050949, | |
| "step": 750 | |
| }, | |
| { | |
| "epoch": 0.39915966386554624, | |
| "grad_norm": 34.1827992343062, | |
| "kl": 0.0, | |
| "learning_rate": 3.2788167448334784e-07, | |
| "logps/chosen": -262.08561812106916, | |
| "logps/rejected": -274.4614227484472, | |
| "loss": 0.374, | |
| "rewards/chosen": -1.007453294670057, | |
| "rewards/margins": 2.7304841113130562, | |
| "rewards/rejected": -3.7379374059831134, | |
| "step": 760 | |
| }, | |
| { | |
| "epoch": 0.40441176470588236, | |
| "grad_norm": 45.645905851061976, | |
| "kl": 0.0, | |
| "learning_rate": 3.2395153087531763e-07, | |
| "logps/chosen": -290.2492659395973, | |
| "logps/rejected": -247.69814510233917, | |
| "loss": 0.3295, | |
| "rewards/chosen": -0.16713363852276897, | |
| "rewards/margins": 3.4756729221497453, | |
| "rewards/rejected": -3.6428065606725144, | |
| "step": 770 | |
| }, | |
| { | |
| "epoch": 0.4096638655462185, | |
| "grad_norm": 52.083381199139474, | |
| "kl": 0.0, | |
| "learning_rate": 3.20001254513933e-07, | |
| "logps/chosen": -301.3512216862416, | |
| "logps/rejected": -320.7805875365497, | |
| "loss": 0.3564, | |
| "rewards/chosen": 0.2707460390641385, | |
| "rewards/margins": 3.1255901602721066, | |
| "rewards/rejected": -2.854844121207968, | |
| "step": 780 | |
| }, | |
| { | |
| "epoch": 0.41491596638655465, | |
| "grad_norm": 39.20450298136074, | |
| "kl": 0.0, | |
| "learning_rate": 3.160319208325044e-07, | |
| "logps/chosen": -320.55803206699346, | |
| "logps/rejected": -272.5804172904192, | |
| "loss": 0.3116, | |
| "rewards/chosen": 0.21087132871540543, | |
| "rewards/margins": 3.79839062605822, | |
| "rewards/rejected": -3.5875192973428143, | |
| "step": 790 | |
| }, | |
| { | |
| "epoch": 0.42016806722689076, | |
| "grad_norm": 49.5026281906496, | |
| "kl": 0.0, | |
| "learning_rate": 3.1204461045255597e-07, | |
| "logps/chosen": -305.87476748511904, | |
| "logps/rejected": -282.5663291529605, | |
| "loss": 0.3302, | |
| "rewards/chosen": 0.5268175942557198, | |
| "rewards/margins": 3.157528597609441, | |
| "rewards/rejected": -2.630711003353721, | |
| "step": 800 | |
| }, | |
| { | |
| "epoch": 0.4254201680672269, | |
| "grad_norm": 41.129555544878784, | |
| "kl": 0.0, | |
| "learning_rate": 3.0804040888963367e-07, | |
| "logps/chosen": -305.3494718309859, | |
| "logps/rejected": -257.2256846910112, | |
| "loss": 0.2981, | |
| "rewards/chosen": 0.24150611984897669, | |
| "rewards/margins": 4.293326681975907, | |
| "rewards/rejected": -4.051820562126931, | |
| "step": 810 | |
| }, | |
| { | |
| "epoch": 0.43067226890756305, | |
| "grad_norm": 42.46230974734401, | |
| "kl": 0.0, | |
| "learning_rate": 3.040204062577824e-07, | |
| "logps/chosen": -312.58470262096773, | |
| "logps/rejected": -261.4371448863636, | |
| "loss": 0.316, | |
| "rewards/chosen": 0.39130327778477825, | |
| "rewards/margins": 3.736164117628528, | |
| "rewards/rejected": -3.34486083984375, | |
| "step": 820 | |
| }, | |
| { | |
| "epoch": 0.43592436974789917, | |
| "grad_norm": 61.00539417491642, | |
| "kl": 0.0, | |
| "learning_rate": 2.999856969727704e-07, | |
| "logps/chosen": -305.4535361842105, | |
| "logps/rejected": -265.3578020134228, | |
| "loss": 0.3327, | |
| "rewards/chosen": 1.0649953920241686, | |
| "rewards/margins": 3.993097206857432, | |
| "rewards/rejected": -2.9281018148332634, | |
| "step": 830 | |
| }, | |
| { | |
| "epoch": 0.4411764705882353, | |
| "grad_norm": 37.0047213388179, | |
| "kl": 0.0, | |
| "learning_rate": 2.959373794541426e-07, | |
| "logps/chosen": -282.06252472310126, | |
| "logps/rejected": -256.6561053240741, | |
| "loss": 0.3265, | |
| "rewards/chosen": 0.3667142361025267, | |
| "rewards/margins": 3.1375505835176503, | |
| "rewards/rejected": -2.7708363474151234, | |
| "step": 840 | |
| }, | |
| { | |
| "epoch": 0.44642857142857145, | |
| "grad_norm": 36.25355732454002, | |
| "kl": 0.0, | |
| "learning_rate": 2.9187655582618407e-07, | |
| "logps/chosen": -313.9366851993865, | |
| "logps/rejected": -287.78662420382165, | |
| "loss": 0.3112, | |
| "rewards/chosen": 0.5094557686086081, | |
| "rewards/margins": 3.8714661655781546, | |
| "rewards/rejected": -3.3620103969695463, | |
| "step": 850 | |
| }, | |
| { | |
| "epoch": 0.45168067226890757, | |
| "grad_norm": 64.53574065510352, | |
| "kl": 0.0, | |
| "learning_rate": 2.878043316178753e-07, | |
| "logps/chosen": -283.32109866352204, | |
| "logps/rejected": -264.0886063664596, | |
| "loss": 0.3276, | |
| "rewards/chosen": 0.1551594764181653, | |
| "rewards/margins": 4.185146314054035, | |
| "rewards/rejected": -4.029986837635869, | |
| "step": 860 | |
| }, | |
| { | |
| "epoch": 0.4569327731092437, | |
| "grad_norm": 35.86505890578853, | |
| "kl": 0.0, | |
| "learning_rate": 2.837218154619193e-07, | |
| "logps/chosen": -312.15406436011904, | |
| "logps/rejected": -274.98959189967104, | |
| "loss": 0.3348, | |
| "rewards/chosen": -0.1483011245727539, | |
| "rewards/margins": 4.045587087932386, | |
| "rewards/rejected": -4.19388821250514, | |
| "step": 870 | |
| }, | |
| { | |
| "epoch": 0.46218487394957986, | |
| "grad_norm": 46.44246066253616, | |
| "kl": 0.0, | |
| "learning_rate": 2.796301187929257e-07, | |
| "logps/chosen": -316.67644557823127, | |
| "logps/rejected": -274.3472949783237, | |
| "loss": 0.331, | |
| "rewards/chosen": -0.024288294266681283, | |
| "rewards/margins": 3.5203153511903276, | |
| "rewards/rejected": -3.5446036454570087, | |
| "step": 880 | |
| }, | |
| { | |
| "epoch": 0.46743697478991597, | |
| "grad_norm": 67.03947638674309, | |
| "kl": 0.0, | |
| "learning_rate": 2.755303555448301e-07, | |
| "logps/chosen": -296.37250766871165, | |
| "logps/rejected": -287.87425358280257, | |
| "loss": 0.2935, | |
| "rewards/chosen": 0.2513600448889235, | |
| "rewards/margins": 4.716395453054728, | |
| "rewards/rejected": -4.465035408165805, | |
| "step": 890 | |
| }, | |
| { | |
| "epoch": 0.4726890756302521, | |
| "grad_norm": 44.0580333940613, | |
| "kl": 0.0, | |
| "learning_rate": 2.7142364184763424e-07, | |
| "logps/chosen": -275.6317777317881, | |
| "logps/rejected": -295.6349852071006, | |
| "loss": 0.3427, | |
| "rewards/chosen": -0.019468964330407956, | |
| "rewards/margins": 3.7003294679689636, | |
| "rewards/rejected": -3.7197984322993713, | |
| "step": 900 | |
| }, | |
| { | |
| "epoch": 0.47794117647058826, | |
| "grad_norm": 58.59520003164634, | |
| "kl": 0.0, | |
| "learning_rate": 2.673110957235479e-07, | |
| "logps/chosen": -305.7110357704403, | |
| "logps/rejected": -299.5851853649068, | |
| "loss": 0.2967, | |
| "rewards/chosen": 0.24971195436873525, | |
| "rewards/margins": 4.398466382260817, | |
| "rewards/rejected": -4.148754427892081, | |
| "step": 910 | |
| }, | |
| { | |
| "epoch": 0.4831932773109244, | |
| "grad_norm": 49.15023451078887, | |
| "kl": 0.0, | |
| "learning_rate": 2.6319383678261557e-07, | |
| "logps/chosen": -326.3589082154088, | |
| "logps/rejected": -291.8697350543478, | |
| "loss": 0.34, | |
| "rewards/chosen": -0.1070428524377211, | |
| "rewards/margins": 3.931509757592364, | |
| "rewards/rejected": -4.038552610030085, | |
| "step": 920 | |
| }, | |
| { | |
| "epoch": 0.4884453781512605, | |
| "grad_norm": 60.143948744622776, | |
| "kl": 0.0, | |
| "learning_rate": 2.5907298591791105e-07, | |
| "logps/chosen": -293.6828878012048, | |
| "logps/rejected": -316.3496601055195, | |
| "loss": 0.3121, | |
| "rewards/chosen": 0.37629242402961455, | |
| "rewards/margins": 4.533702884662731, | |
| "rewards/rejected": -4.157410460633117, | |
| "step": 930 | |
| }, | |
| { | |
| "epoch": 0.49369747899159666, | |
| "grad_norm": 41.39145780505376, | |
| "kl": 0.0, | |
| "learning_rate": 2.5494966500038264e-07, | |
| "logps/chosen": -298.37648703835225, | |
| "logps/rejected": -283.5568033854167, | |
| "loss": 0.3381, | |
| "rewards/chosen": 0.19053040851246228, | |
| "rewards/margins": 2.8964626018447106, | |
| "rewards/rejected": -2.705932193332248, | |
| "step": 940 | |
| }, | |
| { | |
| "epoch": 0.4989495798319328, | |
| "grad_norm": 44.85935706825639, | |
| "kl": 0.0, | |
| "learning_rate": 2.508249965734319e-07, | |
| "logps/chosen": -315.8209355828221, | |
| "logps/rejected": -277.33056329617835, | |
| "loss": 0.2815, | |
| "rewards/chosen": 0.5839554488293233, | |
| "rewards/margins": 4.811377494883264, | |
| "rewards/rejected": -4.227422046053941, | |
| "step": 950 | |
| }, | |
| { | |
| "epoch": 0.5042016806722689, | |
| "grad_norm": 86.97621065062417, | |
| "kl": 0.0, | |
| "learning_rate": 2.467001035473103e-07, | |
| "logps/chosen": -302.4603059505988, | |
| "logps/rejected": -287.89797794117646, | |
| "loss": 0.3293, | |
| "rewards/chosen": 0.40462347704493357, | |
| "rewards/margins": 3.534475173582924, | |
| "rewards/rejected": -3.1298516965379903, | |
| "step": 960 | |
| }, | |
| { | |
| "epoch": 0.509453781512605, | |
| "grad_norm": 32.22216772039752, | |
| "kl": 0.0, | |
| "learning_rate": 2.425761088934142e-07, | |
| "logps/chosen": -273.99361898291926, | |
| "logps/rejected": -258.7056308962264, | |
| "loss": 0.3326, | |
| "rewards/chosen": -0.09355034443162243, | |
| "rewards/margins": 4.204330392105327, | |
| "rewards/rejected": -4.29788073653695, | |
| "step": 970 | |
| }, | |
| { | |
| "epoch": 0.5147058823529411, | |
| "grad_norm": 53.215396805561575, | |
| "kl": 0.0, | |
| "learning_rate": 2.3845413533856514e-07, | |
| "logps/chosen": -299.4835397012579, | |
| "logps/rejected": -287.7150621118012, | |
| "loss": 0.2911, | |
| "rewards/chosen": 0.30934901207498033, | |
| "rewards/margins": 4.708151206611083, | |
| "rewards/rejected": -4.398802194536103, | |
| "step": 980 | |
| }, | |
| { | |
| "epoch": 0.5199579831932774, | |
| "grad_norm": 70.13460679739038, | |
| "kl": 0.0, | |
| "learning_rate": 2.343353050593553e-07, | |
| "logps/chosen": -318.12441770186336, | |
| "logps/rejected": -282.98125491352204, | |
| "loss": 0.3042, | |
| "rewards/chosen": 0.9980694314707881, | |
| "rewards/margins": 5.077612059345198, | |
| "rewards/rejected": -4.07954262787441, | |
| "step": 990 | |
| }, | |
| { | |
| "epoch": 0.5252100840336135, | |
| "grad_norm": 91.66490200361514, | |
| "kl": 0.0, | |
| "learning_rate": 2.3022073937664383e-07, | |
| "logps/chosen": -318.62791700487014, | |
| "logps/rejected": -299.1699924698795, | |
| "loss": 0.2987, | |
| "rewards/chosen": -0.4350905975738129, | |
| "rewards/margins": 3.7659268511272415, | |
| "rewards/rejected": -4.201017448701054, | |
| "step": 1000 | |
| }, | |
| { | |
| "epoch": 0.5304621848739496, | |
| "grad_norm": 47.014009528040546, | |
| "kl": 0.0, | |
| "learning_rate": 2.261115584502849e-07, | |
| "logps/chosen": -322.26196442610063, | |
| "logps/rejected": -273.58297748447205, | |
| "loss": 0.3504, | |
| "rewards/chosen": -0.4901385037404186, | |
| "rewards/margins": 2.980573340567617, | |
| "rewards/rejected": -3.4707118443080356, | |
| "step": 1010 | |
| }, | |
| { | |
| "epoch": 0.5357142857142857, | |
| "grad_norm": 65.73363572647297, | |
| "kl": 0.0, | |
| "learning_rate": 2.2200888097417302e-07, | |
| "logps/chosen": -265.984912109375, | |
| "logps/rejected": -273.97919921875, | |
| "loss": 0.3327, | |
| "rewards/chosen": 0.2538748264312744, | |
| "rewards/margins": 3.538307809829712, | |
| "rewards/rejected": -3.2844329833984376, | |
| "step": 1020 | |
| }, | |
| { | |
| "epoch": 0.5409663865546218, | |
| "grad_norm": 37.39061372036576, | |
| "kl": 0.0, | |
| "learning_rate": 2.1791382387168684e-07, | |
| "logps/chosen": -294.4272490530303, | |
| "logps/rejected": -283.10264616935484, | |
| "loss": 0.3218, | |
| "rewards/chosen": 0.5568286317767519, | |
| "rewards/margins": 4.048672366025744, | |
| "rewards/rejected": -3.491843734248992, | |
| "step": 1030 | |
| }, | |
| { | |
| "epoch": 0.5462184873949579, | |
| "grad_norm": 38.42991974376242, | |
| "kl": 0.0, | |
| "learning_rate": 2.1382750199161495e-07, | |
| "logps/chosen": -314.6301011029412, | |
| "logps/rejected": -277.451328125, | |
| "loss": 0.3087, | |
| "rewards/chosen": 0.9661560956169577, | |
| "rewards/margins": 3.718452645096124, | |
| "rewards/rejected": -2.7522965494791665, | |
| "step": 1040 | |
| }, | |
| { | |
| "epoch": 0.5514705882352942, | |
| "grad_norm": 58.32907293331433, | |
| "kl": 0.0, | |
| "learning_rate": 2.0975102780464673e-07, | |
| "logps/chosen": -306.8538490853659, | |
| "logps/rejected": -290.90675080128204, | |
| "loss": 0.3062, | |
| "rewards/chosen": 0.684349339182784, | |
| "rewards/margins": 3.878205602954819, | |
| "rewards/rejected": -3.193856263772035, | |
| "step": 1050 | |
| }, | |
| { | |
| "epoch": 0.5567226890756303, | |
| "grad_norm": 42.31783649576492, | |
| "kl": 0.0, | |
| "learning_rate": 2.0568551110051074e-07, | |
| "logps/chosen": -265.01874508647796, | |
| "logps/rejected": -256.457298136646, | |
| "loss": 0.3375, | |
| "rewards/chosen": 0.7610905485333137, | |
| "rewards/margins": 3.0508450962211047, | |
| "rewards/rejected": -2.289754547687791, | |
| "step": 1060 | |
| }, | |
| { | |
| "epoch": 0.5619747899159664, | |
| "grad_norm": 43.282189967649956, | |
| "kl": 0.0, | |
| "learning_rate": 2.016320586858422e-07, | |
| "logps/chosen": -270.2985276442308, | |
| "logps/rejected": -287.5562357088415, | |
| "loss": 0.3143, | |
| "rewards/chosen": 1.108019804343199, | |
| "rewards/margins": 3.3270524641064423, | |
| "rewards/rejected": -2.219032659763243, | |
| "step": 1070 | |
| }, | |
| { | |
| "epoch": 0.5672268907563025, | |
| "grad_norm": 59.248701734675386, | |
| "kl": 0.0, | |
| "learning_rate": 1.9759177408286337e-07, | |
| "logps/chosen": -312.609375, | |
| "logps/rejected": -267.16066028225805, | |
| "loss": 0.3365, | |
| "rewards/chosen": 1.2182994495738637, | |
| "rewards/margins": 3.07108659045088, | |
| "rewards/rejected": -1.8527871408770162, | |
| "step": 1080 | |
| }, | |
| { | |
| "epoch": 0.5724789915966386, | |
| "grad_norm": 45.19596654926437, | |
| "kl": 0.0, | |
| "learning_rate": 1.9356575722895808e-07, | |
| "logps/chosen": -299.373361013986, | |
| "logps/rejected": -271.5683042019774, | |
| "loss": 0.3178, | |
| "rewards/chosen": 1.0334772496790319, | |
| "rewards/margins": 3.244971479960194, | |
| "rewards/rejected": -2.2114942302811618, | |
| "step": 1090 | |
| }, | |
| { | |
| "epoch": 0.5777310924369747, | |
| "grad_norm": 44.21402337263303, | |
| "kl": 0.0, | |
| "learning_rate": 1.895551041772216e-07, | |
| "logps/chosen": -309.0861150568182, | |
| "logps/rejected": -255.60587565104166, | |
| "loss": 0.3402, | |
| "rewards/chosen": 0.8877213217995383, | |
| "rewards/margins": 3.0730268112336745, | |
| "rewards/rejected": -2.1853054894341364, | |
| "step": 1100 | |
| }, | |
| { | |
| "epoch": 0.582983193277311, | |
| "grad_norm": 61.69332868228022, | |
| "kl": 0.0, | |
| "learning_rate": 1.8556090679806847e-07, | |
| "logps/chosen": -298.82965455572287, | |
| "logps/rejected": -267.7830509334416, | |
| "loss": 0.2942, | |
| "rewards/chosen": 1.1525323706937123, | |
| "rewards/margins": 4.799233143065871, | |
| "rewards/rejected": -3.646700772372159, | |
| "step": 1110 | |
| }, | |
| { | |
| "epoch": 0.5882352941176471, | |
| "grad_norm": 31.77308050751697, | |
| "kl": 0.0, | |
| "learning_rate": 1.8158425248197928e-07, | |
| "logps/chosen": -330.5060499237805, | |
| "logps/rejected": -304.98880709134613, | |
| "loss": 0.2989, | |
| "rewards/chosen": 1.3816723707245617, | |
| "rewards/margins": 5.020397847111781, | |
| "rewards/rejected": -3.6387254763872194, | |
| "step": 1120 | |
| }, | |
| { | |
| "epoch": 0.5934873949579832, | |
| "grad_norm": 59.622620200950486, | |
| "kl": 0.0, | |
| "learning_rate": 1.7762622384346609e-07, | |
| "logps/chosen": -292.1063850308642, | |
| "logps/rejected": -255.3093848892405, | |
| "loss": 0.311, | |
| "rewards/chosen": 1.0882685155044367, | |
| "rewards/margins": 3.8796188681176687, | |
| "rewards/rejected": -2.791350352613232, | |
| "step": 1130 | |
| }, | |
| { | |
| "epoch": 0.5987394957983193, | |
| "grad_norm": 92.89241064630508, | |
| "kl": 0.0, | |
| "learning_rate": 1.7368789842633907e-07, | |
| "logps/chosen": -324.4837362421384, | |
| "logps/rejected": -303.8497670807453, | |
| "loss": 0.2984, | |
| "rewards/chosen": 0.8251325499336675, | |
| "rewards/margins": 4.589403024385337, | |
| "rewards/rejected": -3.7642704744516693, | |
| "step": 1140 | |
| }, | |
| { | |
| "epoch": 0.6039915966386554, | |
| "grad_norm": 42.994311331506765, | |
| "kl": 0.0, | |
| "learning_rate": 1.697703484103532e-07, | |
| "logps/chosen": -292.4465987042683, | |
| "logps/rejected": -259.40554887820514, | |
| "loss": 0.3131, | |
| "rewards/chosen": 0.9567713853789539, | |
| "rewards/margins": 4.075035216884363, | |
| "rewards/rejected": -3.1182638315054088, | |
| "step": 1150 | |
| }, | |
| { | |
| "epoch": 0.6092436974789915, | |
| "grad_norm": 63.999900745814266, | |
| "kl": 0.0, | |
| "learning_rate": 1.6587464031931526e-07, | |
| "logps/chosen": -303.69353428171644, | |
| "logps/rejected": -277.118405577957, | |
| "loss": 0.3398, | |
| "rewards/chosen": 0.867815444718546, | |
| "rewards/margins": 3.5977176506472253, | |
| "rewards/rejected": -2.7299022059286795, | |
| "step": 1160 | |
| }, | |
| { | |
| "epoch": 0.6144957983193278, | |
| "grad_norm": 33.78499266925509, | |
| "kl": 0.0, | |
| "learning_rate": 1.6200183473073048e-07, | |
| "logps/chosen": -299.6548768939394, | |
| "logps/rejected": -280.22903225806454, | |
| "loss": 0.2992, | |
| "rewards/chosen": 0.6621969512014678, | |
| "rewards/margins": 3.7750548928597336, | |
| "rewards/rejected": -3.112857941658266, | |
| "step": 1170 | |
| }, | |
| { | |
| "epoch": 0.6197478991596639, | |
| "grad_norm": 31.77803791723765, | |
| "kl": 0.0, | |
| "learning_rate": 1.5815298598706888e-07, | |
| "logps/chosen": -291.1797602872671, | |
| "logps/rejected": -286.78439465408803, | |
| "loss": 0.3176, | |
| "rewards/chosen": -0.010194837676812403, | |
| "rewards/margins": 4.250023752633722, | |
| "rewards/rejected": -4.260218590310535, | |
| "step": 1180 | |
| }, | |
| { | |
| "epoch": 0.625, | |
| "grad_norm": 37.859068866257594, | |
| "kl": 0.0, | |
| "learning_rate": 1.5432914190872756e-07, | |
| "logps/chosen": -283.2485907832278, | |
| "logps/rejected": -299.72665895061726, | |
| "loss": 0.3475, | |
| "rewards/chosen": 0.012598955178562599, | |
| "rewards/margins": 4.188036976614712, | |
| "rewards/rejected": -4.1754380214361495, | |
| "step": 1190 | |
| }, | |
| { | |
| "epoch": 0.6302521008403361, | |
| "grad_norm": 34.730160989586295, | |
| "kl": 0.0, | |
| "learning_rate": 1.505313435087698e-07, | |
| "logps/chosen": -300.31629922945206, | |
| "logps/rejected": -295.37841235632186, | |
| "loss": 0.3124, | |
| "rewards/chosen": 0.33270381248160585, | |
| "rewards/margins": 4.571263930005852, | |
| "rewards/rejected": -4.238560117524246, | |
| "step": 1200 | |
| }, | |
| { | |
| "epoch": 0.6355042016806722, | |
| "grad_norm": 73.6570576556008, | |
| "kl": 0.0, | |
| "learning_rate": 1.4676062470951705e-07, | |
| "logps/chosen": -316.37948140337426, | |
| "logps/rejected": -306.59623805732485, | |
| "loss": 0.318, | |
| "rewards/chosen": 0.23967742919921875, | |
| "rewards/margins": 4.2318552102252935, | |
| "rewards/rejected": -3.992177781026075, | |
| "step": 1210 | |
| }, | |
| { | |
| "epoch": 0.6407563025210085, | |
| "grad_norm": 86.8105694173325, | |
| "kl": 0.0, | |
| "learning_rate": 1.430180120610711e-07, | |
| "logps/chosen": -283.5435161226115, | |
| "logps/rejected": -276.21841449386505, | |
| "loss": 0.2916, | |
| "rewards/chosen": 0.35688832762894357, | |
| "rewards/margins": 4.4911308476394876, | |
| "rewards/rejected": -4.134242520010544, | |
| "step": 1220 | |
| }, | |
| { | |
| "epoch": 0.6460084033613446, | |
| "grad_norm": 65.59031614397247, | |
| "kl": 0.0, | |
| "learning_rate": 1.3930452446184385e-07, | |
| "logps/chosen": -324.24284699675326, | |
| "logps/rejected": -301.1185523343373, | |
| "loss": 0.3146, | |
| "rewards/chosen": 0.3668579745602298, | |
| "rewards/margins": 4.802705980843173, | |
| "rewards/rejected": -4.435848006282944, | |
| "step": 1230 | |
| }, | |
| { | |
| "epoch": 0.6512605042016807, | |
| "grad_norm": 82.83869882352816, | |
| "kl": 0.0, | |
| "learning_rate": 1.3562117288116923e-07, | |
| "logps/chosen": -315.7268702651515, | |
| "logps/rejected": -280.67207661290325, | |
| "loss": 0.3109, | |
| "rewards/chosen": 0.6113425514914773, | |
| "rewards/margins": 4.853238264067082, | |
| "rewards/rejected": -4.241895712575605, | |
| "step": 1240 | |
| }, | |
| { | |
| "epoch": 0.6565126050420168, | |
| "grad_norm": 53.82976674411678, | |
| "kl": 0.0, | |
| "learning_rate": 1.319689600840747e-07, | |
| "logps/chosen": -304.3841062898089, | |
| "logps/rejected": -285.0993577453988, | |
| "loss": 0.3324, | |
| "rewards/chosen": 0.08269594884981775, | |
| "rewards/margins": 3.7769891991853237, | |
| "rewards/rejected": -3.694293250335506, | |
| "step": 1250 | |
| }, | |
| { | |
| "epoch": 0.6617647058823529, | |
| "grad_norm": 34.58790215772178, | |
| "kl": 0.0, | |
| "learning_rate": 1.2834888035828596e-07, | |
| "logps/chosen": -301.4619855182927, | |
| "logps/rejected": -272.6404246794872, | |
| "loss": 0.2994, | |
| "rewards/chosen": 0.32138896569973086, | |
| "rewards/margins": 4.973683418073529, | |
| "rewards/rejected": -4.652294452373798, | |
| "step": 1260 | |
| }, | |
| { | |
| "epoch": 0.667016806722689, | |
| "grad_norm": 29.984315889657353, | |
| "kl": 0.0, | |
| "learning_rate": 1.2476191924353932e-07, | |
| "logps/chosen": -345.89991071428574, | |
| "logps/rejected": -282.63448275862066, | |
| "loss": 0.326, | |
| "rewards/chosen": 0.5500016566685267, | |
| "rewards/margins": 4.73898333770301, | |
| "rewards/rejected": -4.188981681034483, | |
| "step": 1270 | |
| }, | |
| { | |
| "epoch": 0.6722689075630253, | |
| "grad_norm": 61.179118361711794, | |
| "kl": 0.0, | |
| "learning_rate": 1.2120905326327596e-07, | |
| "logps/chosen": -298.07992788461536, | |
| "logps/rejected": -293.07983236754967, | |
| "loss": 0.3307, | |
| "rewards/chosen": 0.1393580013478296, | |
| "rewards/margins": 4.935787970175437, | |
| "rewards/rejected": -4.796429968827607, | |
| "step": 1280 | |
| }, | |
| { | |
| "epoch": 0.6775210084033614, | |
| "grad_norm": 35.426220175192356, | |
| "kl": 0.0, | |
| "learning_rate": 1.1769124965879091e-07, | |
| "logps/chosen": -309.7057636589404, | |
| "logps/rejected": -253.99468380177515, | |
| "loss": 0.3127, | |
| "rewards/chosen": 0.3561096696664166, | |
| "rewards/margins": 4.979999841056024, | |
| "rewards/rejected": -4.623890171389608, | |
| "step": 1290 | |
| }, | |
| { | |
| "epoch": 0.6827731092436975, | |
| "grad_norm": 43.346023169027426, | |
| "kl": 0.0, | |
| "learning_rate": 1.1420946612590837e-07, | |
| "logps/chosen": -270.4368489583333, | |
| "logps/rejected": -281.6362047697368, | |
| "loss": 0.3397, | |
| "rewards/chosen": 0.1588947886512393, | |
| "rewards/margins": 3.5548700019530486, | |
| "rewards/rejected": -3.395975213301809, | |
| "step": 1300 | |
| }, | |
| { | |
| "epoch": 0.6880252100840336, | |
| "grad_norm": 45.324757492893184, | |
| "kl": 0.0, | |
| "learning_rate": 1.1076465055425646e-07, | |
| "logps/chosen": -290.0300690406977, | |
| "logps/rejected": -258.5001319679054, | |
| "loss": 0.3082, | |
| "rewards/chosen": 0.7295433754144713, | |
| "rewards/margins": 4.570315849548762, | |
| "rewards/rejected": -3.8407724741342903, | |
| "step": 1310 | |
| }, | |
| { | |
| "epoch": 0.6932773109243697, | |
| "grad_norm": 263.89653772607954, | |
| "kl": 0.0, | |
| "learning_rate": 1.0735774076921128e-07, | |
| "logps/chosen": -287.7608642578125, | |
| "logps/rejected": -237.489306640625, | |
| "loss": 0.3446, | |
| "rewards/chosen": 0.27030837535858154, | |
| "rewards/margins": 3.555664134025574, | |
| "rewards/rejected": -3.2853557586669924, | |
| "step": 1320 | |
| }, | |
| { | |
| "epoch": 0.6985294117647058, | |
| "grad_norm": 61.837360014332575, | |
| "kl": 0.0, | |
| "learning_rate": 1.039896642765809e-07, | |
| "logps/chosen": -297.6917317708333, | |
| "logps/rejected": -314.11572265625, | |
| "loss": 0.2958, | |
| "rewards/chosen": 0.6960362328423394, | |
| "rewards/margins": 5.248802512583106, | |
| "rewards/rejected": -4.552766279740767, | |
| "step": 1330 | |
| }, | |
| { | |
| "epoch": 0.7037815126050421, | |
| "grad_norm": 51.294856185176336, | |
| "kl": 0.0, | |
| "learning_rate": 1.0066133801009871e-07, | |
| "logps/chosen": -328.30787295386904, | |
| "logps/rejected": -244.40604440789474, | |
| "loss": 0.3097, | |
| "rewards/chosen": 0.7595687593732562, | |
| "rewards/margins": 4.201100693609481, | |
| "rewards/rejected": -3.441531934236225, | |
| "step": 1340 | |
| }, | |
| { | |
| "epoch": 0.7090336134453782, | |
| "grad_norm": 43.77043748464739, | |
| "kl": 0.0, | |
| "learning_rate": 9.737366808179553e-08, | |
| "logps/chosen": -303.54956371753246, | |
| "logps/rejected": -250.67248682228916, | |
| "loss": 0.3096, | |
| "rewards/chosen": 0.9003205732865767, | |
| "rewards/margins": 4.412543414848136, | |
| "rewards/rejected": -3.5122228415615586, | |
| "step": 1350 | |
| }, | |
| { | |
| "epoch": 0.7142857142857143, | |
| "grad_norm": 72.17773141718298, | |
| "kl": 0.0, | |
| "learning_rate": 9.412754953531663e-08, | |
| "logps/chosen": -262.4382017215569, | |
| "logps/rejected": -292.5083997140523, | |
| "loss": 0.3377, | |
| "rewards/chosen": 0.434063574511134, | |
| "rewards/margins": 3.880319925358765, | |
| "rewards/rejected": -3.446256350847631, | |
| "step": 1360 | |
| }, | |
| { | |
| "epoch": 0.7195378151260504, | |
| "grad_norm": 61.94872379316782, | |
| "kl": 0.0, | |
| "learning_rate": 9.092386610225325e-08, | |
| "logps/chosen": -265.7097810444079, | |
| "logps/rejected": -279.9903273809524, | |
| "loss": 0.3299, | |
| "rewards/chosen": 0.8910425085770456, | |
| "rewards/margins": 3.7223031897293892, | |
| "rewards/rejected": -2.8312606811523438, | |
| "step": 1370 | |
| }, | |
| { | |
| "epoch": 0.7247899159663865, | |
| "grad_norm": 47.60678426366227, | |
| "kl": 0.0, | |
| "learning_rate": 8.776348996155317e-08, | |
| "logps/chosen": -290.0428740530303, | |
| "logps/rejected": -278.20430947580644, | |
| "loss": 0.3123, | |
| "rewards/chosen": 0.3940057927911932, | |
| "rewards/margins": 4.4418612930432095, | |
| "rewards/rejected": -4.047855500252016, | |
| "step": 1380 | |
| }, | |
| { | |
| "epoch": 0.7300420168067226, | |
| "grad_norm": 41.557754145037535, | |
| "kl": 0.0, | |
| "learning_rate": 8.464728150207636e-08, | |
| "logps/chosen": -295.76380699685535, | |
| "logps/rejected": -294.7458753881988, | |
| "loss": 0.3185, | |
| "rewards/chosen": 0.6997887953272406, | |
| "rewards/margins": 4.221460825000183, | |
| "rewards/rejected": -3.5216720296729425, | |
| "step": 1390 | |
| }, | |
| { | |
| "epoch": 0.7352941176470589, | |
| "grad_norm": 45.38199821794469, | |
| "kl": 0.0, | |
| "learning_rate": 8.15760890883607e-08, | |
| "logps/chosen": -275.74736265923565, | |
| "logps/rejected": -261.5596721625767, | |
| "loss": 0.3109, | |
| "rewards/chosen": 0.7350611109642466, | |
| "rewards/margins": 4.5443022829590705, | |
| "rewards/rejected": -3.809241171994824, | |
| "step": 1400 | |
| }, | |
| { | |
| "epoch": 0.740546218487395, | |
| "grad_norm": 22.290814745353508, | |
| "kl": 0.0, | |
| "learning_rate": 7.855074882966103e-08, | |
| "logps/chosen": -311.78690011160717, | |
| "logps/rejected": -279.9527652138158, | |
| "loss": 0.2985, | |
| "rewards/chosen": 0.7244267236618769, | |
| "rewards/margins": 5.052972545002337, | |
| "rewards/rejected": -4.328545821340461, | |
| "step": 1410 | |
| }, | |
| { | |
| "epoch": 0.7457983193277311, | |
| "grad_norm": 40.315164584155504, | |
| "kl": 0.0, | |
| "learning_rate": 7.557208435232449e-08, | |
| "logps/chosen": -274.712925295858, | |
| "logps/rejected": -278.3036268625828, | |
| "loss": 0.3147, | |
| "rewards/chosen": 1.077583267843935, | |
| "rewards/margins": 4.07225724310004, | |
| "rewards/rejected": -2.9946739752561053, | |
| "step": 1420 | |
| }, | |
| { | |
| "epoch": 0.7510504201680672, | |
| "grad_norm": 36.626625075926555, | |
| "kl": 0.0, | |
| "learning_rate": 7.264090657556443e-08, | |
| "logps/chosen": -292.9152413091716, | |
| "logps/rejected": -242.9820467715232, | |
| "loss": 0.3085, | |
| "rewards/chosen": 0.9502442489714312, | |
| "rewards/margins": 4.543455846136166, | |
| "rewards/rejected": -3.593211597164735, | |
| "step": 1430 | |
| }, | |
| { | |
| "epoch": 0.7563025210084033, | |
| "grad_norm": 43.54130112641965, | |
| "kl": 0.0, | |
| "learning_rate": 6.975801349069385e-08, | |
| "logps/chosen": -280.07919034090907, | |
| "logps/rejected": -276.84921875, | |
| "loss": 0.2987, | |
| "rewards/chosen": 0.817291444720644, | |
| "rewards/margins": 5.2739698721399995, | |
| "rewards/rejected": -4.456678427419355, | |
| "step": 1440 | |
| }, | |
| { | |
| "epoch": 0.7615546218487395, | |
| "grad_norm": 39.41222523190064, | |
| "kl": 0.0, | |
| "learning_rate": 6.692418994387799e-08, | |
| "logps/chosen": -288.40252246732024, | |
| "logps/rejected": -284.47291354790417, | |
| "loss": 0.2993, | |
| "rewards/chosen": 0.9826148538028493, | |
| "rewards/margins": 5.268814549138741, | |
| "rewards/rejected": -4.286199695335891, | |
| "step": 1450 | |
| }, | |
| { | |
| "epoch": 0.7668067226890757, | |
| "grad_norm": 42.76397567606349, | |
| "kl": 0.0, | |
| "learning_rate": 6.414020742246593e-08, | |
| "logps/chosen": -288.7798685213415, | |
| "logps/rejected": -293.82729867788464, | |
| "loss": 0.2617, | |
| "rewards/chosen": 0.9620797692275629, | |
| "rewards/margins": 5.808872528267026, | |
| "rewards/rejected": -4.846792759039463, | |
| "step": 1460 | |
| }, | |
| { | |
| "epoch": 0.7720588235294118, | |
| "grad_norm": 85.5452615105662, | |
| "kl": 0.0, | |
| "learning_rate": 6.140682384495902e-08, | |
| "logps/chosen": -311.87120301573424, | |
| "logps/rejected": -282.2307997881356, | |
| "loss": 0.2877, | |
| "rewards/chosen": 0.8217522681176246, | |
| "rewards/margins": 4.416604004412646, | |
| "rewards/rejected": -3.594851736295021, | |
| "step": 1470 | |
| }, | |
| { | |
| "epoch": 0.7773109243697479, | |
| "grad_norm": 47.14939912048749, | |
| "kl": 0.0, | |
| "learning_rate": 5.872478335467298e-08, | |
| "logps/chosen": -270.7433546112805, | |
| "logps/rejected": -274.62745392628204, | |
| "loss": 0.329, | |
| "rewards/chosen": -0.1099286428311976, | |
| "rewards/margins": 4.24396219187934, | |
| "rewards/rejected": -4.353890834710537, | |
| "step": 1480 | |
| }, | |
| { | |
| "epoch": 0.782563025210084, | |
| "grad_norm": 75.39266090840431, | |
| "kl": 0.0, | |
| "learning_rate": 5.60948161171505e-08, | |
| "logps/chosen": -298.3712257179054, | |
| "logps/rejected": -296.9724518531977, | |
| "loss": 0.3039, | |
| "rewards/chosen": 0.3439343168928817, | |
| "rewards/margins": 4.58285359718903, | |
| "rewards/rejected": -4.238919280296148, | |
| "step": 1490 | |
| }, | |
| { | |
| "epoch": 0.7878151260504201, | |
| "grad_norm": 101.47398109074602, | |
| "kl": 0.0, | |
| "learning_rate": 5.351763812137916e-08, | |
| "logps/chosen": -285.37012357026146, | |
| "logps/rejected": -305.706282747006, | |
| "loss": 0.3075, | |
| "rewards/chosen": -0.05964530371372996, | |
| "rewards/margins": 4.639130484553485, | |
| "rewards/rejected": -4.698775788267215, | |
| "step": 1500 | |
| }, | |
| { | |
| "epoch": 0.7930672268907563, | |
| "grad_norm": 51.64579106329186, | |
| "kl": 0.0, | |
| "learning_rate": 5.0993950984868836e-08, | |
| "logps/chosen": -273.9158336568323, | |
| "logps/rejected": -296.97481819968556, | |
| "loss": 0.3228, | |
| "rewards/chosen": -0.35097162471794935, | |
| "rewards/margins": 4.398951601500603, | |
| "rewards/rejected": -4.749923226218553, | |
| "step": 1510 | |
| }, | |
| { | |
| "epoch": 0.7983193277310925, | |
| "grad_norm": 164.1846468516791, | |
| "kl": 0.0, | |
| "learning_rate": 4.8524441762641284e-08, | |
| "logps/chosen": -323.8402122641509, | |
| "logps/rejected": -265.2092148680124, | |
| "loss": 0.3144, | |
| "rewards/chosen": 0.010938104593528892, | |
| "rewards/margins": 4.735114546994538, | |
| "rewards/rejected": -4.724176442401009, | |
| "step": 1520 | |
| }, | |
| { | |
| "epoch": 0.8035714285714286, | |
| "grad_norm": 57.97209020416802, | |
| "kl": 0.0, | |
| "learning_rate": 4.6109782760184956e-08, | |
| "logps/chosen": -283.25261976047904, | |
| "logps/rejected": -294.36603860294116, | |
| "loss": 0.3462, | |
| "rewards/chosen": 0.19619906305552956, | |
| "rewards/margins": 4.044921234215661, | |
| "rewards/rejected": -3.848722171160131, | |
| "step": 1530 | |
| }, | |
| { | |
| "epoch": 0.8088235294117647, | |
| "grad_norm": 41.10976267046813, | |
| "kl": 0.0, | |
| "learning_rate": 4.375063135042445e-08, | |
| "logps/chosen": -301.61559264053255, | |
| "logps/rejected": -321.35994412251654, | |
| "loss": 0.3079, | |
| "rewards/chosen": 0.2954674895698502, | |
| "rewards/margins": 5.432189877426845, | |
| "rewards/rejected": -5.136722387856995, | |
| "step": 1540 | |
| }, | |
| { | |
| "epoch": 0.8140756302521008, | |
| "grad_norm": 54.92221531592682, | |
| "kl": 0.0, | |
| "learning_rate": 4.144762979475575e-08, | |
| "logps/chosen": -306.94077797202794, | |
| "logps/rejected": -280.2052436440678, | |
| "loss": 0.315, | |
| "rewards/chosen": -0.0729095952494161, | |
| "rewards/margins": 4.834762834239461, | |
| "rewards/rejected": -4.907672429488877, | |
| "step": 1550 | |
| }, | |
| { | |
| "epoch": 0.819327731092437, | |
| "grad_norm": 48.43314600236666, | |
| "kl": 0.0, | |
| "learning_rate": 3.9201405068195385e-08, | |
| "logps/chosen": -312.2001696134868, | |
| "logps/rejected": -292.7070777529762, | |
| "loss": 0.3395, | |
| "rewards/chosen": 0.025735384539553995, | |
| "rewards/margins": 4.3085366947609085, | |
| "rewards/rejected": -4.2828013102213545, | |
| "step": 1560 | |
| }, | |
| { | |
| "epoch": 0.8245798319327731, | |
| "grad_norm": 82.79685734368942, | |
| "kl": 0.0, | |
| "learning_rate": 3.701256868869124e-08, | |
| "logps/chosen": -321.1130847091195, | |
| "logps/rejected": -254.03585986024845, | |
| "loss": 0.3356, | |
| "rewards/chosen": 0.22610101759808618, | |
| "rewards/margins": 4.19767455700026, | |
| "rewards/rejected": -3.971573539402174, | |
| "step": 1570 | |
| }, | |
| { | |
| "epoch": 0.8298319327731093, | |
| "grad_norm": 69.63465669746606, | |
| "kl": 0.0, | |
| "learning_rate": 3.488171655064107e-08, | |
| "logps/chosen": -289.7723396650327, | |
| "logps/rejected": -288.95059880239523, | |
| "loss": 0.3227, | |
| "rewards/chosen": -0.3906313079634523, | |
| "rewards/margins": 4.496159441592124, | |
| "rewards/rejected": -4.886790749555576, | |
| "step": 1580 | |
| }, | |
| { | |
| "epoch": 0.8350840336134454, | |
| "grad_norm": 66.42429414400927, | |
| "kl": 0.0, | |
| "learning_rate": 3.28094287626651e-08, | |
| "logps/chosen": -340.9739889705882, | |
| "logps/rejected": -299.48609375, | |
| "loss": 0.3308, | |
| "rewards/chosen": 0.048118759604061354, | |
| "rewards/margins": 4.789248316895728, | |
| "rewards/rejected": -4.741129557291667, | |
| "step": 1590 | |
| }, | |
| { | |
| "epoch": 0.8403361344537815, | |
| "grad_norm": 42.16586593578885, | |
| "kl": 0.0, | |
| "learning_rate": 3.079626948967534e-08, | |
| "logps/chosen": -312.65874787414964, | |
| "logps/rejected": -287.83103775289015, | |
| "loss": 0.3062, | |
| "rewards/chosen": -0.0730894049819635, | |
| "rewards/margins": 4.740564265312472, | |
| "rewards/rejected": -4.813653670294436, | |
| "step": 1600 | |
| }, | |
| { | |
| "epoch": 0.8455882352941176, | |
| "grad_norm": 42.50176028584879, | |
| "kl": 0.0, | |
| "learning_rate": 2.88427867992862e-08, | |
| "logps/chosen": -298.3803453947368, | |
| "logps/rejected": -256.55287856543623, | |
| "loss": 0.3216, | |
| "rewards/chosen": 0.292915611936335, | |
| "rewards/margins": 4.54555282224464, | |
| "rewards/rejected": -4.252637210308305, | |
| "step": 1610 | |
| }, | |
| { | |
| "epoch": 0.8508403361344538, | |
| "grad_norm": 83.97476475765178, | |
| "kl": 0.0, | |
| "learning_rate": 2.6949512512606965e-08, | |
| "logps/chosen": -293.518928433642, | |
| "logps/rejected": -296.5833910205696, | |
| "loss": 0.2953, | |
| "rewards/chosen": 0.5612210874204282, | |
| "rewards/margins": 5.312669320336765, | |
| "rewards/rejected": -4.751448232916337, | |
| "step": 1620 | |
| }, | |
| { | |
| "epoch": 0.8560924369747899, | |
| "grad_norm": 35.09955655518953, | |
| "kl": 0.0, | |
| "learning_rate": 2.5116962059457653e-08, | |
| "logps/chosen": -307.30205078125, | |
| "logps/rejected": -280.090478515625, | |
| "loss": 0.2881, | |
| "rewards/chosen": 0.4744985580444336, | |
| "rewards/margins": 5.155851554870606, | |
| "rewards/rejected": -4.681352996826172, | |
| "step": 1630 | |
| }, | |
| { | |
| "epoch": 0.8613445378151261, | |
| "grad_norm": 100.20739187851474, | |
| "kl": 0.0, | |
| "learning_rate": 2.334563433804687e-08, | |
| "logps/chosen": -319.6615634704969, | |
| "logps/rejected": -282.31458824685535, | |
| "loss": 0.3014, | |
| "rewards/chosen": 0.6534406768609278, | |
| "rewards/margins": 4.888445989606604, | |
| "rewards/rejected": -4.235005312745676, | |
| "step": 1640 | |
| }, | |
| { | |
| "epoch": 0.8665966386554622, | |
| "grad_norm": 88.81840072824282, | |
| "kl": 0.0, | |
| "learning_rate": 2.1636011579150793e-08, | |
| "logps/chosen": -295.85146484375, | |
| "logps/rejected": -290.18388671875, | |
| "loss": 0.2796, | |
| "rewards/chosen": 0.5988120079040528, | |
| "rewards/margins": 4.9494441032409675, | |
| "rewards/rejected": -4.350632095336914, | |
| "step": 1650 | |
| }, | |
| { | |
| "epoch": 0.8718487394957983, | |
| "grad_norm": 41.43708238784869, | |
| "kl": 0.0, | |
| "learning_rate": 1.998855921482906e-08, | |
| "logps/chosen": -276.827294921875, | |
| "logps/rejected": -240.9879638671875, | |
| "loss": 0.3411, | |
| "rewards/chosen": 0.4913971424102783, | |
| "rewards/margins": 4.203098344802856, | |
| "rewards/rejected": -3.711701202392578, | |
| "step": 1660 | |
| }, | |
| { | |
| "epoch": 0.8771008403361344, | |
| "grad_norm": 61.90087346557062, | |
| "kl": 0.0, | |
| "learning_rate": 1.8403725751714615e-08, | |
| "logps/chosen": -287.8762538580247, | |
| "logps/rejected": -314.7857001582278, | |
| "loss": 0.2888, | |
| "rewards/chosen": 0.6117756690508054, | |
| "rewards/margins": 5.216582303495031, | |
| "rewards/rejected": -4.604806634444225, | |
| "step": 1670 | |
| }, | |
| { | |
| "epoch": 0.8823529411764706, | |
| "grad_norm": 43.47883434595641, | |
| "kl": 0.0, | |
| "learning_rate": 1.6881942648911074e-08, | |
| "logps/chosen": -286.86601796407183, | |
| "logps/rejected": -269.488791870915, | |
| "loss": 0.3423, | |
| "rewards/chosen": 0.3007767500277765, | |
| "rewards/margins": 4.408972534852123, | |
| "rewards/rejected": -4.108195784824346, | |
| "step": 1680 | |
| }, | |
| { | |
| "epoch": 0.8876050420168067, | |
| "grad_norm": 43.77865447701488, | |
| "kl": 0.0, | |
| "learning_rate": 1.5423624200531115e-08, | |
| "logps/chosen": -300.1749822443182, | |
| "logps/rejected": -305.69072808159723, | |
| "loss": 0.3209, | |
| "rewards/chosen": 0.8302790901877664, | |
| "rewards/margins": 4.638904205476395, | |
| "rewards/rejected": -3.8086251152886286, | |
| "step": 1690 | |
| }, | |
| { | |
| "epoch": 0.8928571428571429, | |
| "grad_norm": 126.94155434263403, | |
| "kl": 0.0, | |
| "learning_rate": 1.4029167422908105e-08, | |
| "logps/chosen": -279.4980215097403, | |
| "logps/rejected": -270.3898013930723, | |
| "loss": 0.3016, | |
| "rewards/chosen": 1.0889219804243608, | |
| "rewards/margins": 5.310351644392734, | |
| "rewards/rejected": -4.221429663968373, | |
| "step": 1700 | |
| }, | |
| { | |
| "epoch": 0.898109243697479, | |
| "grad_norm": 61.01512892000863, | |
| "kl": 0.0, | |
| "learning_rate": 1.2698951946511327e-08, | |
| "logps/chosen": -302.3543386399371, | |
| "logps/rejected": -267.2125873447205, | |
| "loss": 0.3007, | |
| "rewards/chosen": 0.34754651147614485, | |
| "rewards/margins": 4.579919861631424, | |
| "rewards/rejected": -4.23237335015528, | |
| "step": 1710 | |
| }, | |
| { | |
| "epoch": 0.9033613445378151, | |
| "grad_norm": 41.136022643204946, | |
| "kl": 0.0, | |
| "learning_rate": 1.1433339912594265e-08, | |
| "logps/chosen": -308.92118158284023, | |
| "logps/rejected": -271.45762624172187, | |
| "loss": 0.3021, | |
| "rewards/chosen": 0.45501975336018397, | |
| "rewards/margins": 5.495058411653852, | |
| "rewards/rejected": -5.040038658293668, | |
| "step": 1720 | |
| }, | |
| { | |
| "epoch": 0.9086134453781513, | |
| "grad_norm": 110.83416055708793, | |
| "kl": 0.0, | |
| "learning_rate": 1.0232675874604608e-08, | |
| "logps/chosen": -312.7164007867133, | |
| "logps/rejected": -318.51147598870057, | |
| "loss": 0.2863, | |
| "rewards/chosen": 0.45250099022071677, | |
| "rewards/margins": 4.343536339576296, | |
| "rewards/rejected": -3.891035349355579, | |
| "step": 1730 | |
| }, | |
| { | |
| "epoch": 0.9138655462184874, | |
| "grad_norm": 26.122066134245365, | |
| "kl": 0.0, | |
| "learning_rate": 9.097286704381896e-09, | |
| "logps/chosen": -315.04382560483873, | |
| "logps/rejected": -272.66723484848484, | |
| "loss": 0.3122, | |
| "rewards/chosen": 0.7614468482232863, | |
| "rewards/margins": 4.353459750685407, | |
| "rewards/rejected": -3.592012902462121, | |
| "step": 1740 | |
| }, | |
| { | |
| "epoch": 0.9191176470588235, | |
| "grad_norm": 53.34556224286923, | |
| "kl": 0.0, | |
| "learning_rate": 8.02748150316937e-09, | |
| "logps/chosen": -283.3781151107595, | |
| "logps/rejected": -270.81105324074076, | |
| "loss": 0.3131, | |
| "rewards/chosen": 0.739828278746786, | |
| "rewards/margins": 4.5300670241653815, | |
| "rewards/rejected": -3.7902387454185957, | |
| "step": 1750 | |
| }, | |
| { | |
| "epoch": 0.9243697478991597, | |
| "grad_norm": 87.26559549436006, | |
| "kl": 0.0, | |
| "learning_rate": 7.023551517463089e-09, | |
| "logps/chosen": -271.55817018072287, | |
| "logps/rejected": -290.5874594155844, | |
| "loss": 0.3368, | |
| "rewards/chosen": 0.3112575117363987, | |
| "rewards/margins": 3.797579691753647, | |
| "rewards/rejected": -3.4863221800172486, | |
| "step": 1760 | |
| }, | |
| { | |
| "epoch": 0.9296218487394958, | |
| "grad_norm": 31.56919055846343, | |
| "kl": 0.0, | |
| "learning_rate": 6.085770059722634e-09, | |
| "logps/chosen": -272.63416371855345, | |
| "logps/rejected": -279.8696380046584, | |
| "loss": 0.2818, | |
| "rewards/chosen": 0.9944836118686124, | |
| "rewards/margins": 5.193192850999047, | |
| "rewards/rejected": -4.198709239130435, | |
| "step": 1770 | |
| }, | |
| { | |
| "epoch": 0.9348739495798319, | |
| "grad_norm": 37.83348753574193, | |
| "kl": 0.0, | |
| "learning_rate": 5.214392433963488e-09, | |
| "logps/chosen": -287.8191769622093, | |
| "logps/rejected": -277.8159311655405, | |
| "loss": 0.3277, | |
| "rewards/chosen": 0.5125378453454306, | |
| "rewards/margins": 3.781902073465152, | |
| "rewards/rejected": -3.2693642281197213, | |
| "step": 1780 | |
| }, | |
| { | |
| "epoch": 0.9401260504201681, | |
| "grad_norm": 45.4550208744577, | |
| "kl": 0.0, | |
| "learning_rate": 4.409655866252693e-09, | |
| "logps/chosen": -282.79800397398844, | |
| "logps/rejected": -302.68510841836735, | |
| "loss": 0.3304, | |
| "rewards/chosen": 0.5359358153591266, | |
| "rewards/margins": 4.274966925850198, | |
| "rewards/rejected": -3.7390311104910716, | |
| "step": 1790 | |
| }, | |
| { | |
| "epoch": 0.9453781512605042, | |
| "grad_norm": 47.92999315794789, | |
| "kl": 0.0, | |
| "learning_rate": 3.671779440125644e-09, | |
| "logps/chosen": -299.54941281847135, | |
| "logps/rejected": -283.9407831671779, | |
| "loss": 0.3024, | |
| "rewards/chosen": 0.9072784618207603, | |
| "rewards/margins": 4.687179322753467, | |
| "rewards/rejected": -3.779900860932707, | |
| "step": 1800 | |
| }, | |
| { | |
| "epoch": 0.9506302521008403, | |
| "grad_norm": 32.60296033727532, | |
| "kl": 0.0, | |
| "learning_rate": 3.000964036942305e-09, | |
| "logps/chosen": -278.7902070063694, | |
| "logps/rejected": -298.68158071319016, | |
| "loss": 0.2903, | |
| "rewards/chosen": 0.4591269523474821, | |
| "rewards/margins": 4.919044079336555, | |
| "rewards/rejected": -4.4599171269890725, | |
| "step": 1810 | |
| }, | |
| { | |
| "epoch": 0.9558823529411765, | |
| "grad_norm": 38.232447498754475, | |
| "kl": 0.0, | |
| "learning_rate": 2.397392281198729e-09, | |
| "logps/chosen": -302.90917338709676, | |
| "logps/rejected": -267.12088068181816, | |
| "loss": 0.301, | |
| "rewards/chosen": 0.69748047859438, | |
| "rewards/margins": 4.834925731908774, | |
| "rewards/rejected": -4.137445253314394, | |
| "step": 1820 | |
| }, | |
| { | |
| "epoch": 0.9611344537815126, | |
| "grad_norm": 22.326350506185886, | |
| "kl": 0.0, | |
| "learning_rate": 1.861228490808886e-09, | |
| "logps/chosen": -294.7616077769886, | |
| "logps/rejected": -289.75987413194446, | |
| "loss": 0.3229, | |
| "rewards/chosen": 0.6278800530867144, | |
| "rewards/margins": 4.9228116767575045, | |
| "rewards/rejected": -4.29493162367079, | |
| "step": 1830 | |
| }, | |
| { | |
| "epoch": 0.9663865546218487, | |
| "grad_norm": 40.257301749189914, | |
| "kl": 0.0, | |
| "learning_rate": 1.3926186323703903e-09, | |
| "logps/chosen": -274.1588462271341, | |
| "logps/rejected": -264.75538361378204, | |
| "loss": 0.3362, | |
| "rewards/chosen": 0.5637630369605088, | |
| "rewards/margins": 4.09795015822954, | |
| "rewards/rejected": -3.5341871212690306, | |
| "step": 1840 | |
| }, | |
| { | |
| "epoch": 0.9716386554621849, | |
| "grad_norm": 52.95763177365781, | |
| "kl": 0.0, | |
| "learning_rate": 9.916902814261774e-10, | |
| "logps/chosen": -288.03719758064517, | |
| "logps/rejected": -290.255634469697, | |
| "loss": 0.271, | |
| "rewards/chosen": 0.5408120432207661, | |
| "rewards/margins": 5.058847746937622, | |
| "rewards/rejected": -4.518035703716856, | |
| "step": 1850 | |
| }, | |
| { | |
| "epoch": 0.976890756302521, | |
| "grad_norm": 48.81020773750272, | |
| "kl": 0.0, | |
| "learning_rate": 6.585525877328968e-10, | |
| "logps/chosen": -290.9931640625, | |
| "logps/rejected": -323.75256631540697, | |
| "loss": 0.3325, | |
| "rewards/chosen": 0.10595728899981524, | |
| "rewards/margins": 4.301412795790331, | |
| "rewards/rejected": -4.195455506790516, | |
| "step": 1860 | |
| }, | |
| { | |
| "epoch": 0.9821428571428571, | |
| "grad_norm": 35.96392604697334, | |
| "kl": 0.0, | |
| "learning_rate": 3.9329624554584883e-10, | |
| "logps/chosen": -281.2289627259036, | |
| "logps/rejected": -280.72377232142856, | |
| "loss": 0.3147, | |
| "rewards/chosen": 0.46981898847832737, | |
| "rewards/margins": 4.369042811530275, | |
| "rewards/rejected": -3.899223823051948, | |
| "step": 1870 | |
| }, | |
| { | |
| "epoch": 0.9873949579831933, | |
| "grad_norm": 98.21290759884454, | |
| "kl": 0.0, | |
| "learning_rate": 1.959934689280962e-10, | |
| "logps/chosen": -292.91434151785717, | |
| "logps/rejected": -292.6137952302632, | |
| "loss": 0.3104, | |
| "rewards/chosen": 0.6014651343936012, | |
| "rewards/margins": 4.848261591784637, | |
| "rewards/rejected": -4.246796457391036, | |
| "step": 1880 | |
| }, | |
| { | |
| "epoch": 0.9926470588235294, | |
| "grad_norm": 33.92572486964675, | |
| "kl": 0.0, | |
| "learning_rate": 6.669797209069017e-11, | |
| "logps/chosen": -269.0030942766854, | |
| "logps/rejected": -273.2933263644366, | |
| "loss": 0.3204, | |
| "rewards/chosen": 0.08876853042773986, | |
| "rewards/margins": 4.26088122367708, | |
| "rewards/rejected": -4.17211269324934, | |
| "step": 1890 | |
| }, | |
| { | |
| "epoch": 0.9978991596638656, | |
| "grad_norm": 57.704679882719425, | |
| "kl": 0.0, | |
| "learning_rate": 5.444954769395771e-12, | |
| "logps/chosen": -258.240974378882, | |
| "logps/rejected": -283.904309158805, | |
| "loss": 0.3031, | |
| "rewards/chosen": 0.38151166453865004, | |
| "rewards/margins": 4.77392472590854, | |
| "rewards/rejected": -4.39241306136989, | |
| "step": 1900 | |
| }, | |
| { | |
| "epoch": 1.0, | |
| "step": 1904, | |
| "total_flos": 0.0, | |
| "train_loss": 0.3297611388589154, | |
| "train_runtime": 9235.4216, | |
| "train_samples_per_second": 6.596, | |
| "train_steps_per_second": 0.206 | |
| } | |
| ], | |
| "logging_steps": 10, | |
| "max_steps": 1904, | |
| "num_input_tokens_seen": 0, | |
| "num_train_epochs": 1, | |
| "save_steps": 500, | |
| "stateful_callbacks": { | |
| "TrainerControl": { | |
| "args": { | |
| "should_epoch_stop": false, | |
| "should_evaluate": false, | |
| "should_log": false, | |
| "should_save": true, | |
| "should_training_stop": true | |
| }, | |
| "attributes": {} | |
| } | |
| }, | |
| "total_flos": 0.0, | |
| "train_batch_size": 8, | |
| "trial_name": null, | |
| "trial_params": null | |
| } | |