|
{ |
|
"best_metric": null, |
|
"best_model_checkpoint": null, |
|
"epoch": 0.9994767137624281, |
|
"eval_steps": 100, |
|
"global_step": 955, |
|
"is_hyper_param_search": false, |
|
"is_local_process_zero": true, |
|
"is_world_process_zero": true, |
|
"log_history": [ |
|
{ |
|
"epoch": 0.0010465724751439038, |
|
"grad_norm": 127.36319300622706, |
|
"learning_rate": 5.208333333333333e-09, |
|
"logits/chosen": -2.9235777854919434, |
|
"logits/rejected": -2.7912116050720215, |
|
"logps/chosen": -380.68548583984375, |
|
"logps/rejected": -358.4158630371094, |
|
"loss": 0.6931, |
|
"rewards/accuracies": 0.0, |
|
"rewards/chosen": 0.0, |
|
"rewards/margins": 0.0, |
|
"rewards/rejected": 0.0, |
|
"step": 1 |
|
}, |
|
{ |
|
"epoch": 0.010465724751439037, |
|
"grad_norm": 107.9740860657244, |
|
"learning_rate": 5.208333333333333e-08, |
|
"logits/chosen": -2.595984935760498, |
|
"logits/rejected": -2.56972074508667, |
|
"logps/chosen": -256.59259033203125, |
|
"logps/rejected": -234.86553955078125, |
|
"loss": 0.6936, |
|
"rewards/accuracies": 0.4513888955116272, |
|
"rewards/chosen": 0.00012609982513822615, |
|
"rewards/margins": -5.737816172768362e-05, |
|
"rewards/rejected": 0.00018347776494920254, |
|
"step": 10 |
|
}, |
|
{ |
|
"epoch": 0.020931449502878074, |
|
"grad_norm": 123.40563647833262, |
|
"learning_rate": 1.0416666666666667e-07, |
|
"logits/chosen": -2.6136789321899414, |
|
"logits/rejected": -2.5759799480438232, |
|
"logps/chosen": -283.09326171875, |
|
"logps/rejected": -282.3014221191406, |
|
"loss": 0.6899, |
|
"rewards/accuracies": 0.581250011920929, |
|
"rewards/chosen": 0.0011462386464700103, |
|
"rewards/margins": 0.004408253822475672, |
|
"rewards/rejected": -0.0032620157580822706, |
|
"step": 20 |
|
}, |
|
{ |
|
"epoch": 0.03139717425431711, |
|
"grad_norm": 121.3818362170214, |
|
"learning_rate": 1.5624999999999999e-07, |
|
"logits/chosen": -2.6919732093811035, |
|
"logits/rejected": -2.667656660079956, |
|
"logps/chosen": -270.1833801269531, |
|
"logps/rejected": -276.7445068359375, |
|
"loss": 0.683, |
|
"rewards/accuracies": 0.5874999761581421, |
|
"rewards/chosen": 0.01242474652826786, |
|
"rewards/margins": 0.02042328752577305, |
|
"rewards/rejected": -0.007998539134860039, |
|
"step": 30 |
|
}, |
|
{ |
|
"epoch": 0.04186289900575615, |
|
"grad_norm": 115.66860622860622, |
|
"learning_rate": 2.0833333333333333e-07, |
|
"logits/chosen": -2.665289878845215, |
|
"logits/rejected": -2.5902962684631348, |
|
"logps/chosen": -290.4902648925781, |
|
"logps/rejected": -282.1328125, |
|
"loss": 0.6686, |
|
"rewards/accuracies": 0.643750011920929, |
|
"rewards/chosen": 0.04880904406309128, |
|
"rewards/margins": 0.05465905740857124, |
|
"rewards/rejected": -0.005850008223205805, |
|
"step": 40 |
|
}, |
|
{ |
|
"epoch": 0.052328623757195186, |
|
"grad_norm": 113.05355403051037, |
|
"learning_rate": 2.604166666666667e-07, |
|
"logits/chosen": -2.672785997390747, |
|
"logits/rejected": -2.589231014251709, |
|
"logps/chosen": -266.2040100097656, |
|
"logps/rejected": -236.48739624023438, |
|
"loss": 0.641, |
|
"rewards/accuracies": 0.65625, |
|
"rewards/chosen": 0.14585763216018677, |
|
"rewards/margins": 0.13357332348823547, |
|
"rewards/rejected": 0.012284321710467339, |
|
"step": 50 |
|
}, |
|
{ |
|
"epoch": 0.06279434850863422, |
|
"grad_norm": 106.6001336411059, |
|
"learning_rate": 3.1249999999999997e-07, |
|
"logits/chosen": -2.6317365169525146, |
|
"logits/rejected": -2.596348285675049, |
|
"logps/chosen": -300.1541442871094, |
|
"logps/rejected": -274.40191650390625, |
|
"loss": 0.6134, |
|
"rewards/accuracies": 0.7124999761581421, |
|
"rewards/chosen": 0.3813043236732483, |
|
"rewards/margins": 0.24904949963092804, |
|
"rewards/rejected": 0.13225486874580383, |
|
"step": 60 |
|
}, |
|
{ |
|
"epoch": 0.07326007326007326, |
|
"grad_norm": 105.44232045239558, |
|
"learning_rate": 3.645833333333333e-07, |
|
"logits/chosen": -2.5406157970428467, |
|
"logits/rejected": -2.5337507724761963, |
|
"logps/chosen": -256.82513427734375, |
|
"logps/rejected": -259.8224792480469, |
|
"loss": 0.5958, |
|
"rewards/accuracies": 0.71875, |
|
"rewards/chosen": 0.2853667438030243, |
|
"rewards/margins": 0.44680237770080566, |
|
"rewards/rejected": -0.16143563389778137, |
|
"step": 70 |
|
}, |
|
{ |
|
"epoch": 0.0837257980115123, |
|
"grad_norm": 118.11259244631633, |
|
"learning_rate": 4.1666666666666667e-07, |
|
"logits/chosen": -2.583888292312622, |
|
"logits/rejected": -2.516160249710083, |
|
"logps/chosen": -271.23541259765625, |
|
"logps/rejected": -252.84814453125, |
|
"loss": 0.5843, |
|
"rewards/accuracies": 0.7562500238418579, |
|
"rewards/chosen": 0.4816661775112152, |
|
"rewards/margins": 0.6344618797302246, |
|
"rewards/rejected": -0.15279565751552582, |
|
"step": 80 |
|
}, |
|
{ |
|
"epoch": 0.09419152276295134, |
|
"grad_norm": 90.59677124687741, |
|
"learning_rate": 4.6874999999999996e-07, |
|
"logits/chosen": -2.549140691757202, |
|
"logits/rejected": -2.518716335296631, |
|
"logps/chosen": -249.97048950195312, |
|
"logps/rejected": -263.16864013671875, |
|
"loss": 0.5806, |
|
"rewards/accuracies": 0.6937500238418579, |
|
"rewards/chosen": 0.5636419057846069, |
|
"rewards/margins": 0.4112882614135742, |
|
"rewards/rejected": 0.1523536890745163, |
|
"step": 90 |
|
}, |
|
{ |
|
"epoch": 0.10465724751439037, |
|
"grad_norm": 89.93368824706333, |
|
"learning_rate": 4.999732492681437e-07, |
|
"logits/chosen": -2.5437731742858887, |
|
"logits/rejected": -2.4748952388763428, |
|
"logps/chosen": -290.6165771484375, |
|
"logps/rejected": -277.58087158203125, |
|
"loss": 0.5446, |
|
"rewards/accuracies": 0.7124999761581421, |
|
"rewards/chosen": 0.9238502383232117, |
|
"rewards/margins": 0.6838744878768921, |
|
"rewards/rejected": 0.23997588455677032, |
|
"step": 100 |
|
}, |
|
{ |
|
"epoch": 0.10465724751439037, |
|
"eval_logits/chosen": -2.5742745399475098, |
|
"eval_logits/rejected": -2.516094923019409, |
|
"eval_logps/chosen": -271.84344482421875, |
|
"eval_logps/rejected": -256.6897888183594, |
|
"eval_loss": 0.5752763748168945, |
|
"eval_rewards/accuracies": 0.7242063283920288, |
|
"eval_rewards/chosen": 1.0110565423965454, |
|
"eval_rewards/margins": 0.6581434011459351, |
|
"eval_rewards/rejected": 0.35291311144828796, |
|
"eval_runtime": 206.982, |
|
"eval_samples_per_second": 9.663, |
|
"eval_steps_per_second": 0.304, |
|
"step": 100 |
|
}, |
|
{ |
|
"epoch": 0.1151229722658294, |
|
"grad_norm": 91.21630112817637, |
|
"learning_rate": 4.996723692767926e-07, |
|
"logits/chosen": -2.5750699043273926, |
|
"logits/rejected": -2.528003215789795, |
|
"logps/chosen": -234.3852081298828, |
|
"logps/rejected": -236.4423370361328, |
|
"loss": 0.5648, |
|
"rewards/accuracies": 0.706250011920929, |
|
"rewards/chosen": 0.705768346786499, |
|
"rewards/margins": 0.6249028444290161, |
|
"rewards/rejected": 0.08086539804935455, |
|
"step": 110 |
|
}, |
|
{ |
|
"epoch": 0.12558869701726844, |
|
"grad_norm": 106.85996872671201, |
|
"learning_rate": 4.990375746213598e-07, |
|
"logits/chosen": -2.524847984313965, |
|
"logits/rejected": -2.4740397930145264, |
|
"logps/chosen": -258.1869201660156, |
|
"logps/rejected": -292.03643798828125, |
|
"loss": 0.5378, |
|
"rewards/accuracies": 0.731249988079071, |
|
"rewards/chosen": 0.43742185831069946, |
|
"rewards/margins": 0.7921646237373352, |
|
"rewards/rejected": -0.3547428548336029, |
|
"step": 120 |
|
}, |
|
{ |
|
"epoch": 0.1360544217687075, |
|
"grad_norm": 113.00637141782764, |
|
"learning_rate": 4.980697142834314e-07, |
|
"logits/chosen": -2.520475387573242, |
|
"logits/rejected": -2.4845001697540283, |
|
"logps/chosen": -278.5224304199219, |
|
"logps/rejected": -261.1920166015625, |
|
"loss": 0.5472, |
|
"rewards/accuracies": 0.7562500238418579, |
|
"rewards/chosen": 0.26921743154525757, |
|
"rewards/margins": 0.9498673677444458, |
|
"rewards/rejected": -0.6806498765945435, |
|
"step": 130 |
|
}, |
|
{ |
|
"epoch": 0.14652014652014653, |
|
"grad_norm": 98.25615957326066, |
|
"learning_rate": 4.967700826904229e-07, |
|
"logits/chosen": -2.4557089805603027, |
|
"logits/rejected": -2.4235308170318604, |
|
"logps/chosen": -284.69012451171875, |
|
"logps/rejected": -292.3227233886719, |
|
"loss": 0.5192, |
|
"rewards/accuracies": 0.800000011920929, |
|
"rewards/chosen": 0.6424055099487305, |
|
"rewards/margins": 0.9809519648551941, |
|
"rewards/rejected": -0.3385465741157532, |
|
"step": 140 |
|
}, |
|
{ |
|
"epoch": 0.15698587127158556, |
|
"grad_norm": 99.56093992482657, |
|
"learning_rate": 4.951404179843962e-07, |
|
"logits/chosen": -2.601367473602295, |
|
"logits/rejected": -2.5492353439331055, |
|
"logps/chosen": -286.1108093261719, |
|
"logps/rejected": -252.67074584960938, |
|
"loss": 0.5511, |
|
"rewards/accuracies": 0.699999988079071, |
|
"rewards/chosen": 0.17216625809669495, |
|
"rewards/margins": 0.7409141659736633, |
|
"rewards/rejected": -0.568747878074646, |
|
"step": 150 |
|
}, |
|
{ |
|
"epoch": 0.1674515960230246, |
|
"grad_norm": 115.6693954933169, |
|
"learning_rate": 4.931828996974498e-07, |
|
"logits/chosen": -2.568312883377075, |
|
"logits/rejected": -2.4805312156677246, |
|
"logps/chosen": -291.2640380859375, |
|
"logps/rejected": -259.77508544921875, |
|
"loss": 0.5535, |
|
"rewards/accuracies": 0.7437499761581421, |
|
"rewards/chosen": 0.4559265077114105, |
|
"rewards/margins": 0.8324508666992188, |
|
"rewards/rejected": -0.3765243887901306, |
|
"step": 160 |
|
}, |
|
{ |
|
"epoch": 0.17791732077446362, |
|
"grad_norm": 108.75023514338689, |
|
"learning_rate": 4.909001458367866e-07, |
|
"logits/chosen": -2.556863784790039, |
|
"logits/rejected": -2.4760687351226807, |
|
"logps/chosen": -274.7050476074219, |
|
"logps/rejected": -246.76708984375, |
|
"loss": 0.5534, |
|
"rewards/accuracies": 0.737500011920929, |
|
"rewards/chosen": 0.37477999925613403, |
|
"rewards/margins": 0.9715734720230103, |
|
"rewards/rejected": -0.5967934727668762, |
|
"step": 170 |
|
}, |
|
{ |
|
"epoch": 0.18838304552590268, |
|
"grad_norm": 99.91541524541664, |
|
"learning_rate": 4.882952093833627e-07, |
|
"logits/chosen": -2.543196201324463, |
|
"logits/rejected": -2.501505136489868, |
|
"logps/chosen": -242.51748657226562, |
|
"logps/rejected": -259.04241943359375, |
|
"loss": 0.5136, |
|
"rewards/accuracies": 0.7124999761581421, |
|
"rewards/chosen": -0.2591323256492615, |
|
"rewards/margins": 0.9666780233383179, |
|
"rewards/rejected": -1.2258104085922241, |
|
"step": 180 |
|
}, |
|
{ |
|
"epoch": 0.1988487702773417, |
|
"grad_norm": 115.88419375588242, |
|
"learning_rate": 4.853715742087946e-07, |
|
"logits/chosen": -2.4405717849731445, |
|
"logits/rejected": -2.4257616996765137, |
|
"logps/chosen": -240.6494140625, |
|
"logps/rejected": -264.52215576171875, |
|
"loss": 0.5006, |
|
"rewards/accuracies": 0.7875000238418579, |
|
"rewards/chosen": -0.46298712491989136, |
|
"rewards/margins": 0.9989321827888489, |
|
"rewards/rejected": -1.4619193077087402, |
|
"step": 190 |
|
}, |
|
{ |
|
"epoch": 0.20931449502878074, |
|
"grad_norm": 105.94881559115358, |
|
"learning_rate": 4.821331504159906e-07, |
|
"logits/chosen": -2.5927906036376953, |
|
"logits/rejected": -2.517674684524536, |
|
"logps/chosen": -309.7959289550781, |
|
"logps/rejected": -265.2268371582031, |
|
"loss": 0.5475, |
|
"rewards/accuracies": 0.731249988079071, |
|
"rewards/chosen": 0.08473672717809677, |
|
"rewards/margins": 0.8970876932144165, |
|
"rewards/rejected": -0.812350869178772, |
|
"step": 200 |
|
}, |
|
{ |
|
"epoch": 0.20931449502878074, |
|
"eval_logits/chosen": -2.5923333168029785, |
|
"eval_logits/rejected": -2.537993907928467, |
|
"eval_logps/chosen": -277.6067810058594, |
|
"eval_logps/rejected": -265.0432434082031, |
|
"eval_loss": 0.5464460849761963, |
|
"eval_rewards/accuracies": 0.7638888955116272, |
|
"eval_rewards/chosen": 0.4347245991230011, |
|
"eval_rewards/margins": 0.9171593189239502, |
|
"eval_rewards/rejected": -0.48243483901023865, |
|
"eval_runtime": 206.9145, |
|
"eval_samples_per_second": 9.666, |
|
"eval_steps_per_second": 0.304, |
|
"step": 200 |
|
}, |
|
{ |
|
"epoch": 0.21978021978021978, |
|
"grad_norm": 118.18724049066705, |
|
"learning_rate": 4.785842691097342e-07, |
|
"logits/chosen": -2.5883002281188965, |
|
"logits/rejected": -2.524261951446533, |
|
"logps/chosen": -278.22625732421875, |
|
"logps/rejected": -275.1640625, |
|
"loss": 0.5643, |
|
"rewards/accuracies": 0.668749988079071, |
|
"rewards/chosen": 0.45807909965515137, |
|
"rewards/margins": 0.7460596561431885, |
|
"rewards/rejected": -0.2879805266857147, |
|
"step": 210 |
|
}, |
|
{ |
|
"epoch": 0.2302459445316588, |
|
"grad_norm": 156.31813233608617, |
|
"learning_rate": 4.7472967660421603e-07, |
|
"logits/chosen": -2.5972952842712402, |
|
"logits/rejected": -2.5259041786193848, |
|
"logps/chosen": -284.1155700683594, |
|
"logps/rejected": -242.37039184570312, |
|
"loss": 0.5191, |
|
"rewards/accuracies": 0.78125, |
|
"rewards/chosen": 0.30692246556282043, |
|
"rewards/margins": 0.9284420013427734, |
|
"rewards/rejected": -0.6215195655822754, |
|
"step": 220 |
|
}, |
|
{ |
|
"epoch": 0.24071166928309787, |
|
"grad_norm": 103.00603061075961, |
|
"learning_rate": 4.705745280752585e-07, |
|
"logits/chosen": -2.527318000793457, |
|
"logits/rejected": -2.505685329437256, |
|
"logps/chosen": -287.8115539550781, |
|
"logps/rejected": -275.6258850097656, |
|
"loss": 0.5225, |
|
"rewards/accuracies": 0.7124999761581421, |
|
"rewards/chosen": 0.09247669577598572, |
|
"rewards/margins": 0.7993501424789429, |
|
"rewards/rejected": -0.7068735361099243, |
|
"step": 230 |
|
}, |
|
{ |
|
"epoch": 0.25117739403453687, |
|
"grad_norm": 187.82863477728128, |
|
"learning_rate": 4.6612438066572555e-07, |
|
"logits/chosen": -2.536621570587158, |
|
"logits/rejected": -2.525778293609619, |
|
"logps/chosen": -265.84332275390625, |
|
"logps/rejected": -262.84210205078125, |
|
"loss": 0.517, |
|
"rewards/accuracies": 0.78125, |
|
"rewards/chosen": 0.17310741543769836, |
|
"rewards/margins": 1.0395452976226807, |
|
"rewards/rejected": -0.8664379119873047, |
|
"step": 240 |
|
}, |
|
{ |
|
"epoch": 0.2616431187859759, |
|
"grad_norm": 98.47328118938334, |
|
"learning_rate": 4.6138518605333664e-07, |
|
"logits/chosen": -2.5257980823516846, |
|
"logits/rejected": -2.50608229637146, |
|
"logps/chosen": -275.45831298828125, |
|
"logps/rejected": -265.85247802734375, |
|
"loss": 0.4916, |
|
"rewards/accuracies": 0.71875, |
|
"rewards/chosen": 0.23112091422080994, |
|
"rewards/margins": 1.057886004447937, |
|
"rewards/rejected": -0.8267651796340942, |
|
"step": 250 |
|
}, |
|
{ |
|
"epoch": 0.272108843537415, |
|
"grad_norm": 132.81269847647957, |
|
"learning_rate": 4.5636328249082514e-07, |
|
"logits/chosen": -2.6113829612731934, |
|
"logits/rejected": -2.5672502517700195, |
|
"logps/chosen": -288.1708068847656, |
|
"logps/rejected": -290.2517395019531, |
|
"loss": 0.5332, |
|
"rewards/accuracies": 0.6937500238418579, |
|
"rewards/chosen": 0.08042006194591522, |
|
"rewards/margins": 0.8779838681221008, |
|
"rewards/rejected": -0.7975638508796692, |
|
"step": 260 |
|
}, |
|
{ |
|
"epoch": 0.282574568288854, |
|
"grad_norm": 109.69739978094053, |
|
"learning_rate": 4.510653863290871e-07, |
|
"logits/chosen": -2.5312352180480957, |
|
"logits/rejected": -2.514791965484619, |
|
"logps/chosen": -294.85821533203125, |
|
"logps/rejected": -259.7037658691406, |
|
"loss": 0.4994, |
|
"rewards/accuracies": 0.7562500238418579, |
|
"rewards/chosen": 0.4837077260017395, |
|
"rewards/margins": 1.097794771194458, |
|
"rewards/rejected": -0.614086925983429, |
|
"step": 270 |
|
}, |
|
{ |
|
"epoch": 0.29304029304029305, |
|
"grad_norm": 112.60908808433207, |
|
"learning_rate": 4.4549858303465737e-07, |
|
"logits/chosen": -2.577178478240967, |
|
"logits/rejected": -2.527773380279541, |
|
"logps/chosen": -293.6539001464844, |
|
"logps/rejected": -294.2825927734375, |
|
"loss": 0.5343, |
|
"rewards/accuracies": 0.7562500238418579, |
|
"rewards/chosen": 0.25686317682266235, |
|
"rewards/margins": 1.3584675788879395, |
|
"rewards/rejected": -1.1016044616699219, |
|
"step": 280 |
|
}, |
|
{ |
|
"epoch": 0.3035060177917321, |
|
"grad_norm": 136.22152391063145, |
|
"learning_rate": 4.396703177135261e-07, |
|
"logits/chosen": -2.5472209453582764, |
|
"logits/rejected": -2.470147132873535, |
|
"logps/chosen": -272.1517028808594, |
|
"logps/rejected": -246.69384765625, |
|
"loss": 0.5047, |
|
"rewards/accuracies": 0.7562500238418579, |
|
"rewards/chosen": 0.06049398332834244, |
|
"rewards/margins": 1.0268157720565796, |
|
"rewards/rejected": -0.9663218259811401, |
|
"step": 290 |
|
}, |
|
{ |
|
"epoch": 0.3139717425431711, |
|
"grad_norm": 103.27642265277515, |
|
"learning_rate": 4.335883851539693e-07, |
|
"logits/chosen": -2.571641206741333, |
|
"logits/rejected": -2.5657002925872803, |
|
"logps/chosen": -269.2393493652344, |
|
"logps/rejected": -277.0874938964844, |
|
"loss": 0.5359, |
|
"rewards/accuracies": 0.71875, |
|
"rewards/chosen": -0.024735305458307266, |
|
"rewards/margins": 0.9094675183296204, |
|
"rewards/rejected": -0.9342027902603149, |
|
"step": 300 |
|
}, |
|
{ |
|
"epoch": 0.3139717425431711, |
|
"eval_logits/chosen": -2.559617757797241, |
|
"eval_logits/rejected": -2.5065526962280273, |
|
"eval_logps/chosen": -281.25714111328125, |
|
"eval_logps/rejected": -270.3889465332031, |
|
"eval_loss": 0.5473023653030396, |
|
"eval_rewards/accuracies": 0.7579365372657776, |
|
"eval_rewards/chosen": 0.0696859061717987, |
|
"eval_rewards/margins": 1.0866899490356445, |
|
"eval_rewards/rejected": -1.017004132270813, |
|
"eval_runtime": 206.5677, |
|
"eval_samples_per_second": 9.682, |
|
"eval_steps_per_second": 0.305, |
|
"step": 300 |
|
}, |
|
{ |
|
"epoch": 0.32443746729461015, |
|
"grad_norm": 113.48472631776741, |
|
"learning_rate": 4.272609194017105e-07, |
|
"logits/chosen": -2.5980334281921387, |
|
"logits/rejected": -2.5018410682678223, |
|
"logps/chosen": -284.5079650878906, |
|
"logps/rejected": -245.64804077148438, |
|
"loss": 0.5341, |
|
"rewards/accuracies": 0.7437499761581421, |
|
"rewards/chosen": -0.07721372693777084, |
|
"rewards/margins": 1.208656907081604, |
|
"rewards/rejected": -1.2858705520629883, |
|
"step": 310 |
|
}, |
|
{ |
|
"epoch": 0.3349031920460492, |
|
"grad_norm": 94.90226304989493, |
|
"learning_rate": 4.2069638288135547e-07, |
|
"logits/chosen": -2.53539776802063, |
|
"logits/rejected": -2.496711254119873, |
|
"logps/chosen": -254.16464233398438, |
|
"logps/rejected": -263.13336181640625, |
|
"loss": 0.5831, |
|
"rewards/accuracies": 0.7875000238418579, |
|
"rewards/chosen": -0.1118149608373642, |
|
"rewards/margins": 1.1433535814285278, |
|
"rewards/rejected": -1.2551685571670532, |
|
"step": 320 |
|
}, |
|
{ |
|
"epoch": 0.3453689167974882, |
|
"grad_norm": 111.02501519646566, |
|
"learning_rate": 4.139035550786494e-07, |
|
"logits/chosen": -2.503291606903076, |
|
"logits/rejected": -2.466763973236084, |
|
"logps/chosen": -260.0853271484375, |
|
"logps/rejected": -246.05831909179688, |
|
"loss": 0.5226, |
|
"rewards/accuracies": 0.737500011920929, |
|
"rewards/chosen": -0.3735058307647705, |
|
"rewards/margins": 1.021803379058838, |
|
"rewards/rejected": -1.395309329032898, |
|
"step": 330 |
|
}, |
|
{ |
|
"epoch": 0.35583464154892724, |
|
"grad_norm": 120.37840185839579, |
|
"learning_rate": 4.0689152079869306e-07, |
|
"logits/chosen": -2.419663667678833, |
|
"logits/rejected": -2.3767175674438477, |
|
"logps/chosen": -270.08642578125, |
|
"logps/rejected": -246.24227905273438, |
|
"loss": 0.6048, |
|
"rewards/accuracies": 0.6875, |
|
"rewards/chosen": -0.054160237312316895, |
|
"rewards/margins": 0.966072678565979, |
|
"rewards/rejected": -1.0202327966690063, |
|
"step": 340 |
|
}, |
|
{ |
|
"epoch": 0.3663003663003663, |
|
"grad_norm": 135.62587519571315, |
|
"learning_rate": 3.99669658015821e-07, |
|
"logits/chosen": -2.512122631072998, |
|
"logits/rejected": -2.480046510696411, |
|
"logps/chosen": -289.42449951171875, |
|
"logps/rejected": -262.649169921875, |
|
"loss": 0.537, |
|
"rewards/accuracies": 0.675000011920929, |
|
"rewards/chosen": 0.17944186925888062, |
|
"rewards/margins": 0.8316653370857239, |
|
"rewards/rejected": -0.6522234678268433, |
|
"step": 350 |
|
}, |
|
{ |
|
"epoch": 0.37676609105180536, |
|
"grad_norm": 95.8864031595669, |
|
"learning_rate": 3.92247625331392e-07, |
|
"logits/chosen": -2.4578399658203125, |
|
"logits/rejected": -2.4470133781433105, |
|
"logps/chosen": -281.417236328125, |
|
"logps/rejected": -271.0404357910156, |
|
"loss": 0.4816, |
|
"rewards/accuracies": 0.7749999761581421, |
|
"rewards/chosen": 0.4489572048187256, |
|
"rewards/margins": 1.2753773927688599, |
|
"rewards/rejected": -0.8264200091362, |
|
"step": 360 |
|
}, |
|
{ |
|
"epoch": 0.3872318158032444, |
|
"grad_norm": 100.50656645709512, |
|
"learning_rate": 3.846353490562664e-07, |
|
"logits/chosen": -2.5093467235565186, |
|
"logits/rejected": -2.4688096046447754, |
|
"logps/chosen": -260.3323669433594, |
|
"logps/rejected": -242.1940155029297, |
|
"loss": 0.5303, |
|
"rewards/accuracies": 0.7250000238418579, |
|
"rewards/chosen": 0.2971992492675781, |
|
"rewards/margins": 0.9509322047233582, |
|
"rewards/rejected": -0.65373295545578, |
|
"step": 370 |
|
}, |
|
{ |
|
"epoch": 0.3976975405546834, |
|
"grad_norm": 106.58256559416274, |
|
"learning_rate": 3.768430099352445e-07, |
|
"logits/chosen": -2.487880229949951, |
|
"logits/rejected": -2.454012870788574, |
|
"logps/chosen": -255.39425659179688, |
|
"logps/rejected": -257.49078369140625, |
|
"loss": 0.5166, |
|
"rewards/accuracies": 0.7437499761581421, |
|
"rewards/chosen": -0.07468878477811813, |
|
"rewards/margins": 1.0435364246368408, |
|
"rewards/rejected": -1.118225336074829, |
|
"step": 380 |
|
}, |
|
{ |
|
"epoch": 0.40816326530612246, |
|
"grad_norm": 101.08281656389065, |
|
"learning_rate": 3.6888102953122304e-07, |
|
"logits/chosen": -2.5439352989196777, |
|
"logits/rejected": -2.506777286529541, |
|
"logps/chosen": -287.0313720703125, |
|
"logps/rejected": -284.038818359375, |
|
"loss": 0.5404, |
|
"rewards/accuracies": 0.668749988079071, |
|
"rewards/chosen": -0.5431143045425415, |
|
"rewards/margins": 1.0249578952789307, |
|
"rewards/rejected": -1.5680720806121826, |
|
"step": 390 |
|
}, |
|
{ |
|
"epoch": 0.4186289900575615, |
|
"grad_norm": 295.883680569105, |
|
"learning_rate": 3.607600562872785e-07, |
|
"logits/chosen": -2.568805456161499, |
|
"logits/rejected": -2.5805678367614746, |
|
"logps/chosen": -274.0014953613281, |
|
"logps/rejected": -281.6232604980469, |
|
"loss": 0.5228, |
|
"rewards/accuracies": 0.7124999761581421, |
|
"rewards/chosen": -0.4060361385345459, |
|
"rewards/margins": 0.9878374338150024, |
|
"rewards/rejected": -1.3938738107681274, |
|
"step": 400 |
|
}, |
|
{ |
|
"epoch": 0.4186289900575615, |
|
"eval_logits/chosen": -2.6470746994018555, |
|
"eval_logits/rejected": -2.59334397315979, |
|
"eval_logps/chosen": -284.2651672363281, |
|
"eval_logps/rejected": -273.2837219238281, |
|
"eval_loss": 0.5321290493011475, |
|
"eval_rewards/accuracies": 0.7539682388305664, |
|
"eval_rewards/chosen": -0.23111547529697418, |
|
"eval_rewards/margins": 1.0753650665283203, |
|
"eval_rewards/rejected": -1.3064805269241333, |
|
"eval_runtime": 206.5734, |
|
"eval_samples_per_second": 9.682, |
|
"eval_steps_per_second": 0.305, |
|
"step": 400 |
|
}, |
|
{ |
|
"epoch": 0.4290947148090005, |
|
"grad_norm": 103.72761100708047, |
|
"learning_rate": 3.5249095128531856e-07, |
|
"logits/chosen": -2.588397741317749, |
|
"logits/rejected": -2.5449459552764893, |
|
"logps/chosen": -274.9619140625, |
|
"logps/rejected": -259.3703308105469, |
|
"loss": 0.5364, |
|
"rewards/accuracies": 0.7124999761581421, |
|
"rewards/chosen": -0.32226118445396423, |
|
"rewards/margins": 0.9323698878288269, |
|
"rewards/rejected": -1.2546310424804688, |
|
"step": 410 |
|
}, |
|
{ |
|
"epoch": 0.43956043956043955, |
|
"grad_norm": 143.18448896096157, |
|
"learning_rate": 3.4408477372034736e-07, |
|
"logits/chosen": -2.6011955738067627, |
|
"logits/rejected": -2.511427402496338, |
|
"logps/chosen": -281.31475830078125, |
|
"logps/rejected": -264.31573486328125, |
|
"loss": 0.5605, |
|
"rewards/accuracies": 0.7875000238418579, |
|
"rewards/chosen": 0.08763083815574646, |
|
"rewards/margins": 1.3145806789398193, |
|
"rewards/rejected": -1.226949691772461, |
|
"step": 420 |
|
}, |
|
{ |
|
"epoch": 0.4500261643118786, |
|
"grad_norm": 105.39381768622819, |
|
"learning_rate": 3.3555276610977276e-07, |
|
"logits/chosen": -2.582556962966919, |
|
"logits/rejected": -2.563215732574463, |
|
"logps/chosen": -251.7899932861328, |
|
"logps/rejected": -261.0627746582031, |
|
"loss": 0.5028, |
|
"rewards/accuracies": 0.768750011920929, |
|
"rewards/chosen": 0.19878244400024414, |
|
"rewards/margins": 1.2880423069000244, |
|
"rewards/rejected": -1.0892596244812012, |
|
"step": 430 |
|
}, |
|
{ |
|
"epoch": 0.4604918890633176, |
|
"grad_norm": 97.95625671060284, |
|
"learning_rate": 3.269063392575352e-07, |
|
"logits/chosen": -2.5603010654449463, |
|
"logits/rejected": -2.5315120220184326, |
|
"logps/chosen": -294.2579040527344, |
|
"logps/rejected": -261.4208679199219, |
|
"loss": 0.5781, |
|
"rewards/accuracies": 0.706250011920929, |
|
"rewards/chosen": 0.100133016705513, |
|
"rewards/margins": 0.6995735168457031, |
|
"rewards/rejected": -0.5994404554367065, |
|
"step": 440 |
|
}, |
|
{ |
|
"epoch": 0.47095761381475665, |
|
"grad_norm": 115.32814881542778, |
|
"learning_rate": 3.1815705699316964e-07, |
|
"logits/chosen": -2.5920796394348145, |
|
"logits/rejected": -2.5575454235076904, |
|
"logps/chosen": -281.134033203125, |
|
"logps/rejected": -289.2639465332031, |
|
"loss": 0.5217, |
|
"rewards/accuracies": 0.762499988079071, |
|
"rewards/chosen": 0.12630310654640198, |
|
"rewards/margins": 1.2271454334259033, |
|
"rewards/rejected": -1.1008423566818237, |
|
"step": 450 |
|
}, |
|
{ |
|
"epoch": 0.48142333856619574, |
|
"grad_norm": 118.44955170701016, |
|
"learning_rate": 3.0931662070620794e-07, |
|
"logits/chosen": -2.5641932487487793, |
|
"logits/rejected": -2.5097055435180664, |
|
"logps/chosen": -257.33160400390625, |
|
"logps/rejected": -255.80221557617188, |
|
"loss": 0.5178, |
|
"rewards/accuracies": 0.7749999761581421, |
|
"rewards/chosen": 0.01436941884458065, |
|
"rewards/margins": 1.0947532653808594, |
|
"rewards/rejected": -1.0803838968276978, |
|
"step": 460 |
|
}, |
|
{ |
|
"epoch": 0.49188906331763477, |
|
"grad_norm": 95.30482890686108, |
|
"learning_rate": 3.003968536966078e-07, |
|
"logits/chosen": -2.6018574237823486, |
|
"logits/rejected": -2.559516429901123, |
|
"logps/chosen": -282.80145263671875, |
|
"logps/rejected": -259.4977111816406, |
|
"loss": 0.4768, |
|
"rewards/accuracies": 0.6937500238418579, |
|
"rewards/chosen": 0.09861959517002106, |
|
"rewards/margins": 1.0542025566101074, |
|
"rewards/rejected": -0.9555829763412476, |
|
"step": 470 |
|
}, |
|
{ |
|
"epoch": 0.5023547880690737, |
|
"grad_norm": 105.76775426891116, |
|
"learning_rate": 2.9140968536213693e-07, |
|
"logits/chosen": -2.546515464782715, |
|
"logits/rejected": -2.4800972938537598, |
|
"logps/chosen": -268.1258850097656, |
|
"logps/rejected": -265.9749450683594, |
|
"loss": 0.4923, |
|
"rewards/accuracies": 0.800000011920929, |
|
"rewards/chosen": -0.09133951365947723, |
|
"rewards/margins": 1.3749868869781494, |
|
"rewards/rejected": -1.466326355934143, |
|
"step": 480 |
|
}, |
|
{ |
|
"epoch": 0.5128205128205128, |
|
"grad_norm": 99.04770574478776, |
|
"learning_rate": 2.823671352438608e-07, |
|
"logits/chosen": -2.5936622619628906, |
|
"logits/rejected": -2.4636528491973877, |
|
"logps/chosen": -310.77911376953125, |
|
"logps/rejected": -268.76055908203125, |
|
"loss": 0.5039, |
|
"rewards/accuracies": 0.800000011920929, |
|
"rewards/chosen": 0.11457942426204681, |
|
"rewards/margins": 1.352496862411499, |
|
"rewards/rejected": -1.237917423248291, |
|
"step": 490 |
|
}, |
|
{ |
|
"epoch": 0.5232862375719518, |
|
"grad_norm": 89.42678222589365, |
|
"learning_rate": 2.73281296951072e-07, |
|
"logits/chosen": -2.482358694076538, |
|
"logits/rejected": -2.4588165283203125, |
|
"logps/chosen": -258.74224853515625, |
|
"logps/rejected": -238.3992156982422, |
|
"loss": 0.5217, |
|
"rewards/accuracies": 0.762499988079071, |
|
"rewards/chosen": -0.006418359465897083, |
|
"rewards/margins": 1.0917348861694336, |
|
"rewards/rejected": -1.0981531143188477, |
|
"step": 500 |
|
}, |
|
{ |
|
"epoch": 0.5232862375719518, |
|
"eval_logits/chosen": -2.5772807598114014, |
|
"eval_logits/rejected": -2.5195107460021973, |
|
"eval_logps/chosen": -281.8111267089844, |
|
"eval_logps/rejected": -272.2919006347656, |
|
"eval_loss": 0.526000440120697, |
|
"eval_rewards/accuracies": 0.7876983880996704, |
|
"eval_rewards/chosen": 0.014284107834100723, |
|
"eval_rewards/margins": 1.2215818166732788, |
|
"eval_rewards/rejected": -1.2072975635528564, |
|
"eval_runtime": 206.5403, |
|
"eval_samples_per_second": 9.683, |
|
"eval_steps_per_second": 0.305, |
|
"step": 500 |
|
}, |
|
{ |
|
"epoch": 0.533751962323391, |
|
"grad_norm": 95.80547597223038, |
|
"learning_rate": 2.641643219871597e-07, |
|
"logits/chosen": -2.5779619216918945, |
|
"logits/rejected": -2.567397356033325, |
|
"logps/chosen": -281.93707275390625, |
|
"logps/rejected": -275.4586486816406, |
|
"loss": 0.4822, |
|
"rewards/accuracies": 0.7875000238418579, |
|
"rewards/chosen": 0.19970571994781494, |
|
"rewards/margins": 1.36869215965271, |
|
"rewards/rejected": -1.1689863204956055, |
|
"step": 510 |
|
}, |
|
{ |
|
"epoch": 0.54421768707483, |
|
"grad_norm": 88.16712559993171, |
|
"learning_rate": 2.550284034980507e-07, |
|
"logits/chosen": -2.5944199562072754, |
|
"logits/rejected": -2.571303129196167, |
|
"logps/chosen": -274.64373779296875, |
|
"logps/rejected": -263.65179443359375, |
|
"loss": 0.5305, |
|
"rewards/accuracies": 0.7562500238418579, |
|
"rewards/chosen": -0.2565365731716156, |
|
"rewards/margins": 1.113057255744934, |
|
"rewards/rejected": -1.369593858718872, |
|
"step": 520 |
|
}, |
|
{ |
|
"epoch": 0.554683411826269, |
|
"grad_norm": 130.8715598080221, |
|
"learning_rate": 2.4588575996495794e-07, |
|
"logits/chosen": -2.616055965423584, |
|
"logits/rejected": -2.5736825466156006, |
|
"logps/chosen": -271.31500244140625, |
|
"logps/rejected": -268.7001037597656, |
|
"loss": 0.5681, |
|
"rewards/accuracies": 0.706250011920929, |
|
"rewards/chosen": -0.25552767515182495, |
|
"rewards/margins": 1.114168405532837, |
|
"rewards/rejected": -1.3696961402893066, |
|
"step": 530 |
|
}, |
|
{ |
|
"epoch": 0.565149136577708, |
|
"grad_norm": 105.92360698073946, |
|
"learning_rate": 2.367486188632446e-07, |
|
"logits/chosen": -2.6313700675964355, |
|
"logits/rejected": -2.5749824047088623, |
|
"logps/chosen": -290.64703369140625, |
|
"logps/rejected": -272.3555908203125, |
|
"loss": 0.517, |
|
"rewards/accuracies": 0.706250011920929, |
|
"rewards/chosen": -0.02643487975001335, |
|
"rewards/margins": 1.1456679105758667, |
|
"rewards/rejected": -1.172102689743042, |
|
"step": 540 |
|
}, |
|
{ |
|
"epoch": 0.5756148613291471, |
|
"grad_norm": 125.07827114011454, |
|
"learning_rate": 2.276292003092593e-07, |
|
"logits/chosen": -2.5747551918029785, |
|
"logits/rejected": -2.5299413204193115, |
|
"logps/chosen": -273.75311279296875, |
|
"logps/rejected": -243.8401641845703, |
|
"loss": 0.566, |
|
"rewards/accuracies": 0.731249988079071, |
|
"rewards/chosen": -0.24802279472351074, |
|
"rewards/margins": 1.2050869464874268, |
|
"rewards/rejected": -1.453109860420227, |
|
"step": 550 |
|
}, |
|
{ |
|
"epoch": 0.5860805860805861, |
|
"grad_norm": 119.35091354587506, |
|
"learning_rate": 2.185397007170141e-07, |
|
"logits/chosen": -2.539616584777832, |
|
"logits/rejected": -2.483868360519409, |
|
"logps/chosen": -284.36090087890625, |
|
"logps/rejected": -269.1393127441406, |
|
"loss": 0.5529, |
|
"rewards/accuracies": 0.737500011920929, |
|
"rewards/chosen": -0.15324077010154724, |
|
"rewards/margins": 1.269974708557129, |
|
"rewards/rejected": -1.4232155084609985, |
|
"step": 560 |
|
}, |
|
{ |
|
"epoch": 0.5965463108320251, |
|
"grad_norm": 121.35724227176418, |
|
"learning_rate": 2.094922764865619e-07, |
|
"logits/chosen": -2.620558977127075, |
|
"logits/rejected": -2.552006244659424, |
|
"logps/chosen": -282.87921142578125, |
|
"logps/rejected": -279.60931396484375, |
|
"loss": 0.5355, |
|
"rewards/accuracies": 0.737500011920929, |
|
"rewards/chosen": -0.3395797610282898, |
|
"rewards/margins": 1.0040457248687744, |
|
"rewards/rejected": -1.343625545501709, |
|
"step": 570 |
|
}, |
|
{ |
|
"epoch": 0.6070120355834642, |
|
"grad_norm": 114.95741828344511, |
|
"learning_rate": 2.0049902774588797e-07, |
|
"logits/chosen": -2.604160785675049, |
|
"logits/rejected": -2.53875994682312, |
|
"logps/chosen": -254.87548828125, |
|
"logps/rejected": -224.6983184814453, |
|
"loss": 0.5335, |
|
"rewards/accuracies": 0.7437499761581421, |
|
"rewards/chosen": -0.15681779384613037, |
|
"rewards/margins": 1.0275707244873047, |
|
"rewards/rejected": -1.1843883991241455, |
|
"step": 580 |
|
}, |
|
{ |
|
"epoch": 0.6174777603349032, |
|
"grad_norm": 116.63197017075697, |
|
"learning_rate": 1.9157198216806238e-07, |
|
"logits/chosen": -2.5892062187194824, |
|
"logits/rejected": -2.523660659790039, |
|
"logps/chosen": -282.70709228515625, |
|
"logps/rejected": -283.9153137207031, |
|
"loss": 0.5121, |
|
"rewards/accuracies": 0.737500011920929, |
|
"rewards/chosen": -0.4720216691493988, |
|
"rewards/margins": 1.0485048294067383, |
|
"rewards/rejected": -1.52052640914917, |
|
"step": 590 |
|
}, |
|
{ |
|
"epoch": 0.6279434850863422, |
|
"grad_norm": 90.93991377080742, |
|
"learning_rate": 1.8272307888529274e-07, |
|
"logits/chosen": -2.502859592437744, |
|
"logits/rejected": -2.44142746925354, |
|
"logps/chosen": -288.04864501953125, |
|
"logps/rejected": -274.6844787597656, |
|
"loss": 0.517, |
|
"rewards/accuracies": 0.699999988079071, |
|
"rewards/chosen": -0.5497933626174927, |
|
"rewards/margins": 0.9119491577148438, |
|
"rewards/rejected": -1.461742639541626, |
|
"step": 600 |
|
}, |
|
{ |
|
"epoch": 0.6279434850863422, |
|
"eval_logits/chosen": -2.5743534564971924, |
|
"eval_logits/rejected": -2.518336296081543, |
|
"eval_logps/chosen": -284.8755187988281, |
|
"eval_logps/rejected": -274.78082275390625, |
|
"eval_loss": 0.5262279510498047, |
|
"eval_rewards/accuracies": 0.7698412537574768, |
|
"eval_rewards/chosen": -0.29215142130851746, |
|
"eval_rewards/margins": 1.1640383005142212, |
|
"eval_rewards/rejected": -1.4561898708343506, |
|
"eval_runtime": 206.851, |
|
"eval_samples_per_second": 9.669, |
|
"eval_steps_per_second": 0.305, |
|
"step": 600 |
|
}, |
|
{ |
|
"epoch": 0.6384092098377813, |
|
"grad_norm": 109.87505024379428, |
|
"learning_rate": 1.7396415252139288e-07, |
|
"logits/chosen": -2.54246187210083, |
|
"logits/rejected": -2.4643430709838867, |
|
"logps/chosen": -308.33099365234375, |
|
"logps/rejected": -259.29583740234375, |
|
"loss": 0.5252, |
|
"rewards/accuracies": 0.762499988079071, |
|
"rewards/chosen": -0.2741188406944275, |
|
"rewards/margins": 1.187243103981018, |
|
"rewards/rejected": -1.4613618850708008, |
|
"step": 610 |
|
}, |
|
{ |
|
"epoch": 0.6488749345892203, |
|
"grad_norm": 92.62549956674485, |
|
"learning_rate": 1.6530691736402316e-07, |
|
"logits/chosen": -2.4948151111602783, |
|
"logits/rejected": -2.4823691844940186, |
|
"logps/chosen": -271.5950927734375, |
|
"logps/rejected": -249.6556396484375, |
|
"loss": 0.4543, |
|
"rewards/accuracies": 0.8062499761581421, |
|
"rewards/chosen": -0.05750391632318497, |
|
"rewards/margins": 1.2674978971481323, |
|
"rewards/rejected": -1.3250019550323486, |
|
"step": 620 |
|
}, |
|
{ |
|
"epoch": 0.6593406593406593, |
|
"grad_norm": 115.38512973551775, |
|
"learning_rate": 1.5676295169786864e-07, |
|
"logits/chosen": -2.588836193084717, |
|
"logits/rejected": -2.5063748359680176, |
|
"logps/chosen": -272.53277587890625, |
|
"logps/rejected": -246.722900390625, |
|
"loss": 0.4912, |
|
"rewards/accuracies": 0.78125, |
|
"rewards/chosen": 0.21716764569282532, |
|
"rewards/margins": 1.2922747135162354, |
|
"rewards/rejected": -1.0751070976257324, |
|
"step": 630 |
|
}, |
|
{ |
|
"epoch": 0.6698063840920984, |
|
"grad_norm": 98.58039473996509, |
|
"learning_rate": 1.483436823197092e-07, |
|
"logits/chosen": -2.5013747215270996, |
|
"logits/rejected": -2.4492290019989014, |
|
"logps/chosen": -267.44415283203125, |
|
"logps/rejected": -240.54727172851562, |
|
"loss": 0.4593, |
|
"rewards/accuracies": 0.7124999761581421, |
|
"rewards/chosen": -0.04387538880109787, |
|
"rewards/margins": 1.1440480947494507, |
|
"rewards/rejected": -1.1879234313964844, |
|
"step": 640 |
|
}, |
|
{ |
|
"epoch": 0.6802721088435374, |
|
"grad_norm": 108.19817847647745, |
|
"learning_rate": 1.4006036925609243e-07, |
|
"logits/chosen": -2.545503616333008, |
|
"logits/rejected": -2.5265164375305176, |
|
"logps/chosen": -264.9919738769531, |
|
"logps/rejected": -279.6017150878906, |
|
"loss": 0.5124, |
|
"rewards/accuracies": 0.7124999761581421, |
|
"rewards/chosen": 0.26580336689949036, |
|
"rewards/margins": 1.0504064559936523, |
|
"rewards/rejected": -0.7846031188964844, |
|
"step": 650 |
|
}, |
|
{ |
|
"epoch": 0.6907378335949764, |
|
"grad_norm": 103.96694126480867, |
|
"learning_rate": 1.319240907040458e-07, |
|
"logits/chosen": -2.545365571975708, |
|
"logits/rejected": -2.4856820106506348, |
|
"logps/chosen": -279.41656494140625, |
|
"logps/rejected": -267.95294189453125, |
|
"loss": 0.5078, |
|
"rewards/accuracies": 0.7749999761581421, |
|
"rewards/chosen": 0.28060275316238403, |
|
"rewards/margins": 1.3304187059402466, |
|
"rewards/rejected": -1.0498158931732178, |
|
"step": 660 |
|
}, |
|
{ |
|
"epoch": 0.7012035583464155, |
|
"grad_norm": 126.63379779239241, |
|
"learning_rate": 1.239457282149695e-07, |
|
"logits/chosen": -2.5796685218811035, |
|
"logits/rejected": -2.5014591217041016, |
|
"logps/chosen": -288.4901428222656, |
|
"logps/rejected": -296.0643615722656, |
|
"loss": 0.4813, |
|
"rewards/accuracies": 0.75, |
|
"rewards/chosen": 0.004715624265372753, |
|
"rewards/margins": 1.2996948957443237, |
|
"rewards/rejected": -1.2949790954589844, |
|
"step": 670 |
|
}, |
|
{ |
|
"epoch": 0.7116692830978545, |
|
"grad_norm": 104.43312059048667, |
|
"learning_rate": 1.1613595214152711e-07, |
|
"logits/chosen": -2.5504367351531982, |
|
"logits/rejected": -2.536426067352295, |
|
"logps/chosen": -310.7039489746094, |
|
"logps/rejected": -300.8048400878906, |
|
"loss": 0.5083, |
|
"rewards/accuracies": 0.7749999761581421, |
|
"rewards/chosen": 0.10008208453655243, |
|
"rewards/margins": 1.3875254392623901, |
|
"rewards/rejected": -1.2874433994293213, |
|
"step": 680 |
|
}, |
|
{ |
|
"epoch": 0.7221350078492935, |
|
"grad_norm": 86.78408797627893, |
|
"learning_rate": 1.0850520736699362e-07, |
|
"logits/chosen": -2.5602102279663086, |
|
"logits/rejected": -2.476207971572876, |
|
"logps/chosen": -272.1448974609375, |
|
"logps/rejected": -247.1821746826172, |
|
"loss": 0.4535, |
|
"rewards/accuracies": 0.7749999761581421, |
|
"rewards/chosen": -0.14905604720115662, |
|
"rewards/margins": 1.301754355430603, |
|
"rewards/rejected": -1.4508103132247925, |
|
"step": 690 |
|
}, |
|
{ |
|
"epoch": 0.7326007326007326, |
|
"grad_norm": 101.551211026177, |
|
"learning_rate": 1.0106369933615042e-07, |
|
"logits/chosen": -2.5397322177886963, |
|
"logits/rejected": -2.493654251098633, |
|
"logps/chosen": -256.18463134765625, |
|
"logps/rejected": -252.2621612548828, |
|
"loss": 0.4766, |
|
"rewards/accuracies": 0.7749999761581421, |
|
"rewards/chosen": -0.2093852460384369, |
|
"rewards/margins": 1.1600673198699951, |
|
"rewards/rejected": -1.369452714920044, |
|
"step": 700 |
|
}, |
|
{ |
|
"epoch": 0.7326007326007326, |
|
"eval_logits/chosen": -2.575119972229004, |
|
"eval_logits/rejected": -2.5193893909454346, |
|
"eval_logps/chosen": -282.1366271972656, |
|
"eval_logps/rejected": -273.1544494628906, |
|
"eval_loss": 0.527851402759552, |
|
"eval_rewards/accuracies": 0.7797619104385376, |
|
"eval_rewards/chosen": -0.018264232203364372, |
|
"eval_rewards/margins": 1.2752907276153564, |
|
"eval_rewards/rejected": -1.2935550212860107, |
|
"eval_runtime": 206.8622, |
|
"eval_samples_per_second": 9.668, |
|
"eval_steps_per_second": 0.305, |
|
"step": 700 |
|
}, |
|
{ |
|
"epoch": 0.7430664573521716, |
|
"grad_norm": 195.86678751064096, |
|
"learning_rate": 9.382138040640714e-08, |
|
"logits/chosen": -2.5042312145233154, |
|
"logits/rejected": -2.4359371662139893, |
|
"logps/chosen": -261.2381896972656, |
|
"logps/rejected": -257.90142822265625, |
|
"loss": 0.5532, |
|
"rewards/accuracies": 0.7437499761581421, |
|
"rewards/chosen": -0.27821090817451477, |
|
"rewards/margins": 1.218544602394104, |
|
"rewards/rejected": -1.4967554807662964, |
|
"step": 710 |
|
}, |
|
{ |
|
"epoch": 0.7535321821036107, |
|
"grad_norm": 97.34497217574805, |
|
"learning_rate": 8.678793653740632e-08, |
|
"logits/chosen": -2.5442166328430176, |
|
"logits/rejected": -2.506136417388916, |
|
"logps/chosen": -283.2102966308594, |
|
"logps/rejected": -260.00079345703125, |
|
"loss": 0.5108, |
|
"rewards/accuracies": 0.768750011920929, |
|
"rewards/chosen": 0.05546964332461357, |
|
"rewards/margins": 1.3087044954299927, |
|
"rewards/rejected": -1.25323486328125, |
|
"step": 720 |
|
}, |
|
{ |
|
"epoch": 0.7639979068550498, |
|
"grad_norm": 112.03077715841201, |
|
"learning_rate": 7.997277433690983e-08, |
|
"logits/chosen": -2.5620975494384766, |
|
"logits/rejected": -2.50207781791687, |
|
"logps/chosen": -268.3021545410156, |
|
"logps/rejected": -249.5152587890625, |
|
"loss": 0.4936, |
|
"rewards/accuracies": 0.7124999761581421, |
|
"rewards/chosen": -0.2925592064857483, |
|
"rewards/margins": 0.9584562182426453, |
|
"rewards/rejected": -1.251015543937683, |
|
"step": 730 |
|
}, |
|
{ |
|
"epoch": 0.7744636316064888, |
|
"grad_norm": 104.0919829579398, |
|
"learning_rate": 7.338500848029602e-08, |
|
"logits/chosen": -2.5124528408050537, |
|
"logits/rejected": -2.4893691539764404, |
|
"logps/chosen": -255.79995727539062, |
|
"logps/rejected": -245.2656707763672, |
|
"loss": 0.557, |
|
"rewards/accuracies": 0.699999988079071, |
|
"rewards/chosen": -0.2949948012828827, |
|
"rewards/margins": 1.0080726146697998, |
|
"rewards/rejected": -1.3030673265457153, |
|
"step": 740 |
|
}, |
|
{ |
|
"epoch": 0.7849293563579278, |
|
"grad_norm": 99.74133982899394, |
|
"learning_rate": 6.70334495204884e-08, |
|
"logits/chosen": -2.4846601486206055, |
|
"logits/rejected": -2.44720458984375, |
|
"logps/chosen": -250.17013549804688, |
|
"logps/rejected": -258.7621765136719, |
|
"loss": 0.4753, |
|
"rewards/accuracies": 0.737500011920929, |
|
"rewards/chosen": -0.3299402594566345, |
|
"rewards/margins": 1.1778974533081055, |
|
"rewards/rejected": -1.5078377723693848, |
|
"step": 750 |
|
}, |
|
{ |
|
"epoch": 0.7953950811093669, |
|
"grad_norm": 73.11096892518727, |
|
"learning_rate": 6.092659210462231e-08, |
|
"logits/chosen": -2.5241293907165527, |
|
"logits/rejected": -2.492896318435669, |
|
"logps/chosen": -244.87533569335938, |
|
"logps/rejected": -258.718017578125, |
|
"loss": 0.5085, |
|
"rewards/accuracies": 0.762499988079071, |
|
"rewards/chosen": -0.351382315158844, |
|
"rewards/margins": 1.2206757068634033, |
|
"rewards/rejected": -1.5720579624176025, |
|
"step": 760 |
|
}, |
|
{ |
|
"epoch": 0.8058608058608059, |
|
"grad_norm": 75.76619723672131, |
|
"learning_rate": 5.507260361320737e-08, |
|
"logits/chosen": -2.559986114501953, |
|
"logits/rejected": -2.5477547645568848, |
|
"logps/chosen": -305.89483642578125, |
|
"logps/rejected": -336.5893859863281, |
|
"loss": 0.4732, |
|
"rewards/accuracies": 0.7875000238418579, |
|
"rewards/chosen": -0.25339916348457336, |
|
"rewards/margins": 1.2115254402160645, |
|
"rewards/rejected": -1.4649248123168945, |
|
"step": 770 |
|
}, |
|
{ |
|
"epoch": 0.8163265306122449, |
|
"grad_norm": 105.8401947481118, |
|
"learning_rate": 4.947931323697982e-08, |
|
"logits/chosen": -2.517803430557251, |
|
"logits/rejected": -2.4777727127075195, |
|
"logps/chosen": -319.5518493652344, |
|
"logps/rejected": -279.0348815917969, |
|
"loss": 0.5237, |
|
"rewards/accuracies": 0.6875, |
|
"rewards/chosen": -0.0607873797416687, |
|
"rewards/margins": 1.0895849466323853, |
|
"rewards/rejected": -1.1503721475601196, |
|
"step": 780 |
|
}, |
|
{ |
|
"epoch": 0.826792255363684, |
|
"grad_norm": 113.35027564232854, |
|
"learning_rate": 4.415420150605398e-08, |
|
"logits/chosen": -2.5854082107543945, |
|
"logits/rejected": -2.4739928245544434, |
|
"logps/chosen": -280.592529296875, |
|
"logps/rejected": -259.7514343261719, |
|
"loss": 0.5226, |
|
"rewards/accuracies": 0.7562500238418579, |
|
"rewards/chosen": -0.20229856669902802, |
|
"rewards/margins": 1.2445086240768433, |
|
"rewards/rejected": -1.4468071460723877, |
|
"step": 790 |
|
}, |
|
{ |
|
"epoch": 0.837257980115123, |
|
"grad_norm": 115.9328163130989, |
|
"learning_rate": 3.9104390285376374e-08, |
|
"logits/chosen": -2.4794726371765137, |
|
"logits/rejected": -2.463261365890503, |
|
"logps/chosen": -268.74591064453125, |
|
"logps/rejected": -277.9842834472656, |
|
"loss": 0.4894, |
|
"rewards/accuracies": 0.7250000238418579, |
|
"rewards/chosen": -0.2617167830467224, |
|
"rewards/margins": 1.2013636827468872, |
|
"rewards/rejected": -1.4630804061889648, |
|
"step": 800 |
|
}, |
|
{ |
|
"epoch": 0.837257980115123, |
|
"eval_logits/chosen": -2.5850720405578613, |
|
"eval_logits/rejected": -2.531074047088623, |
|
"eval_logps/chosen": -282.5210876464844, |
|
"eval_logps/rejected": -272.81268310546875, |
|
"eval_loss": 0.5257476568222046, |
|
"eval_rewards/accuracies": 0.7777777910232544, |
|
"eval_rewards/chosen": -0.05670681595802307, |
|
"eval_rewards/margins": 1.2026678323745728, |
|
"eval_rewards/rejected": -1.259374737739563, |
|
"eval_runtime": 207.4453, |
|
"eval_samples_per_second": 9.641, |
|
"eval_steps_per_second": 0.304, |
|
"step": 800 |
|
}, |
|
{ |
|
"epoch": 0.847723704866562, |
|
"grad_norm": 87.80187909830617, |
|
"learning_rate": 3.433663324986208e-08, |
|
"logits/chosen": -2.5537991523742676, |
|
"logits/rejected": -2.495201826095581, |
|
"logps/chosen": -292.41900634765625, |
|
"logps/rejected": -258.72625732421875, |
|
"loss": 0.5129, |
|
"rewards/accuracies": 0.737500011920929, |
|
"rewards/chosen": -0.12009086459875107, |
|
"rewards/margins": 1.1076496839523315, |
|
"rewards/rejected": -1.2277406454086304, |
|
"step": 810 |
|
}, |
|
{ |
|
"epoch": 0.858189429618001, |
|
"grad_norm": 113.52567438362996, |
|
"learning_rate": 2.9857306851953897e-08, |
|
"logits/chosen": -2.5736160278320312, |
|
"logits/rejected": -2.5441737174987793, |
|
"logps/chosen": -305.28387451171875, |
|
"logps/rejected": -295.17120361328125, |
|
"loss": 0.508, |
|
"rewards/accuracies": 0.75, |
|
"rewards/chosen": -0.05176614597439766, |
|
"rewards/margins": 1.149815320968628, |
|
"rewards/rejected": -1.201581358909607, |
|
"step": 820 |
|
}, |
|
{ |
|
"epoch": 0.8686551543694401, |
|
"grad_norm": 94.35513967454145, |
|
"learning_rate": 2.567240179368185e-08, |
|
"logits/chosen": -2.5325002670288086, |
|
"logits/rejected": -2.5152266025543213, |
|
"logps/chosen": -259.08319091796875, |
|
"logps/rejected": -262.1970520019531, |
|
"loss": 0.5463, |
|
"rewards/accuracies": 0.7437499761581421, |
|
"rewards/chosen": -0.09561798721551895, |
|
"rewards/margins": 1.1467230319976807, |
|
"rewards/rejected": -1.242341160774231, |
|
"step": 830 |
|
}, |
|
{ |
|
"epoch": 0.8791208791208791, |
|
"grad_norm": 129.97386022506407, |
|
"learning_rate": 2.1787515014630357e-08, |
|
"logits/chosen": -2.540358304977417, |
|
"logits/rejected": -2.4877097606658936, |
|
"logps/chosen": -282.8505859375, |
|
"logps/rejected": -278.73236083984375, |
|
"loss": 0.4725, |
|
"rewards/accuracies": 0.7437499761581421, |
|
"rewards/chosen": -0.05927306413650513, |
|
"rewards/margins": 1.2071725130081177, |
|
"rewards/rejected": -1.266445517539978, |
|
"step": 840 |
|
}, |
|
{ |
|
"epoch": 0.8895866038723181, |
|
"grad_norm": 79.91861676789057, |
|
"learning_rate": 1.820784220652766e-08, |
|
"logits/chosen": -2.480196714401245, |
|
"logits/rejected": -2.413062572479248, |
|
"logps/chosen": -282.0146179199219, |
|
"logps/rejected": -246.9482421875, |
|
"loss": 0.4622, |
|
"rewards/accuracies": 0.78125, |
|
"rewards/chosen": -0.1173391118645668, |
|
"rewards/margins": 1.2294926643371582, |
|
"rewards/rejected": -1.3468319177627563, |
|
"step": 850 |
|
}, |
|
{ |
|
"epoch": 0.9000523286237572, |
|
"grad_norm": 108.90343136375496, |
|
"learning_rate": 1.4938170864468636e-08, |
|
"logits/chosen": -2.517470359802246, |
|
"logits/rejected": -2.458211898803711, |
|
"logps/chosen": -289.65374755859375, |
|
"logps/rejected": -274.5342102050781, |
|
"loss": 0.4958, |
|
"rewards/accuracies": 0.768750011920929, |
|
"rewards/chosen": -0.1372273564338684, |
|
"rewards/margins": 1.1825356483459473, |
|
"rewards/rejected": -1.3197630643844604, |
|
"step": 860 |
|
}, |
|
{ |
|
"epoch": 0.9105180533751962, |
|
"grad_norm": 153.358042630482, |
|
"learning_rate": 1.1982873884064465e-08, |
|
"logits/chosen": -2.578099012374878, |
|
"logits/rejected": -2.503145933151245, |
|
"logps/chosen": -271.9369201660156, |
|
"logps/rejected": -258.82427978515625, |
|
"loss": 0.5166, |
|
"rewards/accuracies": 0.7250000238418579, |
|
"rewards/chosen": -0.014210554771125317, |
|
"rewards/margins": 1.2173117399215698, |
|
"rewards/rejected": -1.2315223217010498, |
|
"step": 870 |
|
}, |
|
{ |
|
"epoch": 0.9209837781266352, |
|
"grad_norm": 88.58371346729444, |
|
"learning_rate": 9.345903713082304e-09, |
|
"logits/chosen": -2.5621767044067383, |
|
"logits/rejected": -2.545060396194458, |
|
"logps/chosen": -303.8287658691406, |
|
"logps/rejected": -306.15374755859375, |
|
"loss": 0.4932, |
|
"rewards/accuracies": 0.7562500238418579, |
|
"rewards/chosen": 0.02834125980734825, |
|
"rewards/margins": 1.1778244972229004, |
|
"rewards/rejected": -1.149483323097229, |
|
"step": 880 |
|
}, |
|
{ |
|
"epoch": 0.9314495028780743, |
|
"grad_norm": 123.70760581405742, |
|
"learning_rate": 7.030787065396865e-09, |
|
"logits/chosen": -2.5063490867614746, |
|
"logits/rejected": -2.4980926513671875, |
|
"logps/chosen": -288.7598571777344, |
|
"logps/rejected": -291.5402526855469, |
|
"loss": 0.5139, |
|
"rewards/accuracies": 0.737500011920929, |
|
"rewards/chosen": -0.11085305362939835, |
|
"rewards/margins": 1.0220136642456055, |
|
"rewards/rejected": -1.132866621017456, |
|
"step": 890 |
|
}, |
|
{ |
|
"epoch": 0.9419152276295133, |
|
"grad_norm": 123.75723552086023, |
|
"learning_rate": 5.04062020432286e-09, |
|
"logits/chosen": -2.5003843307495117, |
|
"logits/rejected": -2.4942610263824463, |
|
"logps/chosen": -281.16094970703125, |
|
"logps/rejected": -294.83892822265625, |
|
"loss": 0.4722, |
|
"rewards/accuracies": 0.7250000238418579, |
|
"rewards/chosen": -0.08161640912294388, |
|
"rewards/margins": 1.2573249340057373, |
|
"rewards/rejected": -1.3389413356781006, |
|
"step": 900 |
|
}, |
|
{ |
|
"epoch": 0.9419152276295133, |
|
"eval_logits/chosen": -2.590116024017334, |
|
"eval_logits/rejected": -2.5362370014190674, |
|
"eval_logps/chosen": -282.11407470703125, |
|
"eval_logps/rejected": -272.7223205566406, |
|
"eval_loss": 0.5279979109764099, |
|
"eval_rewards/accuracies": 0.7797619104385376, |
|
"eval_rewards/chosen": -0.01600908488035202, |
|
"eval_rewards/margins": 1.2343299388885498, |
|
"eval_rewards/rejected": -1.2503389120101929, |
|
"eval_runtime": 207.5627, |
|
"eval_samples_per_second": 9.636, |
|
"eval_steps_per_second": 0.304, |
|
"step": 900 |
|
}, |
|
{ |
|
"epoch": 0.9523809523809523, |
|
"grad_norm": 117.42904817244555, |
|
"learning_rate": 3.3780648016376866e-09, |
|
"logits/chosen": -2.4798412322998047, |
|
"logits/rejected": -2.4555060863494873, |
|
"logps/chosen": -224.1981658935547, |
|
"logps/rejected": -224.94448852539062, |
|
"loss": 0.4853, |
|
"rewards/accuracies": 0.78125, |
|
"rewards/chosen": -0.0745500773191452, |
|
"rewards/margins": 1.1322568655014038, |
|
"rewards/rejected": -1.206807017326355, |
|
"step": 910 |
|
}, |
|
{ |
|
"epoch": 0.9628466771323915, |
|
"grad_norm": 118.07550601245399, |
|
"learning_rate": 2.0453443778310766e-09, |
|
"logits/chosen": -2.59162974357605, |
|
"logits/rejected": -2.536531925201416, |
|
"logps/chosen": -287.6225280761719, |
|
"logps/rejected": -264.23828125, |
|
"loss": 0.5345, |
|
"rewards/accuracies": 0.78125, |
|
"rewards/chosen": -0.09747320413589478, |
|
"rewards/margins": 1.166621208190918, |
|
"rewards/rejected": -1.264094591140747, |
|
"step": 920 |
|
}, |
|
{ |
|
"epoch": 0.9733124018838305, |
|
"grad_norm": 76.103065820101, |
|
"learning_rate": 1.0442413283435758e-09, |
|
"logits/chosen": -2.567437171936035, |
|
"logits/rejected": -2.467308521270752, |
|
"logps/chosen": -315.1402893066406, |
|
"logps/rejected": -279.08343505859375, |
|
"loss": 0.4831, |
|
"rewards/accuracies": 0.768750011920929, |
|
"rewards/chosen": -0.03391196206212044, |
|
"rewards/margins": 1.449866533279419, |
|
"rewards/rejected": -1.4837785959243774, |
|
"step": 930 |
|
}, |
|
{ |
|
"epoch": 0.9837781266352695, |
|
"grad_norm": 105.2080573269562, |
|
"learning_rate": 3.760945397705828e-10, |
|
"logits/chosen": -2.5792009830474854, |
|
"logits/rejected": -2.542511463165283, |
|
"logps/chosen": -261.95806884765625, |
|
"logps/rejected": -271.60552978515625, |
|
"loss": 0.4913, |
|
"rewards/accuracies": 0.78125, |
|
"rewards/chosen": 0.12502527236938477, |
|
"rewards/margins": 1.313679575920105, |
|
"rewards/rejected": -1.1886541843414307, |
|
"step": 940 |
|
}, |
|
{ |
|
"epoch": 0.9942438513867086, |
|
"grad_norm": 121.4273155412146, |
|
"learning_rate": 4.17975992204056e-11, |
|
"logits/chosen": -2.5649020671844482, |
|
"logits/rejected": -2.5450055599212646, |
|
"logps/chosen": -303.64208984375, |
|
"logps/rejected": -295.95794677734375, |
|
"loss": 0.4961, |
|
"rewards/accuracies": 0.7124999761581421, |
|
"rewards/chosen": -0.08054888248443604, |
|
"rewards/margins": 1.018601417541504, |
|
"rewards/rejected": -1.0991504192352295, |
|
"step": 950 |
|
}, |
|
{ |
|
"epoch": 0.9994767137624281, |
|
"step": 955, |
|
"total_flos": 0.0, |
|
"train_loss": 0.528570184657711, |
|
"train_runtime": 17823.4182, |
|
"train_samples_per_second": 3.43, |
|
"train_steps_per_second": 0.054 |
|
} |
|
], |
|
"logging_steps": 10, |
|
"max_steps": 955, |
|
"num_input_tokens_seen": 0, |
|
"num_train_epochs": 1, |
|
"save_steps": 100, |
|
"total_flos": 0.0, |
|
"train_batch_size": 8, |
|
"trial_name": null, |
|
"trial_params": null |
|
} |
|
|