diff --git "a/trainer_state.json" "b/trainer_state.json" new file mode 100644--- /dev/null +++ "b/trainer_state.json" @@ -0,0 +1,18200 @@ +{ + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 2.999297541394882, + "eval_steps": 400, + "global_step": 5604, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.002676032781401572, + "grad_norm": 6.283021946330416, + "learning_rate": 8.9126559714795e-09, + "logits/chosen": -0.05665075033903122, + "logits/rejected": 0.15200912952423096, + "logps/chosen": -1.7162984609603882, + "logps/rejected": -1.889739990234375, + "loss": 0.5728, + "rewards/accuracies": 0.5687500238418579, + "rewards/chosen": -1.7162984609603882, + "rewards/margins": 0.17344175279140472, + "rewards/rejected": -1.889739990234375, + "semantic_entropy": 0.6583643555641174, + "step": 5 + }, + { + "epoch": 0.005352065562803144, + "grad_norm": 5.209798699126062, + "learning_rate": 1.7825311942959e-08, + "logits/chosen": 0.010867366567254066, + "logits/rejected": 0.13097676634788513, + "logps/chosen": -1.8017022609710693, + "logps/rejected": -1.846153974533081, + "loss": 0.5698, + "rewards/accuracies": 0.5562499761581421, + "rewards/chosen": -1.8017022609710693, + "rewards/margins": 0.04445166885852814, + "rewards/rejected": -1.846153974533081, + "semantic_entropy": 0.6395805478096008, + "step": 10 + }, + { + "epoch": 0.008028098344204716, + "grad_norm": 5.586761639648108, + "learning_rate": 2.67379679144385e-08, + "logits/chosen": -0.03902262821793556, + "logits/rejected": 0.05713306739926338, + "logps/chosen": -1.6349042654037476, + "logps/rejected": -1.7645162343978882, + "loss": 0.6338, + "rewards/accuracies": 0.4937500059604645, + "rewards/chosen": -1.6349042654037476, + "rewards/margins": 0.1296120136976242, + "rewards/rejected": -1.7645162343978882, + "semantic_entropy": 0.6933382749557495, + "step": 15 + }, + { + "epoch": 0.010704131125606288, + "grad_norm": 6.56499151474477, + "learning_rate": 3.5650623885918e-08, + "logits/chosen": -0.0506710521876812, + "logits/rejected": 0.031738776713609695, + "logps/chosen": -1.7259511947631836, + "logps/rejected": -1.8071794509887695, + "loss": 0.6122, + "rewards/accuracies": 0.48750001192092896, + "rewards/chosen": -1.7259511947631836, + "rewards/margins": 0.08122824132442474, + "rewards/rejected": -1.8071794509887695, + "semantic_entropy": 0.6682224869728088, + "step": 20 + }, + { + "epoch": 0.013380163907007862, + "grad_norm": 6.873589227199821, + "learning_rate": 4.45632798573975e-08, + "logits/chosen": -0.049850888550281525, + "logits/rejected": 0.03644446283578873, + "logps/chosen": -1.8693091869354248, + "logps/rejected": -1.779761552810669, + "loss": 0.6281, + "rewards/accuracies": 0.375, + "rewards/chosen": -1.8693091869354248, + "rewards/margins": -0.08954761922359467, + "rewards/rejected": -1.779761552810669, + "semantic_entropy": 0.6430644392967224, + "step": 25 + }, + { + "epoch": 0.016056196688409432, + "grad_norm": 6.0582854700474815, + "learning_rate": 5.3475935828877e-08, + "logits/chosen": -0.09021981805562973, + "logits/rejected": 0.00014630556688643992, + "logps/chosen": -1.9092113971710205, + "logps/rejected": -1.832843542098999, + "loss": 0.5563, + "rewards/accuracies": 0.4375, + "rewards/chosen": -1.9092113971710205, + "rewards/margins": -0.07636785507202148, + "rewards/rejected": -1.832843542098999, + "semantic_entropy": 0.6176734566688538, + "step": 30 + }, + { + "epoch": 0.018732229469811006, + "grad_norm": 6.953077360616749, + "learning_rate": 6.23885918003565e-08, + "logits/chosen": -0.058985866606235504, + "logits/rejected": 0.09778233617544174, + "logps/chosen": -1.8485629558563232, + "logps/rejected": -1.9991296529769897, + "loss": 0.5799, + "rewards/accuracies": 0.48750001192092896, + "rewards/chosen": -1.8485629558563232, + "rewards/margins": 0.15056660771369934, + "rewards/rejected": -1.9991296529769897, + "semantic_entropy": 0.6344074010848999, + "step": 35 + }, + { + "epoch": 0.021408262251212576, + "grad_norm": 5.95861907781093, + "learning_rate": 7.1301247771836e-08, + "logits/chosen": 0.028498858213424683, + "logits/rejected": 0.20293541252613068, + "logps/chosen": -1.8856210708618164, + "logps/rejected": -1.7465416193008423, + "loss": 0.599, + "rewards/accuracies": 0.45625001192092896, + "rewards/chosen": -1.8856210708618164, + "rewards/margins": -0.13907942175865173, + "rewards/rejected": -1.7465416193008423, + "semantic_entropy": 0.6422087550163269, + "step": 40 + }, + { + "epoch": 0.02408429503261415, + "grad_norm": 6.724726630089442, + "learning_rate": 8.021390374331551e-08, + "logits/chosen": 0.014616099186241627, + "logits/rejected": 0.2100570946931839, + "logps/chosen": -1.8413318395614624, + "logps/rejected": -1.8763275146484375, + "loss": 0.5937, + "rewards/accuracies": 0.48750001192092896, + "rewards/chosen": -1.8413318395614624, + "rewards/margins": 0.03499563783407211, + "rewards/rejected": -1.8763275146484375, + "semantic_entropy": 0.6482739448547363, + "step": 45 + }, + { + "epoch": 0.026760327814015723, + "grad_norm": 7.060246294252313, + "learning_rate": 8.9126559714795e-08, + "logits/chosen": -0.03206641972064972, + "logits/rejected": 0.11953778564929962, + "logps/chosen": -1.9085075855255127, + "logps/rejected": -1.7868010997772217, + "loss": 0.5802, + "rewards/accuracies": 0.512499988079071, + "rewards/chosen": -1.9085075855255127, + "rewards/margins": -0.12170647084712982, + "rewards/rejected": -1.7868010997772217, + "semantic_entropy": 0.6314837336540222, + "step": 50 + }, + { + "epoch": 0.029436360595417294, + "grad_norm": 5.9296298405276495, + "learning_rate": 9.80392156862745e-08, + "logits/chosen": -0.11400948464870453, + "logits/rejected": 0.10222546011209488, + "logps/chosen": -1.8476674556732178, + "logps/rejected": -1.880591630935669, + "loss": 0.572, + "rewards/accuracies": 0.5562499761581421, + "rewards/chosen": -1.8476674556732178, + "rewards/margins": 0.03292415291070938, + "rewards/rejected": -1.880591630935669, + "semantic_entropy": 0.6409212946891785, + "step": 55 + }, + { + "epoch": 0.032112393376818864, + "grad_norm": 6.192609903998256, + "learning_rate": 1.06951871657754e-07, + "logits/chosen": -0.06294813007116318, + "logits/rejected": 0.12941911816596985, + "logps/chosen": -1.8072830438613892, + "logps/rejected": -1.9127168655395508, + "loss": 0.5598, + "rewards/accuracies": 0.5249999761581421, + "rewards/chosen": -1.8072830438613892, + "rewards/margins": 0.10543377697467804, + "rewards/rejected": -1.9127168655395508, + "semantic_entropy": 0.6318107843399048, + "step": 60 + }, + { + "epoch": 0.03478842615822044, + "grad_norm": 5.840367536650179, + "learning_rate": 1.158645276292335e-07, + "logits/chosen": -0.031631387770175934, + "logits/rejected": 0.11730644851922989, + "logps/chosen": -1.6562246084213257, + "logps/rejected": -1.7897517681121826, + "loss": 0.6131, + "rewards/accuracies": 0.518750011920929, + "rewards/chosen": -1.6562246084213257, + "rewards/margins": 0.13352727890014648, + "rewards/rejected": -1.7897517681121826, + "semantic_entropy": 0.691199004650116, + "step": 65 + }, + { + "epoch": 0.03746445893962201, + "grad_norm": 7.339621823064037, + "learning_rate": 1.24777183600713e-07, + "logits/chosen": -0.07941316813230515, + "logits/rejected": 0.07183218747377396, + "logps/chosen": -1.793076753616333, + "logps/rejected": -1.8430589437484741, + "loss": 0.5936, + "rewards/accuracies": 0.42500001192092896, + "rewards/chosen": -1.793076753616333, + "rewards/margins": 0.04998219758272171, + "rewards/rejected": -1.8430589437484741, + "semantic_entropy": 0.6468250155448914, + "step": 70 + }, + { + "epoch": 0.04014049172102358, + "grad_norm": 5.431450354661162, + "learning_rate": 1.3368983957219251e-07, + "logits/chosen": -0.0522337444126606, + "logits/rejected": 0.13029292225837708, + "logps/chosen": -1.8287160396575928, + "logps/rejected": -2.097831964492798, + "loss": 0.5472, + "rewards/accuracies": 0.5249999761581421, + "rewards/chosen": -1.8287160396575928, + "rewards/margins": 0.2691158354282379, + "rewards/rejected": -2.097831964492798, + "semantic_entropy": 0.6201809048652649, + "step": 75 + }, + { + "epoch": 0.04281652450242515, + "grad_norm": 5.120322608884211, + "learning_rate": 1.42602495543672e-07, + "logits/chosen": 0.00244101881980896, + "logits/rejected": 0.10615000873804092, + "logps/chosen": -1.7778211832046509, + "logps/rejected": -1.809617280960083, + "loss": 0.5924, + "rewards/accuracies": 0.512499988079071, + "rewards/chosen": -1.7778211832046509, + "rewards/margins": 0.03179623559117317, + "rewards/rejected": -1.809617280960083, + "semantic_entropy": 0.6536020040512085, + "step": 80 + }, + { + "epoch": 0.04549255728382673, + "grad_norm": 4.807337181522284, + "learning_rate": 1.5151515151515152e-07, + "logits/chosen": -0.14471609890460968, + "logits/rejected": 0.10519156605005264, + "logps/chosen": -1.8619205951690674, + "logps/rejected": -2.0526270866394043, + "loss": 0.5784, + "rewards/accuracies": 0.5062500238418579, + "rewards/chosen": -1.8619205951690674, + "rewards/margins": 0.19070668518543243, + "rewards/rejected": -2.0526270866394043, + "semantic_entropy": 0.6318272352218628, + "step": 85 + }, + { + "epoch": 0.0481685900652283, + "grad_norm": 5.097287930626076, + "learning_rate": 1.6042780748663102e-07, + "logits/chosen": 0.09117968380451202, + "logits/rejected": 0.05077249929308891, + "logps/chosen": -1.8373451232910156, + "logps/rejected": -1.8451576232910156, + "loss": 0.5988, + "rewards/accuracies": 0.4749999940395355, + "rewards/chosen": -1.8373451232910156, + "rewards/margins": 0.007812516763806343, + "rewards/rejected": -1.8451576232910156, + "semantic_entropy": 0.6506969332695007, + "step": 90 + }, + { + "epoch": 0.05084462284662987, + "grad_norm": 4.648043327968244, + "learning_rate": 1.693404634581105e-07, + "logits/chosen": -0.06766053289175034, + "logits/rejected": 0.08066412061452866, + "logps/chosen": -1.9248930215835571, + "logps/rejected": -2.0033318996429443, + "loss": 0.5561, + "rewards/accuracies": 0.581250011920929, + "rewards/chosen": -1.9248930215835571, + "rewards/margins": 0.07843902707099915, + "rewards/rejected": -2.0033318996429443, + "semantic_entropy": 0.6197448968887329, + "step": 95 + }, + { + "epoch": 0.05352065562803145, + "grad_norm": 4.21712461471473, + "learning_rate": 1.7825311942959e-07, + "logits/chosen": -0.04044175148010254, + "logits/rejected": 0.021123576909303665, + "logps/chosen": -1.8103749752044678, + "logps/rejected": -1.9236066341400146, + "loss": 0.567, + "rewards/accuracies": 0.5, + "rewards/chosen": -1.8103749752044678, + "rewards/margins": 0.11323173344135284, + "rewards/rejected": -1.9236066341400146, + "semantic_entropy": 0.6433764696121216, + "step": 100 + }, + { + "epoch": 0.05619668840943302, + "grad_norm": 4.152220901781052, + "learning_rate": 1.8716577540106952e-07, + "logits/chosen": 0.061221588402986526, + "logits/rejected": 0.08823797106742859, + "logps/chosen": -1.7954018115997314, + "logps/rejected": -1.9569883346557617, + "loss": 0.5718, + "rewards/accuracies": 0.5687500238418579, + "rewards/chosen": -1.7954018115997314, + "rewards/margins": 0.16158629953861237, + "rewards/rejected": -1.9569883346557617, + "semantic_entropy": 0.648250937461853, + "step": 105 + }, + { + "epoch": 0.05887272119083459, + "grad_norm": 4.253659296532382, + "learning_rate": 1.96078431372549e-07, + "logits/chosen": 0.019205275923013687, + "logits/rejected": 0.11396453529596329, + "logps/chosen": -1.8869092464447021, + "logps/rejected": -1.9481251239776611, + "loss": 0.5738, + "rewards/accuracies": 0.512499988079071, + "rewards/chosen": -1.8869092464447021, + "rewards/margins": 0.06121576577425003, + "rewards/rejected": -1.9481251239776611, + "semantic_entropy": 0.630905032157898, + "step": 110 + }, + { + "epoch": 0.06154875397223616, + "grad_norm": 5.0490119418622, + "learning_rate": 2.049910873440285e-07, + "logits/chosen": 0.050636060535907745, + "logits/rejected": 0.26533403992652893, + "logps/chosen": -1.8734686374664307, + "logps/rejected": -2.212010383605957, + "loss": 0.5106, + "rewards/accuracies": 0.612500011920929, + "rewards/chosen": -1.8734686374664307, + "rewards/margins": 0.33854159712791443, + "rewards/rejected": -2.212010383605957, + "semantic_entropy": 0.5976813435554504, + "step": 115 + }, + { + "epoch": 0.06422478675363773, + "grad_norm": 3.6994949372266035, + "learning_rate": 2.13903743315508e-07, + "logits/chosen": -0.07818116992712021, + "logits/rejected": 0.09761984646320343, + "logps/chosen": -2.0244534015655518, + "logps/rejected": -2.173408269882202, + "loss": 0.5293, + "rewards/accuracies": 0.4937500059604645, + "rewards/chosen": -2.0244534015655518, + "rewards/margins": 0.14895491302013397, + "rewards/rejected": -2.173408269882202, + "semantic_entropy": 0.6009461879730225, + "step": 120 + }, + { + "epoch": 0.0669008195350393, + "grad_norm": 4.7177100362781434, + "learning_rate": 2.2281639928698751e-07, + "logits/chosen": -0.060094915330410004, + "logits/rejected": 0.07375667989253998, + "logps/chosen": -1.9164215326309204, + "logps/rejected": -1.8263633251190186, + "loss": 0.5804, + "rewards/accuracies": 0.4749999940395355, + "rewards/chosen": -1.9164215326309204, + "rewards/margins": -0.09005783498287201, + "rewards/rejected": -1.8263633251190186, + "semantic_entropy": 0.6356278657913208, + "step": 125 + }, + { + "epoch": 0.06957685231644088, + "grad_norm": 4.3470043435073356, + "learning_rate": 2.31729055258467e-07, + "logits/chosen": 0.06565021723508835, + "logits/rejected": 0.2070547640323639, + "logps/chosen": -1.9880714416503906, + "logps/rejected": -2.111767530441284, + "loss": 0.5254, + "rewards/accuracies": 0.48124998807907104, + "rewards/chosen": -1.9880714416503906, + "rewards/margins": 0.12369606643915176, + "rewards/rejected": -2.111767530441284, + "semantic_entropy": 0.5927776098251343, + "step": 130 + }, + { + "epoch": 0.07225288509784245, + "grad_norm": 3.6305162217659905, + "learning_rate": 2.406417112299465e-07, + "logits/chosen": -0.030332911759614944, + "logits/rejected": 0.08955554664134979, + "logps/chosen": -2.080584764480591, + "logps/rejected": -2.0448110103607178, + "loss": 0.5326, + "rewards/accuracies": 0.518750011920929, + "rewards/chosen": -2.080584764480591, + "rewards/margins": -0.035773664712905884, + "rewards/rejected": -2.0448110103607178, + "semantic_entropy": 0.5878274440765381, + "step": 135 + }, + { + "epoch": 0.07492891787924402, + "grad_norm": 4.577194794071166, + "learning_rate": 2.49554367201426e-07, + "logits/chosen": -0.024137228727340698, + "logits/rejected": 0.14822633564472198, + "logps/chosen": -2.0620641708374023, + "logps/rejected": -2.340951919555664, + "loss": 0.4846, + "rewards/accuracies": 0.5249999761581421, + "rewards/chosen": -2.0620641708374023, + "rewards/margins": 0.2788878381252289, + "rewards/rejected": -2.340951919555664, + "semantic_entropy": 0.5582477450370789, + "step": 140 + }, + { + "epoch": 0.0776049506606456, + "grad_norm": 3.5772189076002534, + "learning_rate": 2.5846702317290554e-07, + "logits/chosen": -0.0062357052229344845, + "logits/rejected": 0.15561768412590027, + "logps/chosen": -2.054307699203491, + "logps/rejected": -2.20831298828125, + "loss": 0.5148, + "rewards/accuracies": 0.46875, + "rewards/chosen": -2.054307699203491, + "rewards/margins": 0.15400531888008118, + "rewards/rejected": -2.20831298828125, + "semantic_entropy": 0.5854038000106812, + "step": 145 + }, + { + "epoch": 0.08028098344204716, + "grad_norm": 3.320147224300046, + "learning_rate": 2.6737967914438503e-07, + "logits/chosen": -0.034813135862350464, + "logits/rejected": 0.1387842893600464, + "logps/chosen": -2.0534462928771973, + "logps/rejected": -2.0570671558380127, + "loss": 0.5563, + "rewards/accuracies": 0.512499988079071, + "rewards/chosen": -2.0534462928771973, + "rewards/margins": 0.0036209137178957462, + "rewards/rejected": -2.0570671558380127, + "semantic_entropy": 0.625429093837738, + "step": 150 + }, + { + "epoch": 0.08295701622344874, + "grad_norm": 3.471479341023171, + "learning_rate": 2.762923351158645e-07, + "logits/chosen": -0.029315322637557983, + "logits/rejected": 0.018080363050103188, + "logps/chosen": -2.1867306232452393, + "logps/rejected": -2.2244086265563965, + "loss": 0.5212, + "rewards/accuracies": 0.48750001192092896, + "rewards/chosen": -2.1867306232452393, + "rewards/margins": 0.037678077816963196, + "rewards/rejected": -2.2244086265563965, + "semantic_entropy": 0.5816220045089722, + "step": 155 + }, + { + "epoch": 0.0856330490048503, + "grad_norm": 3.7011763753323144, + "learning_rate": 2.85204991087344e-07, + "logits/chosen": -0.10797729343175888, + "logits/rejected": 0.044836580753326416, + "logps/chosen": -2.4026219844818115, + "logps/rejected": -2.3765602111816406, + "loss": 0.4831, + "rewards/accuracies": 0.48750001192092896, + "rewards/chosen": -2.4026219844818115, + "rewards/margins": -0.026061728596687317, + "rewards/rejected": -2.3765602111816406, + "semantic_entropy": 0.5305178761482239, + "step": 160 + }, + { + "epoch": 0.08830908178625188, + "grad_norm": 3.4705911589732112, + "learning_rate": 2.941176470588235e-07, + "logits/chosen": -0.01804346963763237, + "logits/rejected": 0.16947805881500244, + "logps/chosen": -2.112272024154663, + "logps/rejected": -2.4508056640625, + "loss": 0.5079, + "rewards/accuracies": 0.5687500238418579, + "rewards/chosen": -2.112272024154663, + "rewards/margins": 0.3385334312915802, + "rewards/rejected": -2.4508056640625, + "semantic_entropy": 0.5715000033378601, + "step": 165 + }, + { + "epoch": 0.09098511456765346, + "grad_norm": 3.413708707373631, + "learning_rate": 3.0303030303030305e-07, + "logits/chosen": -0.06662870943546295, + "logits/rejected": -0.008535407483577728, + "logps/chosen": -2.417525291442871, + "logps/rejected": -2.3798742294311523, + "loss": 0.4764, + "rewards/accuracies": 0.45625001192092896, + "rewards/chosen": -2.417525291442871, + "rewards/margins": -0.0376511812210083, + "rewards/rejected": -2.3798742294311523, + "semantic_entropy": 0.540223240852356, + "step": 170 + }, + { + "epoch": 0.09366114734905502, + "grad_norm": 3.543204898829903, + "learning_rate": 3.1194295900178254e-07, + "logits/chosen": 0.09488913416862488, + "logits/rejected": 0.09268101304769516, + "logps/chosen": -2.30230450630188, + "logps/rejected": -2.363039255142212, + "loss": 0.5165, + "rewards/accuracies": 0.46875, + "rewards/chosen": -2.30230450630188, + "rewards/margins": 0.06073470786213875, + "rewards/rejected": -2.363039255142212, + "semantic_entropy": 0.5503964424133301, + "step": 175 + }, + { + "epoch": 0.0963371801304566, + "grad_norm": 3.882410336806739, + "learning_rate": 3.2085561497326203e-07, + "logits/chosen": 0.025393879041075706, + "logits/rejected": 0.028031084686517715, + "logps/chosen": -2.42897367477417, + "logps/rejected": -2.349623203277588, + "loss": 0.4847, + "rewards/accuracies": 0.512499988079071, + "rewards/chosen": -2.42897367477417, + "rewards/margins": -0.079350546002388, + "rewards/rejected": -2.349623203277588, + "semantic_entropy": 0.5456860661506653, + "step": 180 + }, + { + "epoch": 0.09901321291185818, + "grad_norm": 6.489830693027923, + "learning_rate": 3.297682709447415e-07, + "logits/chosen": -0.1303943544626236, + "logits/rejected": -0.034635625779628754, + "logps/chosen": -2.4026060104370117, + "logps/rejected": -2.4866461753845215, + "loss": 0.5071, + "rewards/accuracies": 0.512499988079071, + "rewards/chosen": -2.4026060104370117, + "rewards/margins": 0.08404017984867096, + "rewards/rejected": -2.4866461753845215, + "semantic_entropy": 0.563964307308197, + "step": 185 + }, + { + "epoch": 0.10168924569325974, + "grad_norm": 5.983301960644024, + "learning_rate": 3.38680926916221e-07, + "logits/chosen": -0.03423957899212837, + "logits/rejected": 0.10569383203983307, + "logps/chosen": -3.298650026321411, + "logps/rejected": -3.035168409347534, + "loss": 0.4047, + "rewards/accuracies": 0.48750001192092896, + "rewards/chosen": -3.298650026321411, + "rewards/margins": -0.2634817659854889, + "rewards/rejected": -3.035168409347534, + "semantic_entropy": 0.45990005135536194, + "step": 190 + }, + { + "epoch": 0.10436527847466132, + "grad_norm": 4.991429794604448, + "learning_rate": 3.475935828877005e-07, + "logits/chosen": 0.02364252880215645, + "logits/rejected": 0.19518662989139557, + "logps/chosen": -2.553497076034546, + "logps/rejected": -2.5629844665527344, + "loss": 0.4404, + "rewards/accuracies": 0.5062500238418579, + "rewards/chosen": -2.553497076034546, + "rewards/margins": 0.009487425908446312, + "rewards/rejected": -2.5629844665527344, + "semantic_entropy": 0.5125027894973755, + "step": 195 + }, + { + "epoch": 0.1070413112560629, + "grad_norm": 4.42975674257316, + "learning_rate": 3.5650623885918e-07, + "logits/chosen": -0.04811898618936539, + "logits/rejected": 0.10957398265600204, + "logps/chosen": -3.3466639518737793, + "logps/rejected": -2.866375684738159, + "loss": 0.4166, + "rewards/accuracies": 0.5249999761581421, + "rewards/chosen": -3.3466639518737793, + "rewards/margins": -0.4802883267402649, + "rewards/rejected": -2.866375684738159, + "semantic_entropy": 0.47037285566329956, + "step": 200 + }, + { + "epoch": 0.10971734403746446, + "grad_norm": 3.7097199469373354, + "learning_rate": 3.654188948306595e-07, + "logits/chosen": -0.0347520187497139, + "logits/rejected": 0.13235226273536682, + "logps/chosen": -4.012370586395264, + "logps/rejected": -3.4062774181365967, + "loss": 0.3824, + "rewards/accuracies": 0.512499988079071, + "rewards/chosen": -4.012370586395264, + "rewards/margins": -0.6060926914215088, + "rewards/rejected": -3.4062774181365967, + "semantic_entropy": 0.44439277052879333, + "step": 205 + }, + { + "epoch": 0.11239337681886603, + "grad_norm": 3.006727006054674, + "learning_rate": 3.7433155080213904e-07, + "logits/chosen": -0.12415327876806259, + "logits/rejected": 0.09854158014059067, + "logps/chosen": -3.691174030303955, + "logps/rejected": -4.26821756362915, + "loss": 0.3329, + "rewards/accuracies": 0.606249988079071, + "rewards/chosen": -3.691174030303955, + "rewards/margins": 0.5770431756973267, + "rewards/rejected": -4.26821756362915, + "semantic_entropy": 0.41728726029396057, + "step": 210 + }, + { + "epoch": 0.1150694096002676, + "grad_norm": 3.284414971810367, + "learning_rate": 3.8324420677361853e-07, + "logits/chosen": -0.1523238718509674, + "logits/rejected": 0.121431365609169, + "logps/chosen": -3.3081634044647217, + "logps/rejected": -3.495288133621216, + "loss": 0.3532, + "rewards/accuracies": 0.5625, + "rewards/chosen": -3.3081634044647217, + "rewards/margins": 0.18712471425533295, + "rewards/rejected": -3.495288133621216, + "semantic_entropy": 0.4320215582847595, + "step": 215 + }, + { + "epoch": 0.11774544238166917, + "grad_norm": 4.517057249275933, + "learning_rate": 3.92156862745098e-07, + "logits/chosen": 0.08584829419851303, + "logits/rejected": 0.20471997559070587, + "logps/chosen": -4.001864433288574, + "logps/rejected": -4.645659923553467, + "loss": 0.314, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -4.001864433288574, + "rewards/margins": 0.6437958478927612, + "rewards/rejected": -4.645659923553467, + "semantic_entropy": 0.3854829668998718, + "step": 220 + }, + { + "epoch": 0.12042147516307075, + "grad_norm": 5.140563516027854, + "learning_rate": 4.010695187165775e-07, + "logits/chosen": -0.06603296846151352, + "logits/rejected": 0.13016948103904724, + "logps/chosen": -3.7264492511749268, + "logps/rejected": -3.8007068634033203, + "loss": 0.313, + "rewards/accuracies": 0.5562499761581421, + "rewards/chosen": -3.7264492511749268, + "rewards/margins": 0.07425764948129654, + "rewards/rejected": -3.8007068634033203, + "semantic_entropy": 0.3880666494369507, + "step": 225 + }, + { + "epoch": 0.12309750794447231, + "grad_norm": 4.465321116954656, + "learning_rate": 4.09982174688057e-07, + "logits/chosen": 0.06401034444570541, + "logits/rejected": 0.1629628837108612, + "logps/chosen": -4.803007125854492, + "logps/rejected": -4.7263898849487305, + "loss": 0.2807, + "rewards/accuracies": 0.512499988079071, + "rewards/chosen": -4.803007125854492, + "rewards/margins": -0.07661716639995575, + "rewards/rejected": -4.7263898849487305, + "semantic_entropy": 0.3396259844303131, + "step": 230 + }, + { + "epoch": 0.1257735407258739, + "grad_norm": 2.7042614308205235, + "learning_rate": 4.188948306595365e-07, + "logits/chosen": 0.08987166732549667, + "logits/rejected": 0.27392953634262085, + "logps/chosen": -4.6403398513793945, + "logps/rejected": -5.046909332275391, + "loss": 0.2585, + "rewards/accuracies": 0.5874999761581421, + "rewards/chosen": -4.6403398513793945, + "rewards/margins": 0.406569242477417, + "rewards/rejected": -5.046909332275391, + "semantic_entropy": 0.3378155827522278, + "step": 235 + }, + { + "epoch": 0.12844957350727546, + "grad_norm": 2.753676038637165, + "learning_rate": 4.27807486631016e-07, + "logits/chosen": 0.042416222393512726, + "logits/rejected": 0.18578985333442688, + "logps/chosen": -5.076837539672852, + "logps/rejected": -4.993945121765137, + "loss": 0.2619, + "rewards/accuracies": 0.518750011920929, + "rewards/chosen": -5.076837539672852, + "rewards/margins": -0.08289314806461334, + "rewards/rejected": -4.993945121765137, + "semantic_entropy": 0.31904274225234985, + "step": 240 + }, + { + "epoch": 0.13112560628867703, + "grad_norm": 2.4582120018377376, + "learning_rate": 4.3672014260249554e-07, + "logits/chosen": 0.10127206891775131, + "logits/rejected": 0.2557363510131836, + "logps/chosen": -4.425154685974121, + "logps/rejected": -5.560978889465332, + "loss": 0.2595, + "rewards/accuracies": 0.5874999761581421, + "rewards/chosen": -4.425154685974121, + "rewards/margins": 1.13582444190979, + "rewards/rejected": -5.560978889465332, + "semantic_entropy": 0.32832127809524536, + "step": 245 + }, + { + "epoch": 0.1338016390700786, + "grad_norm": 2.575075964822789, + "learning_rate": 4.4563279857397503e-07, + "logits/chosen": 0.08739835023880005, + "logits/rejected": 0.2964261472225189, + "logps/chosen": -6.85397481918335, + "logps/rejected": -6.661816596984863, + "loss": 0.2524, + "rewards/accuracies": 0.5249999761581421, + "rewards/chosen": -6.85397481918335, + "rewards/margins": -0.19215914607048035, + "rewards/rejected": -6.661816596984863, + "semantic_entropy": 0.31161242723464966, + "step": 250 + }, + { + "epoch": 0.1364776718514802, + "grad_norm": 3.1444323203134688, + "learning_rate": 4.545454545454545e-07, + "logits/chosen": 0.06549052894115448, + "logits/rejected": 0.27386870980262756, + "logps/chosen": -4.550482749938965, + "logps/rejected": -5.5093584060668945, + "loss": 0.2604, + "rewards/accuracies": 0.5625, + "rewards/chosen": -4.550482749938965, + "rewards/margins": 0.9588750600814819, + "rewards/rejected": -5.5093584060668945, + "semantic_entropy": 0.32709717750549316, + "step": 255 + }, + { + "epoch": 0.13915370463288176, + "grad_norm": 3.3065425131874067, + "learning_rate": 4.63458110516934e-07, + "logits/chosen": -0.11962959915399551, + "logits/rejected": 0.018381217494606972, + "logps/chosen": -6.10165548324585, + "logps/rejected": -5.083194732666016, + "loss": 0.2118, + "rewards/accuracies": 0.5375000238418579, + "rewards/chosen": -6.10165548324585, + "rewards/margins": -1.0184614658355713, + "rewards/rejected": -5.083194732666016, + "semantic_entropy": 0.27624058723449707, + "step": 260 + }, + { + "epoch": 0.1418297374142833, + "grad_norm": 2.4937567619586947, + "learning_rate": 4.723707664884135e-07, + "logits/chosen": 0.10635049641132355, + "logits/rejected": 0.20212987065315247, + "logps/chosen": -7.334754943847656, + "logps/rejected": -5.710747241973877, + "loss": 0.1978, + "rewards/accuracies": 0.5062500238418579, + "rewards/chosen": -7.334754943847656, + "rewards/margins": -1.6240085363388062, + "rewards/rejected": -5.710747241973877, + "semantic_entropy": 0.23839779198169708, + "step": 265 + }, + { + "epoch": 0.1445057701956849, + "grad_norm": 6.634798485239572, + "learning_rate": 4.81283422459893e-07, + "logits/chosen": 0.04447219520807266, + "logits/rejected": 0.23252050578594208, + "logps/chosen": -5.903593063354492, + "logps/rejected": -6.297295093536377, + "loss": 0.1997, + "rewards/accuracies": 0.512499988079071, + "rewards/chosen": -5.903593063354492, + "rewards/margins": 0.3937019407749176, + "rewards/rejected": -6.297295093536377, + "semantic_entropy": 0.24874810874462128, + "step": 270 + }, + { + "epoch": 0.14718180297708647, + "grad_norm": 12.381758710987581, + "learning_rate": 4.901960784313725e-07, + "logits/chosen": 0.30282384157180786, + "logits/rejected": 0.38939711451530457, + "logps/chosen": -7.172441005706787, + "logps/rejected": -7.899168491363525, + "loss": 0.182, + "rewards/accuracies": 0.59375, + "rewards/chosen": -7.172441005706787, + "rewards/margins": 0.7267270684242249, + "rewards/rejected": -7.899168491363525, + "semantic_entropy": 0.22203528881072998, + "step": 275 + }, + { + "epoch": 0.14985783575848804, + "grad_norm": 6.239053192073585, + "learning_rate": 4.99108734402852e-07, + "logits/chosen": 0.38646355271339417, + "logits/rejected": 0.6254279613494873, + "logps/chosen": -9.66862964630127, + "logps/rejected": -10.103830337524414, + "loss": 0.0771, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -9.66862964630127, + "rewards/margins": 0.435200035572052, + "rewards/rejected": -10.103830337524414, + "semantic_entropy": 0.10753343254327774, + "step": 280 + }, + { + "epoch": 0.15253386853988962, + "grad_norm": 4.836784229202216, + "learning_rate": 5.080213903743315e-07, + "logits/chosen": 0.8410245180130005, + "logits/rejected": 1.0995079278945923, + "logps/chosen": -11.102112770080566, + "logps/rejected": -11.977313041687012, + "loss": 0.0248, + "rewards/accuracies": 0.5, + "rewards/chosen": -11.102112770080566, + "rewards/margins": 0.8752012252807617, + "rewards/rejected": -11.977313041687012, + "semantic_entropy": 0.03501223772764206, + "step": 285 + }, + { + "epoch": 0.1552099013212912, + "grad_norm": 0.9201296423061804, + "learning_rate": 5.169340463458111e-07, + "logits/chosen": 1.6455614566802979, + "logits/rejected": 1.8774665594100952, + "logps/chosen": -14.004234313964844, + "logps/rejected": -16.590394973754883, + "loss": 0.0067, + "rewards/accuracies": 0.606249988079071, + "rewards/chosen": -14.004234313964844, + "rewards/margins": 2.5861611366271973, + "rewards/rejected": -16.590394973754883, + "semantic_entropy": 0.01032390259206295, + "step": 290 + }, + { + "epoch": 0.15788593410269275, + "grad_norm": 1.4142135623730951, + "learning_rate": 5.258467023172905e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 1.0269, + "rewards/accuracies": 0.38749998807907104, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 295 + }, + { + "epoch": 0.16056196688409433, + "grad_norm": 1.4142135623730951, + "learning_rate": 5.347593582887701e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 300 + }, + { + "epoch": 0.1632379996654959, + "grad_norm": 1.4142135623730951, + "learning_rate": 5.436720142602496e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 305 + }, + { + "epoch": 0.16591403244689748, + "grad_norm": 1.4142135623730951, + "learning_rate": 5.52584670231729e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 310 + }, + { + "epoch": 0.16859006522829906, + "grad_norm": 1.4142135623730951, + "learning_rate": 5.614973262032086e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 315 + }, + { + "epoch": 0.1712660980097006, + "grad_norm": 1.4142135623730951, + "learning_rate": 5.70409982174688e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 320 + }, + { + "epoch": 0.17394213079110218, + "grad_norm": 1.4142135623730951, + "learning_rate": 5.793226381461676e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 325 + }, + { + "epoch": 0.17661816357250376, + "grad_norm": 1.4142135623730951, + "learning_rate": 5.88235294117647e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 330 + }, + { + "epoch": 0.17929419635390534, + "grad_norm": 1.4142135623730951, + "learning_rate": 5.971479500891266e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 335 + }, + { + "epoch": 0.18197022913530692, + "grad_norm": 1.4142135623730951, + "learning_rate": 6.060606060606061e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 340 + }, + { + "epoch": 0.1846462619167085, + "grad_norm": 1.4142135623730951, + "learning_rate": 6.149732620320855e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 345 + }, + { + "epoch": 0.18732229469811004, + "grad_norm": 1.4142135623730951, + "learning_rate": 6.238859180035651e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 350 + }, + { + "epoch": 0.18999832747951162, + "grad_norm": 1.4142135623730951, + "learning_rate": 6.327985739750445e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 355 + }, + { + "epoch": 0.1926743602609132, + "grad_norm": 1.4142135623730951, + "learning_rate": 6.417112299465241e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 360 + }, + { + "epoch": 0.19535039304231477, + "grad_norm": 1.4142135623730951, + "learning_rate": 6.506238859180035e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 365 + }, + { + "epoch": 0.19802642582371635, + "grad_norm": 1.4142135623730951, + "learning_rate": 6.59536541889483e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 370 + }, + { + "epoch": 0.2007024586051179, + "grad_norm": 1.4142135623730951, + "learning_rate": 6.684491978609626e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 375 + }, + { + "epoch": 0.20337849138651948, + "grad_norm": 1.4142135623730951, + "learning_rate": 6.77361853832442e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 380 + }, + { + "epoch": 0.20605452416792105, + "grad_norm": 1.4142135623730951, + "learning_rate": 6.862745098039216e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 385 + }, + { + "epoch": 0.20873055694932263, + "grad_norm": 1.4142135623730951, + "learning_rate": 6.95187165775401e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 390 + }, + { + "epoch": 0.2114065897307242, + "grad_norm": 1.4142135623730951, + "learning_rate": 7.040998217468806e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 395 + }, + { + "epoch": 0.2140826225121258, + "grad_norm": 1.4142135623730951, + "learning_rate": 7.1301247771836e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 400 + }, + { + "epoch": 0.2140826225121258, + "eval_logits/chosen": NaN, + "eval_logits/rejected": NaN, + "eval_logps/chosen": NaN, + "eval_logps/rejected": NaN, + "eval_loss": NaN, + "eval_rewards/accuracies": 0.0, + "eval_rewards/chosen": NaN, + "eval_rewards/margins": NaN, + "eval_rewards/rejected": NaN, + "eval_runtime": 36.0123, + "eval_samples_per_second": 37.348, + "eval_semantic_entropy": NaN, + "eval_steps_per_second": 9.358, + "step": 400 + }, + { + "epoch": 0.21675865529352734, + "grad_norm": 1.4142135623730951, + "learning_rate": 7.219251336898395e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 405 + }, + { + "epoch": 0.2194346880749289, + "grad_norm": 1.4142135623730951, + "learning_rate": 7.30837789661319e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 410 + }, + { + "epoch": 0.2221107208563305, + "grad_norm": 1.4142135623730951, + "learning_rate": 7.397504456327985e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 415 + }, + { + "epoch": 0.22478675363773207, + "grad_norm": 1.4142135623730951, + "learning_rate": 7.486631016042781e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 420 + }, + { + "epoch": 0.22746278641913364, + "grad_norm": 1.4142135623730951, + "learning_rate": 7.575757575757575e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 425 + }, + { + "epoch": 0.2301388192005352, + "grad_norm": 1.4142135623730951, + "learning_rate": 7.664884135472371e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 430 + }, + { + "epoch": 0.23281485198193677, + "grad_norm": 1.4142135623730951, + "learning_rate": 7.754010695187165e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 435 + }, + { + "epoch": 0.23549088476333835, + "grad_norm": 1.4142135623730951, + "learning_rate": 7.84313725490196e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 440 + }, + { + "epoch": 0.23816691754473993, + "grad_norm": 1.4142135623730951, + "learning_rate": 7.932263814616755e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 445 + }, + { + "epoch": 0.2408429503261415, + "grad_norm": 1.4142135623730951, + "learning_rate": 8.02139037433155e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 450 + }, + { + "epoch": 0.24351898310754308, + "grad_norm": 1.4142135623730951, + "learning_rate": 8.110516934046346e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 455 + }, + { + "epoch": 0.24619501588894463, + "grad_norm": 1.4142135623730951, + "learning_rate": 8.19964349376114e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 460 + }, + { + "epoch": 0.2488710486703462, + "grad_norm": 1.4142135623730951, + "learning_rate": 8.288770053475936e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 465 + }, + { + "epoch": 0.2515470814517478, + "grad_norm": 1.4142135623730951, + "learning_rate": 8.37789661319073e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 470 + }, + { + "epoch": 0.25422311423314936, + "grad_norm": 1.4142135623730951, + "learning_rate": 8.467023172905525e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 475 + }, + { + "epoch": 0.2568991470145509, + "grad_norm": 1.4142135623730951, + "learning_rate": 8.55614973262032e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 480 + }, + { + "epoch": 0.2595751797959525, + "grad_norm": 1.4142135623730951, + "learning_rate": 8.645276292335115e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 485 + }, + { + "epoch": 0.26225121257735406, + "grad_norm": 1.4142135623730951, + "learning_rate": 8.734402852049911e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 490 + }, + { + "epoch": 0.26492724535875567, + "grad_norm": 1.4142135623730951, + "learning_rate": 8.823529411764705e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 495 + }, + { + "epoch": 0.2676032781401572, + "grad_norm": 1.4142135623730951, + "learning_rate": 8.912655971479501e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 500 + }, + { + "epoch": 0.27027931092155877, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.001782531194295e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 505 + }, + { + "epoch": 0.2729553437029604, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.09090909090909e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 510 + }, + { + "epoch": 0.2756313764843619, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.180035650623885e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 515 + }, + { + "epoch": 0.27830740926576353, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.26916221033868e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 520 + }, + { + "epoch": 0.2809834420471651, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.358288770053476e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 525 + }, + { + "epoch": 0.2836594748285666, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.44741532976827e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 530 + }, + { + "epoch": 0.28633550760996823, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.536541889483066e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 535 + }, + { + "epoch": 0.2890115403913698, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.62566844919786e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 540 + }, + { + "epoch": 0.2916875731727714, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.714795008912655e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 545 + }, + { + "epoch": 0.29436360595417294, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.80392156862745e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 550 + }, + { + "epoch": 0.2970396387355745, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.893048128342244e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 555 + }, + { + "epoch": 0.2997156715169761, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.98217468805704e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 560 + }, + { + "epoch": 0.30239170429837764, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.999984476788462e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 565 + }, + { + "epoch": 0.30506773707977924, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.999921413906797e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 570 + }, + { + "epoch": 0.3077437698611808, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.999809841765644e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 575 + }, + { + "epoch": 0.3104198026425824, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.999649761447477e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 580 + }, + { + "epoch": 0.31309583542398395, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.999441174505398e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 585 + }, + { + "epoch": 0.3157718682053855, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.999184082963116e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 590 + }, + { + "epoch": 0.3184479009867871, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.998878489314937e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 595 + }, + { + "epoch": 0.32112393376818865, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.99852439652573e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 600 + }, + { + "epoch": 0.32379996654959026, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.998121808030904e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 605 + }, + { + "epoch": 0.3264759993309918, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.997670727736379e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 610 + }, + { + "epoch": 0.32915203211239336, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.99717116001853e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 615 + }, + { + "epoch": 0.33182806489379496, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.996623109724173e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 620 + }, + { + "epoch": 0.3345040976751965, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.996026582170488e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 625 + }, + { + "epoch": 0.3371801304565981, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.995381583144996e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 630 + }, + { + "epoch": 0.33985616323799966, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.994688118905471e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 635 + }, + { + "epoch": 0.3425321960194012, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.993946196179912e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 640 + }, + { + "epoch": 0.3452082288008028, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.993155822166455e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 645 + }, + { + "epoch": 0.34788426158220437, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.992317004533313e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 650 + }, + { + "epoch": 0.350560294363606, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.991429751418696e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 655 + }, + { + "epoch": 0.3532363271450075, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.99049407143074e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 660 + }, + { + "epoch": 0.35591235992640907, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.989509973647416e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 665 + }, + { + "epoch": 0.3585883927078107, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.988477467616445e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 670 + }, + { + "epoch": 0.3612644254892122, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.987396563355205e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 675 + }, + { + "epoch": 0.36394045827061383, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.986267271350631e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 680 + }, + { + "epoch": 0.3666164910520154, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.985089602559123e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 685 + }, + { + "epoch": 0.369292523833417, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.983863568406428e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 690 + }, + { + "epoch": 0.37196855661481854, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.982589180787532e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 695 + }, + { + "epoch": 0.3746445893962201, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.981266452066553e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 700 + }, + { + "epoch": 0.3773206221776217, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.979895395076608e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 705 + }, + { + "epoch": 0.37999665495902324, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.9784760231197e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 710 + }, + { + "epoch": 0.38267268774042484, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.97700834996658e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 715 + }, + { + "epoch": 0.3853487205218264, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.97549238985662e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 720 + }, + { + "epoch": 0.38802475330322794, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.973928157497674e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 725 + }, + { + "epoch": 0.39070078608462955, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.972315668065927e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 730 + }, + { + "epoch": 0.3933768188660311, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.97065493720576e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 735 + }, + { + "epoch": 0.3960528516474327, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.968945981029594e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 740 + }, + { + "epoch": 0.39872888442883425, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.967188816117726e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 745 + }, + { + "epoch": 0.4014049172102358, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.965383459518179e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 750 + }, + { + "epoch": 0.4040809499916374, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.963529928746533e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 755 + }, + { + "epoch": 0.40675698277303896, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.961628241785746e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 760 + }, + { + "epoch": 0.40943301555444056, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.959678417085998e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 765 + }, + { + "epoch": 0.4121090483358421, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.957680473564493e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 770 + }, + { + "epoch": 0.41478508111724366, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.95563443060529e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 775 + }, + { + "epoch": 0.41746111389864526, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.95354030805911e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 780 + }, + { + "epoch": 0.4201371466800468, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.951398126243133e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 785 + }, + { + "epoch": 0.4228131794614484, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.94920790594082e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 790 + }, + { + "epoch": 0.42548921224284997, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.946969668401696e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 795 + }, + { + "epoch": 0.4281652450242516, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.944683435341155e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 800 + }, + { + "epoch": 0.4281652450242516, + "eval_logits/chosen": NaN, + "eval_logits/rejected": NaN, + "eval_logps/chosen": NaN, + "eval_logps/rejected": NaN, + "eval_loss": NaN, + "eval_rewards/accuracies": 0.0, + "eval_rewards/chosen": NaN, + "eval_rewards/margins": NaN, + "eval_rewards/rejected": NaN, + "eval_runtime": 35.0379, + "eval_samples_per_second": 38.387, + "eval_semantic_entropy": NaN, + "eval_steps_per_second": 9.618, + "step": 800 + }, + { + "epoch": 0.4308412778056531, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.942349228940236e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 805 + }, + { + "epoch": 0.43351731058705467, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.939967071845424e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 810 + }, + { + "epoch": 0.4361933433684563, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.937536987168413e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 815 + }, + { + "epoch": 0.4388693761498578, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.935058998485896e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 820 + }, + { + "epoch": 0.44154540893125943, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.932533129839333e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 825 + }, + { + "epoch": 0.444221441712661, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.929959405734711e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 830 + }, + { + "epoch": 0.44689747449406253, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.927337851142314e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 835 + }, + { + "epoch": 0.44957350727546413, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.924668491496474e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 840 + }, + { + "epoch": 0.4522495400568657, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.92195135269533e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 845 + }, + { + "epoch": 0.4549255728382673, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.919186461100574e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 850 + }, + { + "epoch": 0.45760160561966884, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.9163738435372e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 855 + }, + { + "epoch": 0.4602776384010704, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.913513527293234e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 860 + }, + { + "epoch": 0.462953671182472, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.910605540119474e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 865 + }, + { + "epoch": 0.46562970396387354, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.907649910229227e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 870 + }, + { + "epoch": 0.46830573674527515, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.90464666629803e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 875 + }, + { + "epoch": 0.4709817695266767, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.901595837463363e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 880 + }, + { + "epoch": 0.47365780230807825, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.898497453324384e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 885 + }, + { + "epoch": 0.47633383508947985, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.895351543941628e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 890 + }, + { + "epoch": 0.4790098678708814, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.892158139836724e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 895 + }, + { + "epoch": 0.481685900652283, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.88891727199209e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 900 + }, + { + "epoch": 0.48436193343368455, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.885628971850641e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 905 + }, + { + "epoch": 0.48703796621508616, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.882293271315481e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 910 + }, + { + "epoch": 0.4897139989964877, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.878910202749589e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 915 + }, + { + "epoch": 0.49239003177788926, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.875479798975512e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 920 + }, + { + "epoch": 0.49506606455929086, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.87200209327504e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 925 + }, + { + "epoch": 0.4977420973406924, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.868477119388894e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 930 + }, + { + "epoch": 0.500418130122094, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.864904911516383e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 935 + }, + { + "epoch": 0.5030941629034956, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.861285504315084e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 940 + }, + { + "epoch": 0.5057701956848971, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.857618932900502e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 945 + }, + { + "epoch": 0.5084462284662987, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.853905232845727e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 950 + }, + { + "epoch": 0.5111222612477003, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.850144440181095e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 955 + }, + { + "epoch": 0.5137982940291018, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.846336591393832e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 960 + }, + { + "epoch": 0.5164743268105034, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.842481723427704e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 965 + }, + { + "epoch": 0.519150359591905, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.838579873682658e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 970 + }, + { + "epoch": 0.5218263923733065, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.834631080014457e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 975 + }, + { + "epoch": 0.5245024251547081, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.830635380734312e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 980 + }, + { + "epoch": 0.5271784579361097, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.826592814608517e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 985 + }, + { + "epoch": 0.5298544907175113, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.822503420858067e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 990 + }, + { + "epoch": 0.5325305234989128, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.818367239158277e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 995 + }, + { + "epoch": 0.5352065562803144, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.8141843096384e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 1000 + }, + { + "epoch": 0.537882589061716, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.809954672881237e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 1005 + }, + { + "epoch": 0.5405586218431175, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.80567836992274e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 1010 + }, + { + "epoch": 0.5432346546245191, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.801355442251625e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 1015 + }, + { + "epoch": 0.5459106874059207, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.796985931808949e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 1020 + }, + { + "epoch": 0.5485867201873222, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.792569880987724e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 1025 + }, + { + "epoch": 0.5512627529687238, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.788107332632493e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 1030 + }, + { + "epoch": 0.5539387857501255, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.783598330038924e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 1035 + }, + { + "epoch": 0.5566148185315271, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.779042916953376e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 1040 + }, + { + "epoch": 0.5592908513129285, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.774441137572487e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 1045 + }, + { + "epoch": 0.5619668840943302, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.76979303654274e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 1050 + }, + { + "epoch": 0.5646429168757318, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.765098658960035e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 1055 + }, + { + "epoch": 0.5673189496571333, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.76035805036924e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 1060 + }, + { + "epoch": 0.5699949824385349, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.755571256763764e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 1065 + }, + { + "epoch": 0.5726710152199365, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.750738324585097e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 1070 + }, + { + "epoch": 0.5753470480013381, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.74585930072237e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 1075 + }, + { + "epoch": 0.5780230807827396, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.740934232511892e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 1080 + }, + { + "epoch": 0.5806991135641412, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.735963167736698e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 1085 + }, + { + "epoch": 0.5833751463455428, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.730946154626078e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 1090 + }, + { + "epoch": 0.5860511791269443, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.725883241855117e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 1095 + }, + { + "epoch": 0.5887272119083459, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.720774478544218e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 1100 + }, + { + "epoch": 0.5914032446897475, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.715619914258624e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 1105 + }, + { + "epoch": 0.594079277471149, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.710419599007937e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 1110 + }, + { + "epoch": 0.5967553102525506, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.705173583245643e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 1115 + }, + { + "epoch": 0.5994313430339522, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.699881917868609e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 1120 + }, + { + "epoch": 0.6021073758153538, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.694544654216594e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 1125 + }, + { + "epoch": 0.6047834085967553, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.689161844071755e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 1130 + }, + { + "epoch": 0.6074594413781569, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.683733539658138e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 1135 + }, + { + "epoch": 0.6101354741595585, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.678259793641178e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 1140 + }, + { + "epoch": 0.61281150694096, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.672740659127183e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 1145 + }, + { + "epoch": 0.6154875397223616, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.667176189662818e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 1150 + }, + { + "epoch": 0.6181635725037632, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.661566439234592e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 1155 + }, + { + "epoch": 0.6208396052851648, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.655911462268327e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 1160 + }, + { + "epoch": 0.6235156380665663, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.650211313628636e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 1165 + }, + { + "epoch": 0.6261916708479679, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.644466048618386e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 1170 + }, + { + "epoch": 0.6288677036293695, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.63867572297816e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 1175 + }, + { + "epoch": 0.631543736410771, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.632840392885727e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 1180 + }, + { + "epoch": 0.6342197691921726, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.626960114955483e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 1185 + }, + { + "epoch": 0.6368958019735742, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.621034946237909e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 1190 + }, + { + "epoch": 0.6395718347549757, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.615064944219021e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 1195 + }, + { + "epoch": 0.6422478675363773, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.609050166819803e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 1200 + }, + { + "epoch": 0.6422478675363773, + "eval_logits/chosen": NaN, + "eval_logits/rejected": NaN, + "eval_logps/chosen": NaN, + "eval_logps/rejected": NaN, + "eval_loss": NaN, + "eval_rewards/accuracies": 0.0, + "eval_rewards/chosen": NaN, + "eval_rewards/margins": NaN, + "eval_rewards/rejected": NaN, + "eval_runtime": 35.3915, + "eval_samples_per_second": 38.004, + "eval_semantic_entropy": NaN, + "eval_steps_per_second": 9.522, + "step": 1200 + }, + { + "epoch": 0.6449239003177789, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.602990672395653e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 1205 + }, + { + "epoch": 0.6475999330991805, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.59688651973581e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 1210 + }, + { + "epoch": 0.650275965880582, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.590737768062792e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 1215 + }, + { + "epoch": 0.6529519986619836, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.584544477031816e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 1220 + }, + { + "epoch": 0.6556280314433852, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.578306706730215e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 1225 + }, + { + "epoch": 0.6583040642247867, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.572024517676865e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 1230 + }, + { + "epoch": 0.6609800970061883, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.565697970821593e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 1235 + }, + { + "epoch": 0.6636561297875899, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.559327127544585e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 1240 + }, + { + "epoch": 0.6663321625689914, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.552912049655789e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 1245 + }, + { + "epoch": 0.669008195350393, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.546452799394315e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 1250 + }, + { + "epoch": 0.6716842281317946, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.539949439427846e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 1255 + }, + { + "epoch": 0.6743602609131962, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.533402032852002e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 1260 + }, + { + "epoch": 0.6770362936945977, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.526810643189754e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 1265 + }, + { + "epoch": 0.6797123264759993, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.52017533439079e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 1270 + }, + { + "epoch": 0.6823883592574009, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.513496170830909e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 1275 + }, + { + "epoch": 0.6850643920388024, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.506773217311382e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 1280 + }, + { + "epoch": 0.687740424820204, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.500006539058334e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 1285 + }, + { + "epoch": 0.6904164576016056, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.493196201722109e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 1290 + }, + { + "epoch": 0.6930924903830072, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.486342271376628e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 1295 + }, + { + "epoch": 0.6957685231644087, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.479444814518755e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 1300 + }, + { + "epoch": 0.6984445559458103, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.472503898067645e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 1305 + }, + { + "epoch": 0.701120588727212, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.465519589364099e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 1310 + }, + { + "epoch": 0.7037966215086134, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.458491956169914e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 1315 + }, + { + "epoch": 0.706472654290015, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.451421066667215e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 1320 + }, + { + "epoch": 0.7091486870714167, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.444306989457805e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 1325 + }, + { + "epoch": 0.7118247198528181, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.437149793562489e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 1330 + }, + { + "epoch": 0.7145007526342197, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.429949548420417e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 1335 + }, + { + "epoch": 0.7171767854156214, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.422706323888396e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 1340 + }, + { + "epoch": 0.719852818197023, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.415420190240225e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 1345 + }, + { + "epoch": 0.7225288509784245, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.408091218166002e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 1350 + }, + { + "epoch": 0.7252048837598261, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.400719478771449e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 1355 + }, + { + "epoch": 0.7278809165412277, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.393305043577209e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 1360 + }, + { + "epoch": 0.7305569493226292, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.38584798451817e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 1365 + }, + { + "epoch": 0.7332329821040308, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.37834837394275e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 1370 + }, + { + "epoch": 0.7359090148854324, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.370806284612203e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 1375 + }, + { + "epoch": 0.738585047666834, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.363221789699912e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 1380 + }, + { + "epoch": 0.7412610804482355, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.355594962790682e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 1385 + }, + { + "epoch": 0.7439371132296371, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.34792587788002e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 1390 + }, + { + "epoch": 0.7466131460110387, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.34021460937342e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 1395 + }, + { + "epoch": 0.7492891787924402, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.332461232085646e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 1400 + }, + { + "epoch": 0.7519652115738418, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.324665821239998e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 1405 + }, + { + "epoch": 0.7546412443552434, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.316828452467583e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 1410 + }, + { + "epoch": 0.7573172771366449, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.30894920180659e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 1415 + }, + { + "epoch": 0.7599933099180465, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.301028145701543e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 1420 + }, + { + "epoch": 0.7626693426994481, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.293065361002563e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 1425 + }, + { + "epoch": 0.7653453754808497, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.285060924964622e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 1430 + }, + { + "epoch": 0.7680214082622512, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.277014915246792e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 1435 + }, + { + "epoch": 0.7706974410436528, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.268927409911498e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 1440 + }, + { + "epoch": 0.7733734738250544, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.260798487423749e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 1445 + }, + { + "epoch": 0.7760495066064559, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.252628226650389e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 1450 + }, + { + "epoch": 0.7787255393878575, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.244416706859321e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 1455 + }, + { + "epoch": 0.7814015721692591, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.23616400771875e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 1460 + }, + { + "epoch": 0.7840776049506607, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.227870209296395e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 1465 + }, + { + "epoch": 0.7867536377320622, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.219535392058728e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 1470 + }, + { + "epoch": 0.7894296705134638, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.211159636870181e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 1475 + }, + { + "epoch": 0.7921057032948654, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.202743024992367e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 1480 + }, + { + "epoch": 0.7947817360762669, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.194285638083293e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 1485 + }, + { + "epoch": 0.7974577688576685, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.185787558196562e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 1490 + }, + { + "epoch": 0.8001338016390701, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.177248867780583e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 1495 + }, + { + "epoch": 0.8028098344204716, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.168669649677769e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 1500 + }, + { + "epoch": 0.8054858672018732, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.16004998712373e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 1505 + }, + { + "epoch": 0.8081618999832748, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.151389963746472e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 1510 + }, + { + "epoch": 0.8108379327646764, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.142689663565577e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 1515 + }, + { + "epoch": 0.8135139655460779, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.133949170991397e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 1520 + }, + { + "epoch": 0.8161899983274795, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.125168570824231e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 1525 + }, + { + "epoch": 0.8188660311088811, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.116347948253496e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 1530 + }, + { + "epoch": 0.8215420638902826, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.107487388856916e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 1535 + }, + { + "epoch": 0.8242180966716842, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.098586978599673e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 1540 + }, + { + "epoch": 0.8268941294530858, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.089646803833588e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 1545 + }, + { + "epoch": 0.8295701622344873, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.080666951296276e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 1550 + }, + { + "epoch": 0.8322461950158889, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.071647508110305e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 1555 + }, + { + "epoch": 0.8349222277972905, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.062588561782354e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 1560 + }, + { + "epoch": 0.8375982605786921, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.053490200202358e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 1565 + }, + { + "epoch": 0.8402742933600936, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.044352511642661e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 1570 + }, + { + "epoch": 0.8429503261414952, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.03517558475716e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 1575 + }, + { + "epoch": 0.8456263589228968, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.025959508580436e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 1580 + }, + { + "epoch": 0.8483023917042983, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.016704372526905e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 1585 + }, + { + "epoch": 0.8509784244856999, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.007410266389934e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 1590 + }, + { + "epoch": 0.8536544572671015, + "grad_norm": 1.4142135623730951, + "learning_rate": 8.998077280340981e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 1595 + }, + { + "epoch": 0.8563304900485031, + "grad_norm": 1.4142135623730951, + "learning_rate": 8.988705504928722e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 1600 + }, + { + "epoch": 0.8563304900485031, + "eval_logits/chosen": NaN, + "eval_logits/rejected": NaN, + "eval_logps/chosen": NaN, + "eval_logps/rejected": NaN, + "eval_loss": NaN, + "eval_rewards/accuracies": 0.0, + "eval_rewards/chosen": NaN, + "eval_rewards/margins": NaN, + "eval_rewards/rejected": NaN, + "eval_runtime": 34.8813, + "eval_samples_per_second": 38.559, + "eval_semantic_entropy": NaN, + "eval_steps_per_second": 9.661, + "step": 1600 + }, + { + "epoch": 0.8590065228299046, + "grad_norm": 1.4142135623730951, + "learning_rate": 8.979295031078157e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 1605 + }, + { + "epoch": 0.8616825556113062, + "grad_norm": 1.4142135623730951, + "learning_rate": 8.969845950089751e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 1610 + }, + { + "epoch": 0.8643585883927078, + "grad_norm": 1.4142135623730951, + "learning_rate": 8.960358353638526e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 1615 + }, + { + "epoch": 0.8670346211741093, + "grad_norm": 1.4142135623730951, + "learning_rate": 8.950832333773184e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 1620 + }, + { + "epoch": 0.869710653955511, + "grad_norm": 1.4142135623730951, + "learning_rate": 8.941267982915213e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 1625 + }, + { + "epoch": 0.8723866867369126, + "grad_norm": 1.4142135623730951, + "learning_rate": 8.931665393857983e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 1630 + }, + { + "epoch": 0.875062719518314, + "grad_norm": 1.4142135623730951, + "learning_rate": 8.922024659765861e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 1635 + }, + { + "epoch": 0.8777387522997157, + "grad_norm": 1.4142135623730951, + "learning_rate": 8.912345874173288e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 1640 + }, + { + "epoch": 0.8804147850811173, + "grad_norm": 1.4142135623730951, + "learning_rate": 8.902629130983885e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 1645 + }, + { + "epoch": 0.8830908178625189, + "grad_norm": 1.4142135623730951, + "learning_rate": 8.892874524469537e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 1650 + }, + { + "epoch": 0.8857668506439204, + "grad_norm": 1.4142135623730951, + "learning_rate": 8.883082149269478e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 1655 + }, + { + "epoch": 0.888442883425322, + "grad_norm": 1.4142135623730951, + "learning_rate": 8.873252100389377e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 1660 + }, + { + "epoch": 0.8911189162067236, + "grad_norm": 1.4142135623730951, + "learning_rate": 8.863384473200411e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 1665 + }, + { + "epoch": 0.8937949489881251, + "grad_norm": 1.4142135623730951, + "learning_rate": 8.853479363438342e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 1670 + }, + { + "epoch": 0.8964709817695267, + "grad_norm": 1.4142135623730951, + "learning_rate": 8.843536867202588e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 1675 + }, + { + "epoch": 0.8991470145509283, + "grad_norm": 1.4142135623730951, + "learning_rate": 8.833557080955292e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 1680 + }, + { + "epoch": 0.9018230473323299, + "grad_norm": 1.4142135623730951, + "learning_rate": 8.823540101520381e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 1685 + }, + { + "epoch": 0.9044990801137314, + "grad_norm": 1.4142135623730951, + "learning_rate": 8.813486026082637e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 1690 + }, + { + "epoch": 0.907175112895133, + "grad_norm": 1.4142135623730951, + "learning_rate": 8.803394952186742e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 1695 + }, + { + "epoch": 0.9098511456765346, + "grad_norm": 1.4142135623730951, + "learning_rate": 8.793266977736342e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 1700 + }, + { + "epoch": 0.9125271784579361, + "grad_norm": 1.4142135623730951, + "learning_rate": 8.783102200993085e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 1705 + }, + { + "epoch": 0.9152032112393377, + "grad_norm": 1.4142135623730951, + "learning_rate": 8.772900720575683e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 1710 + }, + { + "epoch": 0.9178792440207393, + "grad_norm": 1.4142135623730951, + "learning_rate": 8.762662635458944e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 1715 + }, + { + "epoch": 0.9205552768021408, + "grad_norm": 1.4142135623730951, + "learning_rate": 8.752388044972811e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 1720 + }, + { + "epoch": 0.9232313095835424, + "grad_norm": 1.4142135623730951, + "learning_rate": 8.74207704880141e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 1725 + }, + { + "epoch": 0.925907342364944, + "grad_norm": 1.4142135623730951, + "learning_rate": 8.731729746982068e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 1730 + }, + { + "epoch": 0.9285833751463456, + "grad_norm": 1.4142135623730951, + "learning_rate": 8.721346239904355e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 1735 + }, + { + "epoch": 0.9312594079277471, + "grad_norm": 1.4142135623730951, + "learning_rate": 8.710926628309101e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 1740 + }, + { + "epoch": 0.9339354407091487, + "grad_norm": 1.4142135623730951, + "learning_rate": 8.700471013287424e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 1745 + }, + { + "epoch": 0.9366114734905503, + "grad_norm": 1.4142135623730951, + "learning_rate": 8.689979496279746e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 1750 + }, + { + "epoch": 0.9392875062719518, + "grad_norm": 1.4142135623730951, + "learning_rate": 8.679452179074811e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 1755 + }, + { + "epoch": 0.9419635390533534, + "grad_norm": 1.4142135623730951, + "learning_rate": 8.668889163808698e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 1760 + }, + { + "epoch": 0.944639571834755, + "grad_norm": 1.4142135623730951, + "learning_rate": 8.658290552963827e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 1765 + }, + { + "epoch": 0.9473156046161565, + "grad_norm": 1.4142135623730951, + "learning_rate": 8.647656449367966e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 1770 + }, + { + "epoch": 0.9499916373975581, + "grad_norm": 1.4142135623730951, + "learning_rate": 8.636986956193235e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 1775 + }, + { + "epoch": 0.9526676701789597, + "grad_norm": 1.4142135623730951, + "learning_rate": 8.626282176955104e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 1780 + }, + { + "epoch": 0.9553437029603613, + "grad_norm": 1.4142135623730951, + "learning_rate": 8.615542215511389e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 1785 + }, + { + "epoch": 0.9580197357417628, + "grad_norm": 1.4142135623730951, + "learning_rate": 8.604767176061241e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 1790 + }, + { + "epoch": 0.9606957685231644, + "grad_norm": 1.4142135623730951, + "learning_rate": 8.593957163144141e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 1795 + }, + { + "epoch": 0.963371801304566, + "grad_norm": 1.4142135623730951, + "learning_rate": 8.58311228163888e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 1800 + }, + { + "epoch": 0.9660478340859675, + "grad_norm": 1.4142135623730951, + "learning_rate": 8.57223263676255e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 1805 + }, + { + "epoch": 0.9687238668673691, + "grad_norm": 1.4142135623730951, + "learning_rate": 8.561318334069511e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 1810 + }, + { + "epoch": 0.9713998996487707, + "grad_norm": 1.4142135623730951, + "learning_rate": 8.550369479450375e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 1815 + }, + { + "epoch": 0.9740759324301723, + "grad_norm": 1.4142135623730951, + "learning_rate": 8.539386179130977e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 1820 + }, + { + "epoch": 0.9767519652115738, + "grad_norm": 1.4142135623730951, + "learning_rate": 8.528368539671347e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 1825 + }, + { + "epoch": 0.9794279979929754, + "grad_norm": 1.4142135623730951, + "learning_rate": 8.51731666796467e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 1830 + }, + { + "epoch": 0.982104030774377, + "grad_norm": 1.4142135623730951, + "learning_rate": 8.506230671236254e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 1835 + }, + { + "epoch": 0.9847800635557785, + "grad_norm": 1.4142135623730951, + "learning_rate": 8.495110657042488e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 1840 + }, + { + "epoch": 0.9874560963371801, + "grad_norm": 1.4142135623730951, + "learning_rate": 8.483956733269799e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 1845 + }, + { + "epoch": 0.9901321291185817, + "grad_norm": 1.4142135623730951, + "learning_rate": 8.472769008133602e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 1850 + }, + { + "epoch": 0.9928081618999832, + "grad_norm": 1.4142135623730951, + "learning_rate": 8.461547590177259e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 1855 + }, + { + "epoch": 0.9954841946813848, + "grad_norm": 1.4142135623730951, + "learning_rate": 8.450292588271014e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 1860 + }, + { + "epoch": 0.9981602274627864, + "grad_norm": 1.4142135623730951, + "learning_rate": 8.439004111610945e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 1865 + }, + { + "epoch": 1.000836260244188, + "grad_norm": 1.4142135623730951, + "learning_rate": 8.427682269717901e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 1870 + }, + { + "epoch": 1.0035122930255895, + "grad_norm": 1.4142135623730951, + "learning_rate": 8.416327172436446e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 1875 + }, + { + "epoch": 1.0061883258069912, + "grad_norm": 1.4142135623730951, + "learning_rate": 8.404938929933778e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 1880 + }, + { + "epoch": 1.0088643585883927, + "grad_norm": 1.4142135623730951, + "learning_rate": 8.39351765269868e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 1885 + }, + { + "epoch": 1.0115403913697942, + "grad_norm": 1.4142135623730951, + "learning_rate": 8.382063451540431e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 1890 + }, + { + "epoch": 1.014216424151196, + "grad_norm": 1.4142135623730951, + "learning_rate": 8.370576437587742e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 1895 + }, + { + "epoch": 1.0168924569325974, + "grad_norm": 1.4142135623730951, + "learning_rate": 8.359056722287674e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 1900 + }, + { + "epoch": 1.019568489713999, + "grad_norm": 1.4142135623730951, + "learning_rate": 8.347504417404553e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 1905 + }, + { + "epoch": 1.0222445224954007, + "grad_norm": 1.4142135623730951, + "learning_rate": 8.335919635018893e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 1910 + }, + { + "epoch": 1.0249205552768021, + "grad_norm": 1.4142135623730951, + "learning_rate": 8.324302487526303e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 1915 + }, + { + "epoch": 1.0275965880582036, + "grad_norm": 1.4142135623730951, + "learning_rate": 8.312653087636398e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 1920 + }, + { + "epoch": 1.0302726208396054, + "grad_norm": 1.4142135623730951, + "learning_rate": 8.300971548371711e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 1925 + }, + { + "epoch": 1.0329486536210069, + "grad_norm": 1.4142135623730951, + "learning_rate": 8.289257983066582e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 1930 + }, + { + "epoch": 1.0356246864024083, + "grad_norm": 1.4142135623730951, + "learning_rate": 8.277512505366077e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 1935 + }, + { + "epoch": 1.03830071918381, + "grad_norm": 1.4142135623730951, + "learning_rate": 8.265735229224868e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 1940 + }, + { + "epoch": 1.0409767519652116, + "grad_norm": 1.4142135623730951, + "learning_rate": 8.253926268906144e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 1945 + }, + { + "epoch": 1.043652784746613, + "grad_norm": 1.4142135623730951, + "learning_rate": 8.242085738980487e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 1950 + }, + { + "epoch": 1.0463288175280148, + "grad_norm": 1.4142135623730951, + "learning_rate": 8.230213754324772e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 1955 + }, + { + "epoch": 1.0490048503094163, + "grad_norm": 1.4142135623730951, + "learning_rate": 8.218310430121045e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 1960 + }, + { + "epoch": 1.051680883090818, + "grad_norm": 1.4142135623730951, + "learning_rate": 8.20637588185541e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 1965 + }, + { + "epoch": 1.0543569158722195, + "grad_norm": 1.4142135623730951, + "learning_rate": 8.194410225316906e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 1970 + }, + { + "epoch": 1.057032948653621, + "grad_norm": 1.4142135623730951, + "learning_rate": 8.182413576596385e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 1975 + }, + { + "epoch": 1.0597089814350227, + "grad_norm": 1.4142135623730951, + "learning_rate": 8.170386052085389e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 1980 + }, + { + "epoch": 1.0623850142164242, + "grad_norm": 1.4142135623730951, + "learning_rate": 8.158327768475008e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 1985 + }, + { + "epoch": 1.0650610469978257, + "grad_norm": 1.4142135623730951, + "learning_rate": 8.146238842754767e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 1990 + }, + { + "epoch": 1.0677370797792274, + "grad_norm": 1.4142135623730951, + "learning_rate": 8.134119392211476e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 1995 + }, + { + "epoch": 1.0704131125606289, + "grad_norm": 1.4142135623730951, + "learning_rate": 8.121969534428094e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 2000 + }, + { + "epoch": 1.0704131125606289, + "eval_logits/chosen": NaN, + "eval_logits/rejected": NaN, + "eval_logps/chosen": NaN, + "eval_logps/rejected": NaN, + "eval_loss": NaN, + "eval_rewards/accuracies": 0.0, + "eval_rewards/chosen": NaN, + "eval_rewards/margins": NaN, + "eval_rewards/rejected": NaN, + "eval_runtime": 34.7555, + "eval_samples_per_second": 38.699, + "eval_semantic_entropy": NaN, + "eval_steps_per_second": 9.696, + "step": 2000 + }, + { + "epoch": 1.0730891453420304, + "grad_norm": 1.4142135623730951, + "learning_rate": 8.109789387282599e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 2005 + }, + { + "epoch": 1.075765178123432, + "grad_norm": 1.4142135623730951, + "learning_rate": 8.097579068946827e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 2010 + }, + { + "epoch": 1.0784412109048336, + "grad_norm": 1.4142135623730951, + "learning_rate": 8.085338697885344e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 2015 + }, + { + "epoch": 1.081117243686235, + "grad_norm": 1.4142135623730951, + "learning_rate": 8.073068392854282e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 2020 + }, + { + "epoch": 1.0837932764676368, + "grad_norm": 1.4142135623730951, + "learning_rate": 8.060768272900193e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 2025 + }, + { + "epoch": 1.0864693092490383, + "grad_norm": 1.4142135623730951, + "learning_rate": 8.0484384573589e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 2030 + }, + { + "epoch": 1.0891453420304398, + "grad_norm": 1.4142135623730951, + "learning_rate": 8.03607906585432e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 2035 + }, + { + "epoch": 1.0918213748118415, + "grad_norm": 1.4142135623730951, + "learning_rate": 8.023690218297329e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 2040 + }, + { + "epoch": 1.094497407593243, + "grad_norm": 1.4142135623730951, + "learning_rate": 8.01127203488458e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 2045 + }, + { + "epoch": 1.0971734403746445, + "grad_norm": 1.4142135623730951, + "learning_rate": 7.998824636097339e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 2050 + }, + { + "epoch": 1.0998494731560462, + "grad_norm": 1.4142135623730951, + "learning_rate": 7.986348142700328e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 2055 + }, + { + "epoch": 1.1025255059374477, + "grad_norm": 1.4142135623730951, + "learning_rate": 7.973842675740539e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 2060 + }, + { + "epoch": 1.1052015387188494, + "grad_norm": 1.4142135623730951, + "learning_rate": 7.961308356546066e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 2065 + }, + { + "epoch": 1.107877571500251, + "grad_norm": 1.4142135623730951, + "learning_rate": 7.948745306724931e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 2070 + }, + { + "epoch": 1.1105536042816524, + "grad_norm": 1.4142135623730951, + "learning_rate": 7.936153648163897e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 2075 + }, + { + "epoch": 1.1132296370630541, + "grad_norm": 1.4142135623730951, + "learning_rate": 7.92353350302729e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 2080 + }, + { + "epoch": 1.1159056698444556, + "grad_norm": 1.4142135623730951, + "learning_rate": 7.910884993755816e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 2085 + }, + { + "epoch": 1.118581702625857, + "grad_norm": 1.4142135623730951, + "learning_rate": 7.898208243065367e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 2090 + }, + { + "epoch": 1.1212577354072588, + "grad_norm": 1.4142135623730951, + "learning_rate": 7.88550337394583e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 2095 + }, + { + "epoch": 1.1239337681886603, + "grad_norm": 1.4142135623730951, + "learning_rate": 7.872770509659905e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 2100 + }, + { + "epoch": 1.1266098009700618, + "grad_norm": 1.4142135623730951, + "learning_rate": 7.860009773741896e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 2105 + }, + { + "epoch": 1.1292858337514635, + "grad_norm": 1.4142135623730951, + "learning_rate": 7.84722128999652e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 2110 + }, + { + "epoch": 1.131961866532865, + "grad_norm": 1.4142135623730951, + "learning_rate": 7.834405182497699e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 2115 + }, + { + "epoch": 1.1346378993142665, + "grad_norm": 1.4142135623730951, + "learning_rate": 7.821561575587368e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 2120 + }, + { + "epoch": 1.1373139320956682, + "grad_norm": 1.4142135623730951, + "learning_rate": 7.808690593874254e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 2125 + }, + { + "epoch": 1.1399899648770697, + "grad_norm": 1.4142135623730951, + "learning_rate": 7.79579236223268e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 2130 + }, + { + "epoch": 1.1426659976584714, + "grad_norm": 1.4142135623730951, + "learning_rate": 7.782867005801346e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 2135 + }, + { + "epoch": 1.145342030439873, + "grad_norm": 1.4142135623730951, + "learning_rate": 7.769914649982117e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 2140 + }, + { + "epoch": 1.1480180632212744, + "grad_norm": 1.4142135623730951, + "learning_rate": 7.756935420438803e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 2145 + }, + { + "epoch": 1.1506940960026761, + "grad_norm": 1.4142135623730951, + "learning_rate": 7.743929443095951e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 2150 + }, + { + "epoch": 1.1533701287840776, + "grad_norm": 1.4142135623730951, + "learning_rate": 7.730896844137609e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 2155 + }, + { + "epoch": 1.1560461615654791, + "grad_norm": 1.4142135623730951, + "learning_rate": 7.717837750006106e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 2160 + }, + { + "epoch": 1.1587221943468808, + "grad_norm": 1.4142135623730951, + "learning_rate": 7.704752287400832e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 2165 + }, + { + "epoch": 1.1613982271282823, + "grad_norm": 1.4142135623730951, + "learning_rate": 7.691640583277004e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 2170 + }, + { + "epoch": 1.1640742599096838, + "grad_norm": 1.4142135623730951, + "learning_rate": 7.678502764844433e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 2175 + }, + { + "epoch": 1.1667502926910855, + "grad_norm": 1.4142135623730951, + "learning_rate": 7.665338959566288e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 2180 + }, + { + "epoch": 1.169426325472487, + "grad_norm": 1.4142135623730951, + "learning_rate": 7.652149295157868e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 2185 + }, + { + "epoch": 1.1721023582538885, + "grad_norm": 1.4142135623730951, + "learning_rate": 7.638933899585354e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 2190 + }, + { + "epoch": 1.1747783910352902, + "grad_norm": 1.4142135623730951, + "learning_rate": 7.625692901064573e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 2195 + }, + { + "epoch": 1.1774544238166917, + "grad_norm": 1.4142135623730951, + "learning_rate": 7.61242642805975e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 2200 + }, + { + "epoch": 1.1801304565980932, + "grad_norm": 1.4142135623730951, + "learning_rate": 7.599134609282266e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 2205 + }, + { + "epoch": 1.182806489379495, + "grad_norm": 1.4142135623730951, + "learning_rate": 7.585817573689402e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 2210 + }, + { + "epoch": 1.1854825221608964, + "grad_norm": 1.4142135623730951, + "learning_rate": 7.572475450483098e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 2215 + }, + { + "epoch": 1.188158554942298, + "grad_norm": 1.4142135623730951, + "learning_rate": 7.559108369108689e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 2220 + }, + { + "epoch": 1.1908345877236997, + "grad_norm": 1.4142135623730951, + "learning_rate": 7.54571645925366e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 2225 + }, + { + "epoch": 1.1935106205051011, + "grad_norm": 1.4142135623730951, + "learning_rate": 7.532299850846378e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 2230 + }, + { + "epoch": 1.1961866532865026, + "grad_norm": 1.4142135623730951, + "learning_rate": 7.518858674054838e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 2235 + }, + { + "epoch": 1.1988626860679044, + "grad_norm": 1.4142135623730951, + "learning_rate": 7.505393059285394e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 2240 + }, + { + "epoch": 1.2015387188493059, + "grad_norm": 1.4142135623730951, + "learning_rate": 7.491903137181501e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 2245 + }, + { + "epoch": 1.2042147516307076, + "grad_norm": 1.4142135623730951, + "learning_rate": 7.478389038622441e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 2250 + }, + { + "epoch": 1.206890784412109, + "grad_norm": 1.4142135623730951, + "learning_rate": 7.46485089472206e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 2255 + }, + { + "epoch": 1.2095668171935106, + "grad_norm": 1.4142135623730951, + "learning_rate": 7.451288836827487e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 2260 + }, + { + "epoch": 1.2122428499749123, + "grad_norm": 1.4142135623730951, + "learning_rate": 7.437702996517869e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 2265 + }, + { + "epoch": 1.2149188827563138, + "grad_norm": 1.4142135623730951, + "learning_rate": 7.424093505603087e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 2270 + }, + { + "epoch": 1.2175949155377153, + "grad_norm": 1.4142135623730951, + "learning_rate": 7.410460496122482e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 2275 + }, + { + "epoch": 1.220270948319117, + "grad_norm": 1.4142135623730951, + "learning_rate": 7.396804100343572e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 2280 + }, + { + "epoch": 1.2229469811005185, + "grad_norm": 1.4142135623730951, + "learning_rate": 7.383124450760768e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 2285 + }, + { + "epoch": 1.22562301388192, + "grad_norm": 1.4142135623730951, + "learning_rate": 7.369421680094091e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 2290 + }, + { + "epoch": 1.2282990466633217, + "grad_norm": 1.4142135623730951, + "learning_rate": 7.355695921287881e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 2295 + }, + { + "epoch": 1.2309750794447232, + "grad_norm": 1.4142135623730951, + "learning_rate": 7.341947307509513e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 2300 + }, + { + "epoch": 1.233651112226125, + "grad_norm": 1.4142135623730951, + "learning_rate": 7.328175972148094e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 2305 + }, + { + "epoch": 1.2363271450075264, + "grad_norm": 1.4142135623730951, + "learning_rate": 7.314382048813185e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 2310 + }, + { + "epoch": 1.2390031777889279, + "grad_norm": 1.4142135623730951, + "learning_rate": 7.300565671333486e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 2315 + }, + { + "epoch": 1.2416792105703296, + "grad_norm": 1.4142135623730951, + "learning_rate": 7.286726973755554e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 2320 + }, + { + "epoch": 1.244355243351731, + "grad_norm": 1.4142135623730951, + "learning_rate": 7.272866090342493e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 2325 + }, + { + "epoch": 1.2470312761331326, + "grad_norm": 1.4142135623730951, + "learning_rate": 7.258983155572656e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 2330 + }, + { + "epoch": 1.2497073089145343, + "grad_norm": 1.4142135623730951, + "learning_rate": 7.245078304138335e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 2335 + }, + { + "epoch": 1.2523833416959358, + "grad_norm": 1.4142135623730951, + "learning_rate": 7.231151670944462e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 2340 + }, + { + "epoch": 1.2550593744773373, + "grad_norm": 1.4142135623730951, + "learning_rate": 7.217203391107291e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 2345 + }, + { + "epoch": 1.257735407258739, + "grad_norm": 1.4142135623730951, + "learning_rate": 7.203233599953096e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 2350 + }, + { + "epoch": 1.2604114400401405, + "grad_norm": 1.4142135623730951, + "learning_rate": 7.189242433016852e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 2355 + }, + { + "epoch": 1.263087472821542, + "grad_norm": 1.4142135623730951, + "learning_rate": 7.17523002604092e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 2360 + }, + { + "epoch": 1.2657635056029437, + "grad_norm": 1.4142135623730951, + "learning_rate": 7.161196514973734e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 2365 + }, + { + "epoch": 1.2684395383843452, + "grad_norm": 1.4142135623730951, + "learning_rate": 7.147142035968483e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 2370 + }, + { + "epoch": 1.2711155711657467, + "grad_norm": 1.4142135623730951, + "learning_rate": 7.133066725381781e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 2375 + }, + { + "epoch": 1.2737916039471484, + "grad_norm": 1.4142135623730951, + "learning_rate": 7.118970719772354e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 2380 + }, + { + "epoch": 1.27646763672855, + "grad_norm": 1.4142135623730951, + "learning_rate": 7.104854155899711e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 2385 + }, + { + "epoch": 1.2791436695099514, + "grad_norm": 1.4142135623730951, + "learning_rate": 7.090717170722817e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 2390 + }, + { + "epoch": 1.2818197022913531, + "grad_norm": 1.4142135623730951, + "learning_rate": 7.076559901398762e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 2395 + }, + { + "epoch": 1.2844957350727546, + "grad_norm": 1.4142135623730951, + "learning_rate": 7.062382485281436e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 2400 + }, + { + "epoch": 1.2844957350727546, + "eval_logits/chosen": NaN, + "eval_logits/rejected": NaN, + "eval_logps/chosen": NaN, + "eval_logps/rejected": NaN, + "eval_loss": NaN, + "eval_rewards/accuracies": 0.0, + "eval_rewards/chosen": NaN, + "eval_rewards/margins": NaN, + "eval_rewards/rejected": NaN, + "eval_runtime": 35.1218, + "eval_samples_per_second": 38.295, + "eval_semantic_entropy": NaN, + "eval_steps_per_second": 9.595, + "step": 2400 + }, + { + "epoch": 1.287171767854156, + "grad_norm": 1.4142135623730951, + "learning_rate": 7.048185059920193e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 2405 + }, + { + "epoch": 1.2898478006355578, + "grad_norm": 1.4142135623730951, + "learning_rate": 7.033967763058516e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 2410 + }, + { + "epoch": 1.2925238334169593, + "grad_norm": 1.4142135623730951, + "learning_rate": 7.019730732632681e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 2415 + }, + { + "epoch": 1.2951998661983608, + "grad_norm": 1.4142135623730951, + "learning_rate": 7.005474106770418e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 2420 + }, + { + "epoch": 1.2978758989797625, + "grad_norm": 1.4142135623730951, + "learning_rate": 6.991198023789577e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 2425 + }, + { + "epoch": 1.300551931761164, + "grad_norm": 1.4142135623730951, + "learning_rate": 6.976902622196776e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 2430 + }, + { + "epoch": 1.3032279645425655, + "grad_norm": 1.4142135623730951, + "learning_rate": 6.962588040686064e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 2435 + }, + { + "epoch": 1.3059039973239672, + "grad_norm": 1.4142135623730951, + "learning_rate": 6.948254418137573e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 2440 + }, + { + "epoch": 1.3085800301053687, + "grad_norm": 1.4142135623730951, + "learning_rate": 6.933901893616174e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 2445 + }, + { + "epoch": 1.3112560628867704, + "grad_norm": 1.4142135623730951, + "learning_rate": 6.919530606370121e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 2450 + }, + { + "epoch": 1.313932095668172, + "grad_norm": 1.4142135623730951, + "learning_rate": 6.905140695829706e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 2455 + }, + { + "epoch": 1.3166081284495736, + "grad_norm": 1.4142135623730951, + "learning_rate": 6.890732301605904e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 2460 + }, + { + "epoch": 1.3192841612309751, + "grad_norm": 1.4142135623730951, + "learning_rate": 6.876305563489021e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 2465 + }, + { + "epoch": 1.3219601940123766, + "grad_norm": 1.4142135623730951, + "learning_rate": 6.861860621447331e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 2470 + }, + { + "epoch": 1.3246362267937783, + "grad_norm": 1.4142135623730951, + "learning_rate": 6.847397615625725e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 2475 + }, + { + "epoch": 1.3273122595751798, + "grad_norm": 1.4142135623730951, + "learning_rate": 6.83291668634435e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 2480 + }, + { + "epoch": 1.3299882923565813, + "grad_norm": 1.4142135623730951, + "learning_rate": 6.818417974097246e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 2485 + }, + { + "epoch": 1.332664325137983, + "grad_norm": 1.4142135623730951, + "learning_rate": 6.803901619550981e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 2490 + }, + { + "epoch": 1.3353403579193845, + "grad_norm": 1.4142135623730951, + "learning_rate": 6.789367763543292e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 2495 + }, + { + "epoch": 1.338016390700786, + "grad_norm": 1.4142135623730951, + "learning_rate": 6.774816547081714e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 2500 + }, + { + "epoch": 1.3406924234821878, + "grad_norm": 1.4142135623730951, + "learning_rate": 6.760248111342211e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 2505 + }, + { + "epoch": 1.3433684562635893, + "grad_norm": 1.4142135623730951, + "learning_rate": 6.745662597667813e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 2510 + }, + { + "epoch": 1.3460444890449907, + "grad_norm": 1.4142135623730951, + "learning_rate": 6.731060147567236e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 2515 + }, + { + "epoch": 1.3487205218263925, + "grad_norm": 1.4142135623730951, + "learning_rate": 6.716440902713515e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 2520 + }, + { + "epoch": 1.351396554607794, + "grad_norm": 1.4142135623730951, + "learning_rate": 6.701805004942627e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 2525 + }, + { + "epoch": 1.3540725873891954, + "grad_norm": 1.4142135623730951, + "learning_rate": 6.687152596252119e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 2530 + }, + { + "epoch": 1.3567486201705972, + "grad_norm": 1.4142135623730951, + "learning_rate": 6.672483818799722e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 2535 + }, + { + "epoch": 1.3594246529519987, + "grad_norm": 1.4142135623730951, + "learning_rate": 6.657798814901978e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 2540 + }, + { + "epoch": 1.3621006857334002, + "grad_norm": 1.4142135623730951, + "learning_rate": 6.643097727032863e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 2545 + }, + { + "epoch": 1.3647767185148019, + "grad_norm": 1.4142135623730951, + "learning_rate": 6.628380697822392e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 2550 + }, + { + "epoch": 1.3674527512962034, + "grad_norm": 1.4142135623730951, + "learning_rate": 6.61364787005525e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 2555 + }, + { + "epoch": 1.3701287840776049, + "grad_norm": 1.4142135623730951, + "learning_rate": 6.598899386669395e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 2560 + }, + { + "epoch": 1.3728048168590066, + "grad_norm": 1.4142135623730951, + "learning_rate": 6.584135390754679e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 2565 + }, + { + "epoch": 1.375480849640408, + "grad_norm": 1.4142135623730951, + "learning_rate": 6.569356025551454e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 2570 + }, + { + "epoch": 1.3781568824218096, + "grad_norm": 1.4142135623730951, + "learning_rate": 6.554561434449186e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 2575 + }, + { + "epoch": 1.3808329152032113, + "grad_norm": 1.4142135623730951, + "learning_rate": 6.539751760985063e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 2580 + }, + { + "epoch": 1.3835089479846128, + "grad_norm": 1.4142135623730951, + "learning_rate": 6.524927148842602e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 2585 + }, + { + "epoch": 1.3861849807660143, + "grad_norm": 1.4142135623730951, + "learning_rate": 6.510087741850254e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 2590 + }, + { + "epoch": 1.388861013547416, + "grad_norm": 1.4142135623730951, + "learning_rate": 6.495233683980012e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 2595 + }, + { + "epoch": 1.3915370463288175, + "grad_norm": 1.4142135623730951, + "learning_rate": 6.480365119346011e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 2600 + }, + { + "epoch": 1.394213079110219, + "grad_norm": 1.4142135623730951, + "learning_rate": 6.465482192203129e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 2605 + }, + { + "epoch": 1.3968891118916207, + "grad_norm": 1.4142135623730951, + "learning_rate": 6.45058504694559e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 2610 + }, + { + "epoch": 1.3995651446730222, + "grad_norm": 1.4142135623730951, + "learning_rate": 6.435673828105564e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 2615 + }, + { + "epoch": 1.402241177454424, + "grad_norm": 1.4142135623730951, + "learning_rate": 6.420748680351763e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 2620 + }, + { + "epoch": 1.4049172102358254, + "grad_norm": 1.4142135623730951, + "learning_rate": 6.405809748488032e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 2625 + }, + { + "epoch": 1.4075932430172269, + "grad_norm": 1.4142135623730951, + "learning_rate": 6.390857177451956e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 2630 + }, + { + "epoch": 1.4102692757986286, + "grad_norm": 1.4142135623730951, + "learning_rate": 6.375891112313445e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 2635 + }, + { + "epoch": 1.41294530858003, + "grad_norm": 1.4142135623730951, + "learning_rate": 6.360911698273326e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 2640 + }, + { + "epoch": 1.4156213413614318, + "grad_norm": 1.4142135623730951, + "learning_rate": 6.345919080661944e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 2645 + }, + { + "epoch": 1.4182973741428333, + "grad_norm": 1.4142135623730951, + "learning_rate": 6.330913404937737e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 2650 + }, + { + "epoch": 1.4209734069242348, + "grad_norm": 1.4142135623730951, + "learning_rate": 6.315894816685838e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 2655 + }, + { + "epoch": 1.4236494397056365, + "grad_norm": 1.4142135623730951, + "learning_rate": 6.300863461616657e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 2660 + }, + { + "epoch": 1.426325472487038, + "grad_norm": 1.4142135623730951, + "learning_rate": 6.285819485564465e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 2665 + }, + { + "epoch": 1.4290015052684395, + "grad_norm": 1.4142135623730951, + "learning_rate": 6.270763034485986e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 2670 + }, + { + "epoch": 1.4316775380498412, + "grad_norm": 1.4142135623730951, + "learning_rate": 6.255694254458972e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 2675 + }, + { + "epoch": 1.4343535708312427, + "grad_norm": 1.4142135623730951, + "learning_rate": 6.240613291680795e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 2680 + }, + { + "epoch": 1.4370296036126442, + "grad_norm": 1.4142135623730951, + "learning_rate": 6.225520292467021e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 2685 + }, + { + "epoch": 1.439705636394046, + "grad_norm": 1.4142135623730951, + "learning_rate": 6.210415403249993e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 2690 + }, + { + "epoch": 1.4423816691754474, + "grad_norm": 1.4142135623730951, + "learning_rate": 6.195298770577415e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 2695 + }, + { + "epoch": 1.445057701956849, + "grad_norm": 1.4142135623730951, + "learning_rate": 6.180170541110923e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 2700 + }, + { + "epoch": 1.4477337347382506, + "grad_norm": 1.4142135623730951, + "learning_rate": 6.165030861624663e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 2705 + }, + { + "epoch": 1.4504097675196521, + "grad_norm": 1.4142135623730951, + "learning_rate": 6.149879879003876e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 2710 + }, + { + "epoch": 1.4530858003010536, + "grad_norm": 1.4142135623730951, + "learning_rate": 6.13471774024346e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 2715 + }, + { + "epoch": 1.4557618330824553, + "grad_norm": 1.4142135623730951, + "learning_rate": 6.119544592446551e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 2720 + }, + { + "epoch": 1.4584378658638568, + "grad_norm": 1.4142135623730951, + "learning_rate": 6.104360582823096e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 2725 + }, + { + "epoch": 1.4611138986452583, + "grad_norm": 1.4142135623730951, + "learning_rate": 6.089165858688423e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 2730 + }, + { + "epoch": 1.46378993142666, + "grad_norm": 1.4142135623730951, + "learning_rate": 6.073960567461811e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 2735 + }, + { + "epoch": 1.4664659642080615, + "grad_norm": 1.4142135623730951, + "learning_rate": 6.058744856665065e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 2740 + }, + { + "epoch": 1.469141996989463, + "grad_norm": 1.4142135623730951, + "learning_rate": 6.043518873921074e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 2745 + }, + { + "epoch": 1.4718180297708647, + "grad_norm": 1.4142135623730951, + "learning_rate": 6.028282766952393e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 2750 + }, + { + "epoch": 1.4744940625522662, + "grad_norm": 1.4142135623730951, + "learning_rate": 6.013036683579798e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 2755 + }, + { + "epoch": 1.4771700953336677, + "grad_norm": 1.4142135623730951, + "learning_rate": 5.997780771720854e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 2760 + }, + { + "epoch": 1.4798461281150694, + "grad_norm": 1.4142135623730951, + "learning_rate": 5.982515179388486e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 2765 + }, + { + "epoch": 1.482522160896471, + "grad_norm": 1.4142135623730951, + "learning_rate": 5.967240054689541e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 2770 + }, + { + "epoch": 1.4851981936778724, + "grad_norm": 1.4142135623730951, + "learning_rate": 5.951955545823342e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 2775 + }, + { + "epoch": 1.4878742264592741, + "grad_norm": 1.4142135623730951, + "learning_rate": 5.936661801080263e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 2780 + }, + { + "epoch": 1.4905502592406756, + "grad_norm": 1.4142135623730951, + "learning_rate": 5.92135896884028e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 2785 + }, + { + "epoch": 1.4932262920220774, + "grad_norm": 1.4142135623730951, + "learning_rate": 5.906047197571541e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 2790 + }, + { + "epoch": 1.4959023248034788, + "grad_norm": 1.4142135623730951, + "learning_rate": 5.890726635828919e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 2795 + }, + { + "epoch": 1.4985783575848803, + "grad_norm": 1.4142135623730951, + "learning_rate": 5.875397432252569e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 2800 + }, + { + "epoch": 1.4985783575848803, + "eval_logits/chosen": NaN, + "eval_logits/rejected": NaN, + "eval_logps/chosen": NaN, + "eval_logps/rejected": NaN, + "eval_loss": NaN, + "eval_rewards/accuracies": 0.0, + "eval_rewards/chosen": NaN, + "eval_rewards/margins": NaN, + "eval_rewards/rejected": NaN, + "eval_runtime": 35.2612, + "eval_samples_per_second": 38.144, + "eval_semantic_entropy": NaN, + "eval_steps_per_second": 9.557, + "step": 2800 + }, + { + "epoch": 1.5012543903662818, + "grad_norm": 1.4142135623730951, + "learning_rate": 5.860059735566491e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 2805 + }, + { + "epoch": 1.5039304231476835, + "grad_norm": 1.4142135623730951, + "learning_rate": 5.844713694577087e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 2810 + }, + { + "epoch": 1.5066064559290853, + "grad_norm": 1.4142135623730951, + "learning_rate": 5.829359458171714e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 2815 + }, + { + "epoch": 1.5092824887104868, + "grad_norm": 1.4142135623730951, + "learning_rate": 5.81399717531724e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 2820 + }, + { + "epoch": 1.5119585214918883, + "grad_norm": 1.4142135623730951, + "learning_rate": 5.798626995058602e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 2825 + }, + { + "epoch": 1.51463455427329, + "grad_norm": 1.4142135623730951, + "learning_rate": 5.783249066517354e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 2830 + }, + { + "epoch": 1.5173105870546915, + "grad_norm": 1.4142135623730951, + "learning_rate": 5.767863538890228e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 2835 + }, + { + "epoch": 1.519986619836093, + "grad_norm": 1.4142135623730951, + "learning_rate": 5.75247056144768e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 2840 + }, + { + "epoch": 1.5226626526174947, + "grad_norm": 1.4142135623730951, + "learning_rate": 5.737070283532444e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 2845 + }, + { + "epoch": 1.5253386853988962, + "grad_norm": 1.4142135623730951, + "learning_rate": 5.721662854558084e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 2850 + }, + { + "epoch": 1.5280147181802977, + "grad_norm": 1.4142135623730951, + "learning_rate": 5.706248424007545e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 2855 + }, + { + "epoch": 1.5306907509616994, + "grad_norm": 1.4142135623730951, + "learning_rate": 5.690827141431699e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 2860 + }, + { + "epoch": 1.5333667837431009, + "grad_norm": 1.4142135623730951, + "learning_rate": 5.675399156447897e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 2865 + }, + { + "epoch": 1.5360428165245024, + "grad_norm": 1.4142135623730951, + "learning_rate": 5.659964618738515e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 2870 + }, + { + "epoch": 1.538718849305904, + "grad_norm": 1.4142135623730951, + "learning_rate": 5.644523678049509e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 2875 + }, + { + "epoch": 1.5413948820873056, + "grad_norm": 1.4142135623730951, + "learning_rate": 5.629076484188952e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 2880 + }, + { + "epoch": 1.544070914868707, + "grad_norm": 1.4142135623730951, + "learning_rate": 5.613623187025587e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 2885 + }, + { + "epoch": 1.5467469476501088, + "grad_norm": 1.4142135623730951, + "learning_rate": 5.598163936487369e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 2890 + }, + { + "epoch": 1.5494229804315103, + "grad_norm": 1.4142135623730951, + "learning_rate": 5.582698882560017e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 2895 + }, + { + "epoch": 1.5520990132129118, + "grad_norm": 1.4142135623730951, + "learning_rate": 5.567228175285549e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 2900 + }, + { + "epoch": 1.5547750459943135, + "grad_norm": 1.4142135623730951, + "learning_rate": 5.551751964760838e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 2905 + }, + { + "epoch": 1.557451078775715, + "grad_norm": 1.4142135623730951, + "learning_rate": 5.536270401136145e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 2910 + }, + { + "epoch": 1.5601271115571165, + "grad_norm": 1.4142135623730951, + "learning_rate": 5.520783634613667e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 2915 + }, + { + "epoch": 1.5628031443385182, + "grad_norm": 1.4142135623730951, + "learning_rate": 5.505291815446082e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 2920 + }, + { + "epoch": 1.5654791771199197, + "grad_norm": 1.4142135623730951, + "learning_rate": 5.489795093935089e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 2925 + }, + { + "epoch": 1.5681552099013212, + "grad_norm": 1.4142135623730951, + "learning_rate": 5.474293620429946e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 2930 + }, + { + "epoch": 1.570831242682723, + "grad_norm": 1.4142135623730951, + "learning_rate": 5.458787545326018e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 2935 + }, + { + "epoch": 1.5735072754641244, + "grad_norm": 1.4142135623730951, + "learning_rate": 5.443277019063311e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 2940 + }, + { + "epoch": 1.5761833082455259, + "grad_norm": 1.4142135623730951, + "learning_rate": 5.427762192125023e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 2945 + }, + { + "epoch": 1.5788593410269276, + "grad_norm": 1.4142135623730951, + "learning_rate": 5.41224321503607e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 2950 + }, + { + "epoch": 1.5815353738083293, + "grad_norm": 1.4142135623730951, + "learning_rate": 5.396720238361637e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 2955 + }, + { + "epoch": 1.5842114065897306, + "grad_norm": 1.4142135623730951, + "learning_rate": 5.381193412705711e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 2960 + }, + { + "epoch": 1.5868874393711323, + "grad_norm": 1.4142135623730951, + "learning_rate": 5.365662888709622e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 2965 + }, + { + "epoch": 1.589563472152534, + "grad_norm": 1.4142135623730951, + "learning_rate": 5.350128817050585e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 2970 + }, + { + "epoch": 1.5922395049339353, + "grad_norm": 1.4142135623730951, + "learning_rate": 5.334591348440229e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 2975 + }, + { + "epoch": 1.594915537715337, + "grad_norm": 1.4142135623730951, + "learning_rate": 5.319050633623141e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 2980 + }, + { + "epoch": 1.5975915704967387, + "grad_norm": 1.4142135623730951, + "learning_rate": 5.303506823375409e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 2985 + }, + { + "epoch": 1.60026760327814, + "grad_norm": 1.4142135623730951, + "learning_rate": 5.287960068503143e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 2990 + }, + { + "epoch": 1.6029436360595417, + "grad_norm": 1.4142135623730951, + "learning_rate": 5.272410519841032e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 2995 + }, + { + "epoch": 1.6056196688409434, + "grad_norm": 1.4142135623730951, + "learning_rate": 5.256858328250861e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 3000 + }, + { + "epoch": 1.608295701622345, + "grad_norm": 1.4142135623730951, + "learning_rate": 5.241303644620063e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 3005 + }, + { + "epoch": 1.6109717344037464, + "grad_norm": 1.4142135623730951, + "learning_rate": 5.225746619860248e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 3010 + }, + { + "epoch": 1.6136477671851481, + "grad_norm": 1.4142135623730951, + "learning_rate": 5.210187404905735e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 3015 + }, + { + "epoch": 1.6163237999665496, + "grad_norm": 1.4142135623730951, + "learning_rate": 5.194626150712098e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 3020 + }, + { + "epoch": 1.6189998327479511, + "grad_norm": 1.4142135623730951, + "learning_rate": 5.179063008254695e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 3025 + }, + { + "epoch": 1.6216758655293528, + "grad_norm": 1.4142135623730951, + "learning_rate": 5.163498128527199e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 3030 + }, + { + "epoch": 1.6243518983107543, + "grad_norm": 1.4142135623730951, + "learning_rate": 5.147931662540144e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 3035 + }, + { + "epoch": 1.6270279310921558, + "grad_norm": 1.4142135623730951, + "learning_rate": 5.132363761319449e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 3040 + }, + { + "epoch": 1.6297039638735575, + "grad_norm": 1.4142135623730951, + "learning_rate": 5.116794575904962e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 3045 + }, + { + "epoch": 1.632379996654959, + "grad_norm": 1.4142135623730951, + "learning_rate": 5.101224257348987e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 3050 + }, + { + "epoch": 1.6350560294363605, + "grad_norm": 1.4142135623730951, + "learning_rate": 5.085652956714823e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 3055 + }, + { + "epoch": 1.6377320622177622, + "grad_norm": 1.4142135623730951, + "learning_rate": 5.070080825075298e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 3060 + }, + { + "epoch": 1.6404080949991637, + "grad_norm": 1.4142135623730951, + "learning_rate": 5.0545080135113e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 3065 + }, + { + "epoch": 1.6430841277805652, + "grad_norm": 1.4142135623730951, + "learning_rate": 5.038934673110316e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 3070 + }, + { + "epoch": 1.645760160561967, + "grad_norm": 1.4142135623730951, + "learning_rate": 5.023360954964963e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 3075 + }, + { + "epoch": 1.6484361933433684, + "grad_norm": 1.4142135623730951, + "learning_rate": 5.007787010171524e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 3080 + }, + { + "epoch": 1.65111222612477, + "grad_norm": 1.4142135623730951, + "learning_rate": 4.992212989828477e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 3085 + }, + { + "epoch": 1.6537882589061716, + "grad_norm": 1.4142135623730951, + "learning_rate": 4.976639045035036e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 3090 + }, + { + "epoch": 1.6564642916875731, + "grad_norm": 1.4142135623730951, + "learning_rate": 4.961065326889683e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 3095 + }, + { + "epoch": 1.6591403244689746, + "grad_norm": 1.4142135623730951, + "learning_rate": 4.9454919864887e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 3100 + }, + { + "epoch": 1.6618163572503764, + "grad_norm": 1.4142135623730951, + "learning_rate": 4.929919174924701e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 3105 + }, + { + "epoch": 1.6644923900317778, + "grad_norm": 1.4142135623730951, + "learning_rate": 4.914347043285177e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 3110 + }, + { + "epoch": 1.6671684228131793, + "grad_norm": 1.4142135623730951, + "learning_rate": 4.898775742651013e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 3115 + }, + { + "epoch": 1.669844455594581, + "grad_norm": 1.4142135623730951, + "learning_rate": 4.883205424095037e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 3120 + }, + { + "epoch": 1.6725204883759828, + "grad_norm": 1.4142135623730951, + "learning_rate": 4.86763623868055e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 3125 + }, + { + "epoch": 1.675196521157384, + "grad_norm": 1.4142135623730951, + "learning_rate": 4.852068337459856e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 3130 + }, + { + "epoch": 1.6778725539387858, + "grad_norm": 1.4142135623730951, + "learning_rate": 4.8365018714728e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 3135 + }, + { + "epoch": 1.6805485867201875, + "grad_norm": 1.4142135623730951, + "learning_rate": 4.820936991745304e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 3140 + }, + { + "epoch": 1.6832246195015887, + "grad_norm": 1.4142135623730951, + "learning_rate": 4.8053738492879e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 3145 + }, + { + "epoch": 1.6859006522829905, + "grad_norm": 1.4142135623730951, + "learning_rate": 4.789812595094265e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 3150 + }, + { + "epoch": 1.6885766850643922, + "grad_norm": 1.4142135623730951, + "learning_rate": 4.774253380139752e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 3155 + }, + { + "epoch": 1.6912527178457935, + "grad_norm": 1.4142135623730951, + "learning_rate": 4.758696355379936e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 3160 + }, + { + "epoch": 1.6939287506271952, + "grad_norm": 1.4142135623730951, + "learning_rate": 4.743141671749138e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 3165 + }, + { + "epoch": 1.6966047834085969, + "grad_norm": 1.4142135623730951, + "learning_rate": 4.727589480158968e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 3170 + }, + { + "epoch": 1.6992808161899984, + "grad_norm": 1.4142135623730951, + "learning_rate": 4.712039931496855e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 3175 + }, + { + "epoch": 1.7019568489713999, + "grad_norm": 1.4142135623730951, + "learning_rate": 4.6964931766245905e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 3180 + }, + { + "epoch": 1.7046328817528016, + "grad_norm": 1.4142135623730951, + "learning_rate": 4.6809493663768575e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 3185 + }, + { + "epoch": 1.707308914534203, + "grad_norm": 1.4142135623730951, + "learning_rate": 4.6654086515597716e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 3190 + }, + { + "epoch": 1.7099849473156046, + "grad_norm": 1.4142135623730951, + "learning_rate": 4.6498711829494154e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 3195 + }, + { + "epoch": 1.7126609800970063, + "grad_norm": 1.4142135623730951, + "learning_rate": 4.6343371112903777e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 3200 + }, + { + "epoch": 1.7126609800970063, + "eval_logits/chosen": NaN, + "eval_logits/rejected": NaN, + "eval_logps/chosen": NaN, + "eval_logps/rejected": NaN, + "eval_loss": NaN, + "eval_rewards/accuracies": 0.0, + "eval_rewards/chosen": NaN, + "eval_rewards/margins": NaN, + "eval_rewards/rejected": NaN, + "eval_runtime": 35.2595, + "eval_samples_per_second": 38.146, + "eval_semantic_entropy": NaN, + "eval_steps_per_second": 9.558, + "step": 3200 + }, + { + "epoch": 1.7153370128784078, + "grad_norm": 1.4142135623730951, + "learning_rate": 4.618806587294291e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 3205 + }, + { + "epoch": 1.7180130456598093, + "grad_norm": 1.4142135623730951, + "learning_rate": 4.603279761638365e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 3210 + }, + { + "epoch": 1.720689078441211, + "grad_norm": 1.4142135623730951, + "learning_rate": 4.5877567849639315e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 3215 + }, + { + "epoch": 1.7233651112226125, + "grad_norm": 1.4142135623730951, + "learning_rate": 4.572237807874979e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 3220 + }, + { + "epoch": 1.726041144004014, + "grad_norm": 1.4142135623730951, + "learning_rate": 4.5567229809366895e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 3225 + }, + { + "epoch": 1.7287171767854157, + "grad_norm": 1.4142135623730951, + "learning_rate": 4.541212454673984e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 3230 + }, + { + "epoch": 1.7313932095668172, + "grad_norm": 1.4142135623730951, + "learning_rate": 4.525706379570055e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 3235 + }, + { + "epoch": 1.7340692423482187, + "grad_norm": 1.4142135623730951, + "learning_rate": 4.510204906064911e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 3240 + }, + { + "epoch": 1.7367452751296204, + "grad_norm": 1.4142135623730951, + "learning_rate": 4.4947081845539177e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 3245 + }, + { + "epoch": 1.739421307911022, + "grad_norm": 1.4142135623730951, + "learning_rate": 4.479216365386333e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 3250 + }, + { + "epoch": 1.7420973406924234, + "grad_norm": 1.4142135623730951, + "learning_rate": 4.4637295988638555e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 3255 + }, + { + "epoch": 1.744773373473825, + "grad_norm": 1.4142135623730951, + "learning_rate": 4.4482480352391623e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 3260 + }, + { + "epoch": 1.7474494062552266, + "grad_norm": 1.4142135623730951, + "learning_rate": 4.4327718247144507e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 3265 + }, + { + "epoch": 1.750125439036628, + "grad_norm": 1.4142135623730951, + "learning_rate": 4.417301117439984e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 3270 + }, + { + "epoch": 1.7528014718180298, + "grad_norm": 1.4142135623730951, + "learning_rate": 4.401836063512631e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 3275 + }, + { + "epoch": 1.7554775045994313, + "grad_norm": 1.4142135623730951, + "learning_rate": 4.386376812974413e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 3280 + }, + { + "epoch": 1.7581535373808328, + "grad_norm": 1.4142135623730951, + "learning_rate": 4.370923515811048e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 3285 + }, + { + "epoch": 1.7608295701622345, + "grad_norm": 1.4142135623730951, + "learning_rate": 4.35547632195049e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 3290 + }, + { + "epoch": 1.763505602943636, + "grad_norm": 1.4142135623730951, + "learning_rate": 4.340035381261484e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 3295 + }, + { + "epoch": 1.7661816357250375, + "grad_norm": 1.4142135623730951, + "learning_rate": 4.324600843552104e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 3300 + }, + { + "epoch": 1.7688576685064392, + "grad_norm": 1.4142135623730951, + "learning_rate": 4.309172858568302e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 3305 + }, + { + "epoch": 1.771533701287841, + "grad_norm": 1.4142135623730951, + "learning_rate": 4.293751575992455e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 3310 + }, + { + "epoch": 1.7742097340692422, + "grad_norm": 1.4142135623730951, + "learning_rate": 4.278337145441916e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 3315 + }, + { + "epoch": 1.776885766850644, + "grad_norm": 1.4142135623730951, + "learning_rate": 4.262929716467556e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 3320 + }, + { + "epoch": 1.7795617996320456, + "grad_norm": 1.4142135623730951, + "learning_rate": 4.247529438552321e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 3325 + }, + { + "epoch": 1.782237832413447, + "grad_norm": 1.4142135623730951, + "learning_rate": 4.232136461109773e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 3330 + }, + { + "epoch": 1.7849138651948486, + "grad_norm": 1.4142135623730951, + "learning_rate": 4.216750933482646e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 3335 + }, + { + "epoch": 1.7875898979762503, + "grad_norm": 1.4142135623730951, + "learning_rate": 4.2013730049413986e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 3340 + }, + { + "epoch": 1.7902659307576518, + "grad_norm": 1.4142135623730951, + "learning_rate": 4.1860028246827594e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 3345 + }, + { + "epoch": 1.7929419635390533, + "grad_norm": 1.4142135623730951, + "learning_rate": 4.170640541828285e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 3350 + }, + { + "epoch": 1.795617996320455, + "grad_norm": 1.4142135623730951, + "learning_rate": 4.1552863054229116e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 3355 + }, + { + "epoch": 1.7982940291018565, + "grad_norm": 1.4142135623730951, + "learning_rate": 4.139940264433508e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 3360 + }, + { + "epoch": 1.800970061883258, + "grad_norm": 1.4142135623730951, + "learning_rate": 4.1246025677474303e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 3365 + }, + { + "epoch": 1.8036460946646597, + "grad_norm": 1.4142135623730951, + "learning_rate": 4.10927336417108e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 3370 + }, + { + "epoch": 1.8063221274460612, + "grad_norm": 1.4142135623730951, + "learning_rate": 4.093952802428457e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 3375 + }, + { + "epoch": 1.8089981602274627, + "grad_norm": 1.4142135623730951, + "learning_rate": 4.0786410311597184e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 3380 + }, + { + "epoch": 1.8116741930088645, + "grad_norm": 1.4142135623730951, + "learning_rate": 4.063338198919737e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 3385 + }, + { + "epoch": 1.814350225790266, + "grad_norm": 1.4142135623730951, + "learning_rate": 4.0480444541766575e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 3390 + }, + { + "epoch": 1.8170262585716674, + "grad_norm": 1.4142135623730951, + "learning_rate": 4.0327599453104606e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 3395 + }, + { + "epoch": 1.8197022913530692, + "grad_norm": 1.4142135623730951, + "learning_rate": 4.017484820611514e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 3400 + }, + { + "epoch": 1.8223783241344707, + "grad_norm": 1.4142135623730951, + "learning_rate": 4.002219228279148e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 3405 + }, + { + "epoch": 1.8250543569158721, + "grad_norm": 1.4142135623730951, + "learning_rate": 3.9869633164202045e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 3410 + }, + { + "epoch": 1.8277303896972739, + "grad_norm": 1.4142135623730951, + "learning_rate": 3.9717172330476077e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 3415 + }, + { + "epoch": 1.8304064224786754, + "grad_norm": 1.4142135623730951, + "learning_rate": 3.956481126078927e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 3420 + }, + { + "epoch": 1.8330824552600768, + "grad_norm": 1.4142135623730951, + "learning_rate": 3.941255143334937e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 3425 + }, + { + "epoch": 1.8357584880414786, + "grad_norm": 1.4142135623730951, + "learning_rate": 3.9260394325381895e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 3430 + }, + { + "epoch": 1.83843452082288, + "grad_norm": 1.4142135623730951, + "learning_rate": 3.9108341413115784e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 3435 + }, + { + "epoch": 1.8411105536042816, + "grad_norm": 1.4142135623730951, + "learning_rate": 3.895639417176905e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 3440 + }, + { + "epoch": 1.8437865863856833, + "grad_norm": 1.4142135623730951, + "learning_rate": 3.8804554075534497e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 3445 + }, + { + "epoch": 1.8464626191670848, + "grad_norm": 1.4142135623730951, + "learning_rate": 3.8652822597565403e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 3450 + }, + { + "epoch": 1.8491386519484863, + "grad_norm": 1.4142135623730951, + "learning_rate": 3.850120120996123e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 3455 + }, + { + "epoch": 1.851814684729888, + "grad_norm": 1.4142135623730951, + "learning_rate": 3.8349691383753356e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 3460 + }, + { + "epoch": 1.8544907175112895, + "grad_norm": 1.4142135623730951, + "learning_rate": 3.819829458889078e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 3465 + }, + { + "epoch": 1.857166750292691, + "grad_norm": 1.4142135623730951, + "learning_rate": 3.804701229422585e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 3470 + }, + { + "epoch": 1.8598427830740927, + "grad_norm": 1.4142135623730951, + "learning_rate": 3.789584596750007e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 3475 + }, + { + "epoch": 1.8625188158554944, + "grad_norm": 1.4142135623730951, + "learning_rate": 3.77447970753298e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 3480 + }, + { + "epoch": 1.8651948486368957, + "grad_norm": 1.4142135623730951, + "learning_rate": 3.7593867083192057e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 3485 + }, + { + "epoch": 1.8678708814182974, + "grad_norm": 1.4142135623730951, + "learning_rate": 3.7443057455410276e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 3490 + }, + { + "epoch": 1.870546914199699, + "grad_norm": 1.4142135623730951, + "learning_rate": 3.7292369655140145e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 3495 + }, + { + "epoch": 1.8732229469811004, + "grad_norm": 1.4142135623730951, + "learning_rate": 3.714180514435534e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 3500 + }, + { + "epoch": 1.875898979762502, + "grad_norm": 1.4142135623730951, + "learning_rate": 3.6991365383833426e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 3505 + }, + { + "epoch": 1.8785750125439038, + "grad_norm": 1.4142135623730951, + "learning_rate": 3.684105183314162e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 3510 + }, + { + "epoch": 1.881251045325305, + "grad_norm": 1.4142135623730951, + "learning_rate": 3.669086595062263e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 3515 + }, + { + "epoch": 1.8839270781067068, + "grad_norm": 1.4142135623730951, + "learning_rate": 3.654080919338056e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 3520 + }, + { + "epoch": 1.8866031108881085, + "grad_norm": 1.4142135623730951, + "learning_rate": 3.639088301726673e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 3525 + }, + { + "epoch": 1.88927914366951, + "grad_norm": 1.4142135623730951, + "learning_rate": 3.624108887686556e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 3530 + }, + { + "epoch": 1.8919551764509115, + "grad_norm": 1.4142135623730951, + "learning_rate": 3.6091428225480433e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 3535 + }, + { + "epoch": 1.8946312092323132, + "grad_norm": 1.4142135623730951, + "learning_rate": 3.5941902515119674e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 3540 + }, + { + "epoch": 1.8973072420137147, + "grad_norm": 1.4142135623730951, + "learning_rate": 3.5792513196482373e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 3545 + }, + { + "epoch": 1.8999832747951162, + "grad_norm": 1.4142135623730951, + "learning_rate": 3.5643261718944346e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 3550 + }, + { + "epoch": 1.902659307576518, + "grad_norm": 1.4142135623730951, + "learning_rate": 3.5494149530544087e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 3555 + }, + { + "epoch": 1.9053353403579194, + "grad_norm": 1.4142135623730951, + "learning_rate": 3.534517807796871e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 3560 + }, + { + "epoch": 1.908011373139321, + "grad_norm": 1.4142135623730951, + "learning_rate": 3.519634880653988e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 3565 + }, + { + "epoch": 1.9106874059207226, + "grad_norm": 1.4142135623730951, + "learning_rate": 3.504766316019987e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 3570 + }, + { + "epoch": 1.913363438702124, + "grad_norm": 1.4142135623730951, + "learning_rate": 3.489912258149745e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 3575 + }, + { + "epoch": 1.9160394714835256, + "grad_norm": 1.4142135623730951, + "learning_rate": 3.475072851157397e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 3580 + }, + { + "epoch": 1.9187155042649273, + "grad_norm": 1.4142135623730951, + "learning_rate": 3.460248239014936e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 3585 + }, + { + "epoch": 1.9213915370463288, + "grad_norm": 1.4142135623730951, + "learning_rate": 3.4454385655508134e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 3590 + }, + { + "epoch": 1.9240675698277303, + "grad_norm": 1.4142135623730951, + "learning_rate": 3.4306439744485447e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 3595 + }, + { + "epoch": 1.926743602609132, + "grad_norm": 1.4142135623730951, + "learning_rate": 3.415864609245322e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 3600 + }, + { + "epoch": 1.926743602609132, + "eval_logits/chosen": NaN, + "eval_logits/rejected": NaN, + "eval_logps/chosen": NaN, + "eval_logps/rejected": NaN, + "eval_loss": NaN, + "eval_rewards/accuracies": 0.0, + "eval_rewards/chosen": NaN, + "eval_rewards/margins": NaN, + "eval_rewards/rejected": NaN, + "eval_runtime": 35.2415, + "eval_samples_per_second": 38.165, + "eval_semantic_entropy": NaN, + "eval_steps_per_second": 9.563, + "step": 3600 + }, + { + "epoch": 1.9294196353905335, + "grad_norm": 1.4142135623730951, + "learning_rate": 3.401100613330605e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 3605 + }, + { + "epoch": 1.932095668171935, + "grad_norm": 1.4142135623730951, + "learning_rate": 3.3863521299447514e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 3610 + }, + { + "epoch": 1.9347717009533367, + "grad_norm": 1.4142135623730951, + "learning_rate": 3.371619302177609e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 3615 + }, + { + "epoch": 1.9374477337347382, + "grad_norm": 1.4142135623730951, + "learning_rate": 3.3569022729671393e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 3620 + }, + { + "epoch": 1.9401237665161397, + "grad_norm": 1.4142135623730951, + "learning_rate": 3.342201185098024e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 3625 + }, + { + "epoch": 1.9427997992975414, + "grad_norm": 1.4142135623730951, + "learning_rate": 3.3275161812002807e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 3630 + }, + { + "epoch": 1.945475832078943, + "grad_norm": 1.4142135623730951, + "learning_rate": 3.312847403747883e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 3635 + }, + { + "epoch": 1.9481518648603444, + "grad_norm": 1.4142135623730951, + "learning_rate": 3.2981949950573733e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 3640 + }, + { + "epoch": 1.9508278976417461, + "grad_norm": 1.4142135623730951, + "learning_rate": 3.283559097286486e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 3645 + }, + { + "epoch": 1.9535039304231478, + "grad_norm": 1.4142135623730951, + "learning_rate": 3.268939852432765e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 3650 + }, + { + "epoch": 1.9561799632045491, + "grad_norm": 1.4142135623730951, + "learning_rate": 3.254337402332187e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 3655 + }, + { + "epoch": 1.9588559959859508, + "grad_norm": 1.4142135623730951, + "learning_rate": 3.239751888657788e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 3660 + }, + { + "epoch": 1.9615320287673526, + "grad_norm": 1.4142135623730951, + "learning_rate": 3.2251834529182856e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 3665 + }, + { + "epoch": 1.9642080615487538, + "grad_norm": 1.4142135623730951, + "learning_rate": 3.2106322364567075e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 3670 + }, + { + "epoch": 1.9668840943301555, + "grad_norm": 1.4142135623730951, + "learning_rate": 3.1960983804490183e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 3675 + }, + { + "epoch": 1.9695601271115573, + "grad_norm": 1.4142135623730951, + "learning_rate": 3.1815820259027537e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 3680 + }, + { + "epoch": 1.9722361598929585, + "grad_norm": 1.4142135623730951, + "learning_rate": 3.16708331365565e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 3685 + }, + { + "epoch": 1.9749121926743602, + "grad_norm": 1.4142135623730951, + "learning_rate": 3.152602384374275e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 3690 + }, + { + "epoch": 1.977588225455762, + "grad_norm": 1.4142135623730951, + "learning_rate": 3.1381393785526697e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 3695 + }, + { + "epoch": 1.9802642582371635, + "grad_norm": 1.4142135623730951, + "learning_rate": 3.123694436510979e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 3700 + }, + { + "epoch": 1.982940291018565, + "grad_norm": 1.4142135623730951, + "learning_rate": 3.1092676983940946e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 3705 + }, + { + "epoch": 1.9856163237999667, + "grad_norm": 1.4142135623730951, + "learning_rate": 3.094859304170293e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 3710 + }, + { + "epoch": 1.9882923565813682, + "grad_norm": 1.4142135623730951, + "learning_rate": 3.0804693936298795e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 3715 + }, + { + "epoch": 1.9909683893627697, + "grad_norm": 1.4142135623730951, + "learning_rate": 3.066098106383826e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 3720 + }, + { + "epoch": 1.9936444221441714, + "grad_norm": 1.4142135623730951, + "learning_rate": 3.0517455818624263e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 3725 + }, + { + "epoch": 1.9963204549255729, + "grad_norm": 1.4142135623730951, + "learning_rate": 3.037411959313936e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 3730 + }, + { + "epoch": 1.9989964877069744, + "grad_norm": 1.4142135623730951, + "learning_rate": 3.023097377803224e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 3735 + }, + { + "epoch": 2.001672520488376, + "grad_norm": 1.4142135623730951, + "learning_rate": 3.008801976210423e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 3740 + }, + { + "epoch": 2.0043485532697773, + "grad_norm": 1.4142135623730951, + "learning_rate": 2.994525893229581e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 3745 + }, + { + "epoch": 2.007024586051179, + "grad_norm": 1.4142135623730951, + "learning_rate": 2.98026926736732e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 3750 + }, + { + "epoch": 2.0097006188325808, + "grad_norm": 1.4142135623730951, + "learning_rate": 2.9660322369414846e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 3755 + }, + { + "epoch": 2.0123766516139825, + "grad_norm": 1.4142135623730951, + "learning_rate": 2.9518149400798063e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 3760 + }, + { + "epoch": 2.0150526843953838, + "grad_norm": 1.4142135623730951, + "learning_rate": 2.9376175147185633e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 3765 + }, + { + "epoch": 2.0177287171767855, + "grad_norm": 1.4142135623730951, + "learning_rate": 2.9234400986012376e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 3770 + }, + { + "epoch": 2.020404749958187, + "grad_norm": 1.4142135623730951, + "learning_rate": 2.9092828292771817e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 3775 + }, + { + "epoch": 2.0230807827395885, + "grad_norm": 1.4142135623730951, + "learning_rate": 2.8951458441002875e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 3780 + }, + { + "epoch": 2.02575681552099, + "grad_norm": 1.4142135623730951, + "learning_rate": 2.881029280227643e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 3785 + }, + { + "epoch": 2.028432848302392, + "grad_norm": 1.4142135623730951, + "learning_rate": 2.8669332746182177e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 3790 + }, + { + "epoch": 2.031108881083793, + "grad_norm": 1.4142135623730951, + "learning_rate": 2.8528579640315156e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 3795 + }, + { + "epoch": 2.033784913865195, + "grad_norm": 1.4142135623730951, + "learning_rate": 2.8388034850262646e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 3800 + }, + { + "epoch": 2.0364609466465966, + "grad_norm": 1.4142135623730951, + "learning_rate": 2.824769973959079e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 3805 + }, + { + "epoch": 2.039136979427998, + "grad_norm": 1.4142135623730951, + "learning_rate": 2.81075756698315e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 3810 + }, + { + "epoch": 2.0418130122093996, + "grad_norm": 1.4142135623730951, + "learning_rate": 2.7967664000469035e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 3815 + }, + { + "epoch": 2.0444890449908013, + "grad_norm": 1.4142135623730951, + "learning_rate": 2.7827966088927095e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 3820 + }, + { + "epoch": 2.0471650777722026, + "grad_norm": 1.4142135623730951, + "learning_rate": 2.768848329055538e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 3825 + }, + { + "epoch": 2.0498411105536043, + "grad_norm": 1.4142135623730951, + "learning_rate": 2.7549216958616657e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 3830 + }, + { + "epoch": 2.052517143335006, + "grad_norm": 1.4142135623730951, + "learning_rate": 2.741016844427344e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 3835 + }, + { + "epoch": 2.0551931761164073, + "grad_norm": 1.4142135623730951, + "learning_rate": 2.7271339096575073e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 3840 + }, + { + "epoch": 2.057869208897809, + "grad_norm": 1.4142135623730951, + "learning_rate": 2.713273026244446e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 3845 + }, + { + "epoch": 2.0605452416792107, + "grad_norm": 1.4142135623730951, + "learning_rate": 2.6994343286665156e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 3850 + }, + { + "epoch": 2.063221274460612, + "grad_norm": 1.4142135623730951, + "learning_rate": 2.6856179511868156e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 3855 + }, + { + "epoch": 2.0658973072420137, + "grad_norm": 1.4142135623730951, + "learning_rate": 2.6718240278519056e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 3860 + }, + { + "epoch": 2.0685733400234154, + "grad_norm": 1.4142135623730951, + "learning_rate": 2.6580526924904866e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 3865 + }, + { + "epoch": 2.0712493728048167, + "grad_norm": 1.4142135623730951, + "learning_rate": 2.6443040787121186e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 3870 + }, + { + "epoch": 2.0739254055862184, + "grad_norm": 1.4142135623730951, + "learning_rate": 2.6305783199059084e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 3875 + }, + { + "epoch": 2.07660143836762, + "grad_norm": 1.4142135623730951, + "learning_rate": 2.6168755492392324e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 3880 + }, + { + "epoch": 2.0792774711490214, + "grad_norm": 1.4142135623730951, + "learning_rate": 2.6031958996564274e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 3885 + }, + { + "epoch": 2.081953503930423, + "grad_norm": 1.4142135623730951, + "learning_rate": 2.589539503877518e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 3890 + }, + { + "epoch": 2.084629536711825, + "grad_norm": 1.4142135623730951, + "learning_rate": 2.5759064943969125e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 3895 + }, + { + "epoch": 2.087305569493226, + "grad_norm": 1.4142135623730951, + "learning_rate": 2.562297003482131e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 3900 + }, + { + "epoch": 2.089981602274628, + "grad_norm": 1.4142135623730951, + "learning_rate": 2.548711163172512e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 3905 + }, + { + "epoch": 2.0926576350560295, + "grad_norm": 1.4142135623730951, + "learning_rate": 2.53514910527794e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 3910 + }, + { + "epoch": 2.095333667837431, + "grad_norm": 1.4142135623730951, + "learning_rate": 2.5216109613775573e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 3915 + }, + { + "epoch": 2.0980097006188325, + "grad_norm": 1.4142135623730951, + "learning_rate": 2.5080968628184993e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 3920 + }, + { + "epoch": 2.1006857334002342, + "grad_norm": 1.4142135623730951, + "learning_rate": 2.494606940714605e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 3925 + }, + { + "epoch": 2.103361766181636, + "grad_norm": 1.4142135623730951, + "learning_rate": 2.4811413259451625e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 3930 + }, + { + "epoch": 2.106037798963037, + "grad_norm": 1.4142135623730951, + "learning_rate": 2.46770014915362e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 3935 + }, + { + "epoch": 2.108713831744439, + "grad_norm": 1.4142135623730951, + "learning_rate": 2.45428354074634e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 3940 + }, + { + "epoch": 2.1113898645258407, + "grad_norm": 1.4142135623730951, + "learning_rate": 2.4408916308913105e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 3945 + }, + { + "epoch": 2.114065897307242, + "grad_norm": 1.4142135623730951, + "learning_rate": 2.4275245495169025e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 3950 + }, + { + "epoch": 2.1167419300886436, + "grad_norm": 1.4142135623730951, + "learning_rate": 2.414182426310597e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 3955 + }, + { + "epoch": 2.1194179628700454, + "grad_norm": 1.4142135623730951, + "learning_rate": 2.400865390717734e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 3960 + }, + { + "epoch": 2.1220939956514466, + "grad_norm": 1.4142135623730951, + "learning_rate": 2.3875735719402475e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 3965 + }, + { + "epoch": 2.1247700284328483, + "grad_norm": 1.4142135623730951, + "learning_rate": 2.3743070989354258e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 3970 + }, + { + "epoch": 2.12744606121425, + "grad_norm": 1.4142135623730951, + "learning_rate": 2.3610661004146454e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 3975 + }, + { + "epoch": 2.1301220939956513, + "grad_norm": 1.4142135623730951, + "learning_rate": 2.3478507048421314e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 3980 + }, + { + "epoch": 2.132798126777053, + "grad_norm": 1.4142135623730951, + "learning_rate": 2.334661040433713e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 3985 + }, + { + "epoch": 2.1354741595584548, + "grad_norm": 1.4142135623730951, + "learning_rate": 2.321497235155568e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 3990 + }, + { + "epoch": 2.138150192339856, + "grad_norm": 1.4142135623730951, + "learning_rate": 2.3083594167229965e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 3995 + }, + { + "epoch": 2.1408262251212578, + "grad_norm": 1.4142135623730951, + "learning_rate": 2.295247712599167e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 4000 + }, + { + "epoch": 2.1408262251212578, + "eval_logits/chosen": NaN, + "eval_logits/rejected": NaN, + "eval_logps/chosen": NaN, + "eval_logps/rejected": NaN, + "eval_loss": NaN, + "eval_rewards/accuracies": 0.0, + "eval_rewards/chosen": NaN, + "eval_rewards/margins": NaN, + "eval_rewards/rejected": NaN, + "eval_runtime": 35.0743, + "eval_samples_per_second": 38.347, + "eval_semantic_entropy": NaN, + "eval_steps_per_second": 9.608, + "step": 4000 + }, + { + "epoch": 2.1435022579026595, + "grad_norm": 1.4142135623730951, + "learning_rate": 2.2821622499938948e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 4005 + }, + { + "epoch": 2.1461782906840607, + "grad_norm": 1.4142135623730951, + "learning_rate": 2.269103155862391e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 4010 + }, + { + "epoch": 2.1488543234654625, + "grad_norm": 1.4142135623730951, + "learning_rate": 2.2560705569040483e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 4015 + }, + { + "epoch": 2.151530356246864, + "grad_norm": 1.4142135623730951, + "learning_rate": 2.2430645795611963e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 4020 + }, + { + "epoch": 2.1542063890282654, + "grad_norm": 1.4142135623730951, + "learning_rate": 2.230085350017884e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 4025 + }, + { + "epoch": 2.156882421809667, + "grad_norm": 1.4142135623730951, + "learning_rate": 2.2171329941986554e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 4030 + }, + { + "epoch": 2.159558454591069, + "grad_norm": 1.4142135623730951, + "learning_rate": 2.2042076377673202e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 4035 + }, + { + "epoch": 2.16223448737247, + "grad_norm": 1.4142135623730951, + "learning_rate": 2.1913094061257476e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 4040 + }, + { + "epoch": 2.164910520153872, + "grad_norm": 1.4142135623730951, + "learning_rate": 2.178438424412633e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 4045 + }, + { + "epoch": 2.1675865529352736, + "grad_norm": 1.4142135623730951, + "learning_rate": 2.165594817502302e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 4050 + }, + { + "epoch": 2.170262585716675, + "grad_norm": 1.4142135623730951, + "learning_rate": 2.1527787100034806e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 4055 + }, + { + "epoch": 2.1729386184980766, + "grad_norm": 1.4142135623730951, + "learning_rate": 2.1399902262581037e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 4060 + }, + { + "epoch": 2.1756146512794783, + "grad_norm": 1.4142135623730951, + "learning_rate": 2.127229490340094e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 4065 + }, + { + "epoch": 2.1782906840608796, + "grad_norm": 1.4142135623730951, + "learning_rate": 2.1144966260541698e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 4070 + }, + { + "epoch": 2.1809667168422813, + "grad_norm": 1.4142135623730951, + "learning_rate": 2.1017917569346332e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 4075 + }, + { + "epoch": 2.183642749623683, + "grad_norm": 1.4142135623730951, + "learning_rate": 2.0891150062441837e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 4080 + }, + { + "epoch": 2.1863187824050843, + "grad_norm": 1.4142135623730951, + "learning_rate": 2.0764664969727086e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 4085 + }, + { + "epoch": 2.188994815186486, + "grad_norm": 1.4142135623730951, + "learning_rate": 2.0638463518361033e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 4090 + }, + { + "epoch": 2.1916708479678877, + "grad_norm": 1.4142135623730951, + "learning_rate": 2.0512546932750702e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 4095 + }, + { + "epoch": 2.194346880749289, + "grad_norm": 1.4142135623730951, + "learning_rate": 2.0386916434539343e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 4100 + }, + { + "epoch": 2.1970229135306907, + "grad_norm": 1.4142135623730951, + "learning_rate": 2.0261573242594627e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 4105 + }, + { + "epoch": 2.1996989463120924, + "grad_norm": 1.4142135623730951, + "learning_rate": 2.0136518572996724e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 4110 + }, + { + "epoch": 2.202374979093494, + "grad_norm": 1.4142135623730951, + "learning_rate": 2.0011753639026617e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 4115 + }, + { + "epoch": 2.2050510118748954, + "grad_norm": 1.4142135623730951, + "learning_rate": 1.988727965115421e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 4120 + }, + { + "epoch": 2.207727044656297, + "grad_norm": 1.4142135623730951, + "learning_rate": 1.9763097817026713e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 4125 + }, + { + "epoch": 2.210403077437699, + "grad_norm": 1.4142135623730951, + "learning_rate": 1.9639209341456796e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 4130 + }, + { + "epoch": 2.2130791102191, + "grad_norm": 1.4142135623730951, + "learning_rate": 1.951561542641102e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 4135 + }, + { + "epoch": 2.215755143000502, + "grad_norm": 1.4142135623730951, + "learning_rate": 1.939231727099806e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 4140 + }, + { + "epoch": 2.2184311757819035, + "grad_norm": 1.4142135623730951, + "learning_rate": 1.926931607145719e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 4145 + }, + { + "epoch": 2.221107208563305, + "grad_norm": 1.4142135623730951, + "learning_rate": 1.9146613021146564e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 4150 + }, + { + "epoch": 2.2237832413447065, + "grad_norm": 1.4142135623730951, + "learning_rate": 1.9024209310531736e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 4155 + }, + { + "epoch": 2.2264592741261082, + "grad_norm": 1.4142135623730951, + "learning_rate": 1.890210612717401e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 4160 + }, + { + "epoch": 2.2291353069075095, + "grad_norm": 1.4142135623730951, + "learning_rate": 1.8780304655719054e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 4165 + }, + { + "epoch": 2.231811339688911, + "grad_norm": 1.4142135623730951, + "learning_rate": 1.865880607788523e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 4170 + }, + { + "epoch": 2.234487372470313, + "grad_norm": 1.4142135623730951, + "learning_rate": 1.8537611572452316e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 4175 + }, + { + "epoch": 2.237163405251714, + "grad_norm": 1.4142135623730951, + "learning_rate": 1.84167223152499e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 4180 + }, + { + "epoch": 2.239839438033116, + "grad_norm": 1.4142135623730951, + "learning_rate": 1.8296139479146112e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 4185 + }, + { + "epoch": 2.2425154708145176, + "grad_norm": 1.4142135623730951, + "learning_rate": 1.8175864234036132e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 4190 + }, + { + "epoch": 2.245191503595919, + "grad_norm": 1.4142135623730951, + "learning_rate": 1.805589774683094e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 4195 + }, + { + "epoch": 2.2478675363773206, + "grad_norm": 1.4142135623730951, + "learning_rate": 1.79362411814459e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 4200 + }, + { + "epoch": 2.2505435691587223, + "grad_norm": 1.4142135623730951, + "learning_rate": 1.7816895698789552e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 4205 + }, + { + "epoch": 2.2532196019401236, + "grad_norm": 1.4142135623730951, + "learning_rate": 1.7697862456752271e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 4210 + }, + { + "epoch": 2.2558956347215253, + "grad_norm": 1.4142135623730951, + "learning_rate": 1.7579142610195124e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 4215 + }, + { + "epoch": 2.258571667502927, + "grad_norm": 1.4142135623730951, + "learning_rate": 1.7460737310938568e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 4220 + }, + { + "epoch": 2.2612477002843283, + "grad_norm": 1.4142135623730951, + "learning_rate": 1.734264770775133e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 4225 + }, + { + "epoch": 2.26392373306573, + "grad_norm": 1.4142135623730951, + "learning_rate": 1.7224874946339241e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 4230 + }, + { + "epoch": 2.2665997658471317, + "grad_norm": 1.4142135623730951, + "learning_rate": 1.7107420169334186e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 4235 + }, + { + "epoch": 2.269275798628533, + "grad_norm": 1.4142135623730951, + "learning_rate": 1.6990284516282893e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 4240 + }, + { + "epoch": 2.2719518314099347, + "grad_norm": 1.4142135623730951, + "learning_rate": 1.687346912363602e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 4245 + }, + { + "epoch": 2.2746278641913364, + "grad_norm": 1.4142135623730951, + "learning_rate": 1.675697512473697e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 4250 + }, + { + "epoch": 2.2773038969727377, + "grad_norm": 1.4142135623730951, + "learning_rate": 1.6640803649811087e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 4255 + }, + { + "epoch": 2.2799799297541394, + "grad_norm": 1.4142135623730951, + "learning_rate": 1.6524955825954472e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 4260 + }, + { + "epoch": 2.282655962535541, + "grad_norm": 1.4142135623730951, + "learning_rate": 1.6409432777123277e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 4265 + }, + { + "epoch": 2.285331995316943, + "grad_norm": 1.4142135623730951, + "learning_rate": 1.6294235624122577e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 4270 + }, + { + "epoch": 2.288008028098344, + "grad_norm": 1.4142135623730951, + "learning_rate": 1.6179365484595697e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 4275 + }, + { + "epoch": 2.290684060879746, + "grad_norm": 1.4142135623730951, + "learning_rate": 1.60648234730132e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 4280 + }, + { + "epoch": 2.293360093661147, + "grad_norm": 1.4142135623730951, + "learning_rate": 1.595061070066222e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 4285 + }, + { + "epoch": 2.296036126442549, + "grad_norm": 1.4142135623730951, + "learning_rate": 1.5836728275635542e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 4290 + }, + { + "epoch": 2.2987121592239506, + "grad_norm": 1.4142135623730951, + "learning_rate": 1.5723177302820984e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 4295 + }, + { + "epoch": 2.3013881920053523, + "grad_norm": 1.4142135623730951, + "learning_rate": 1.5609958883890544e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 4300 + }, + { + "epoch": 2.3040642247867535, + "grad_norm": 1.4142135623730951, + "learning_rate": 1.5497074117289865e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 4305 + }, + { + "epoch": 2.3067402575681553, + "grad_norm": 1.4142135623730951, + "learning_rate": 1.5384524098227402e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 4310 + }, + { + "epoch": 2.3094162903495565, + "grad_norm": 1.4142135623730951, + "learning_rate": 1.5272309918663974e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 4315 + }, + { + "epoch": 2.3120923231309582, + "grad_norm": 1.4142135623730951, + "learning_rate": 1.516043266730201e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 4320 + }, + { + "epoch": 2.31476835591236, + "grad_norm": 1.4142135623730951, + "learning_rate": 1.504889342957512e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 4325 + }, + { + "epoch": 2.3174443886937617, + "grad_norm": 1.4142135623730951, + "learning_rate": 1.4937693287637453e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 4330 + }, + { + "epoch": 2.320120421475163, + "grad_norm": 1.4142135623730951, + "learning_rate": 1.4826833320353305e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 4335 + }, + { + "epoch": 2.3227964542565647, + "grad_norm": 1.4142135623730951, + "learning_rate": 1.4716314603286528e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 4340 + }, + { + "epoch": 2.3254724870379664, + "grad_norm": 1.4142135623730951, + "learning_rate": 1.4606138208690233e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 4345 + }, + { + "epoch": 2.3281485198193677, + "grad_norm": 1.4142135623730951, + "learning_rate": 1.4496305205496251e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 4350 + }, + { + "epoch": 2.3308245526007694, + "grad_norm": 1.4142135623730951, + "learning_rate": 1.4386816659304895e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 4355 + }, + { + "epoch": 2.333500585382171, + "grad_norm": 1.4142135623730951, + "learning_rate": 1.4277673632374492e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 4360 + }, + { + "epoch": 2.3361766181635724, + "grad_norm": 1.4142135623730951, + "learning_rate": 1.416887718361119e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 4365 + }, + { + "epoch": 2.338852650944974, + "grad_norm": 1.4142135623730951, + "learning_rate": 1.406042836855859e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 4370 + }, + { + "epoch": 2.341528683726376, + "grad_norm": 1.4142135623730951, + "learning_rate": 1.3952328239387595e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 4375 + }, + { + "epoch": 2.344204716507777, + "grad_norm": 1.4142135623730951, + "learning_rate": 1.3844577844886109e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 4380 + }, + { + "epoch": 2.346880749289179, + "grad_norm": 1.4142135623730951, + "learning_rate": 1.3737178230448955e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 4385 + }, + { + "epoch": 2.3495567820705805, + "grad_norm": 1.4142135623730951, + "learning_rate": 1.363013043806764e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 4390 + }, + { + "epoch": 2.3522328148519818, + "grad_norm": 1.4142135623730951, + "learning_rate": 1.352343550632034e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 4395 + }, + { + "epoch": 2.3549088476333835, + "grad_norm": 1.4142135623730951, + "learning_rate": 1.3417094470361722e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 4400 + }, + { + "epoch": 2.3549088476333835, + "eval_logits/chosen": NaN, + "eval_logits/rejected": NaN, + "eval_logps/chosen": NaN, + "eval_logps/rejected": NaN, + "eval_loss": NaN, + "eval_rewards/accuracies": 0.0, + "eval_rewards/chosen": NaN, + "eval_rewards/margins": NaN, + "eval_rewards/rejected": NaN, + "eval_runtime": 34.9067, + "eval_samples_per_second": 38.531, + "eval_semantic_entropy": NaN, + "eval_steps_per_second": 9.654, + "step": 4400 + }, + { + "epoch": 2.357584880414785, + "grad_norm": 1.4142135623730951, + "learning_rate": 1.3311108361913015e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 4405 + }, + { + "epoch": 2.3602609131961865, + "grad_norm": 1.4142135623730951, + "learning_rate": 1.3205478209251874e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 4410 + }, + { + "epoch": 2.362936945977588, + "grad_norm": 1.4142135623730951, + "learning_rate": 1.310020503720254e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 4415 + }, + { + "epoch": 2.36561297875899, + "grad_norm": 1.4142135623730951, + "learning_rate": 1.2995289867125752e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 4420 + }, + { + "epoch": 2.368289011540391, + "grad_norm": 1.4142135623730951, + "learning_rate": 1.2890733716908986e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 4425 + }, + { + "epoch": 2.370965044321793, + "grad_norm": 1.4142135623730951, + "learning_rate": 1.2786537600956454e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 4430 + }, + { + "epoch": 2.3736410771031946, + "grad_norm": 1.4142135623730951, + "learning_rate": 1.268270253017933e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 4435 + }, + { + "epoch": 2.376317109884596, + "grad_norm": 1.4142135623730951, + "learning_rate": 1.257922951198591e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 4440 + }, + { + "epoch": 2.3789931426659976, + "grad_norm": 1.4142135623730951, + "learning_rate": 1.24761195502719e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 4445 + }, + { + "epoch": 2.3816691754473993, + "grad_norm": 1.4142135623730951, + "learning_rate": 1.2373373645410573e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 4450 + }, + { + "epoch": 2.384345208228801, + "grad_norm": 1.4142135623730951, + "learning_rate": 1.2270992794243175e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 4455 + }, + { + "epoch": 2.3870212410102023, + "grad_norm": 1.4142135623730951, + "learning_rate": 1.2168977990069147e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 4460 + }, + { + "epoch": 2.389697273791604, + "grad_norm": 1.4142135623730951, + "learning_rate": 1.206733022263659e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 4465 + }, + { + "epoch": 2.3923733065730053, + "grad_norm": 1.4142135623730951, + "learning_rate": 1.1966050478132572e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 4470 + }, + { + "epoch": 2.395049339354407, + "grad_norm": 1.4142135623730951, + "learning_rate": 1.1865139739173635e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 4475 + }, + { + "epoch": 2.3977253721358087, + "grad_norm": 1.4142135623730951, + "learning_rate": 1.1764598984796187e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 4480 + }, + { + "epoch": 2.4004014049172104, + "grad_norm": 1.4142135623730951, + "learning_rate": 1.1664429190447095e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 4485 + }, + { + "epoch": 2.4030774376986117, + "grad_norm": 1.4142135623730951, + "learning_rate": 1.1564631327974122e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 4490 + }, + { + "epoch": 2.4057534704800134, + "grad_norm": 1.4142135623730951, + "learning_rate": 1.1465206365616587e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 4495 + }, + { + "epoch": 2.408429503261415, + "grad_norm": 1.4142135623730951, + "learning_rate": 1.1366155267995887e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 4500 + }, + { + "epoch": 2.4111055360428164, + "grad_norm": 1.4142135623730951, + "learning_rate": 1.1267478996106228e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 4505 + }, + { + "epoch": 2.413781568824218, + "grad_norm": 1.4142135623730951, + "learning_rate": 1.116917850730521e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 4510 + }, + { + "epoch": 2.41645760160562, + "grad_norm": 1.4142135623730951, + "learning_rate": 1.1071254755304637e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 4515 + }, + { + "epoch": 2.419133634387021, + "grad_norm": 1.4142135623730951, + "learning_rate": 1.0973708690161143e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 4520 + }, + { + "epoch": 2.421809667168423, + "grad_norm": 1.4142135623730951, + "learning_rate": 1.0876541258267119e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 4525 + }, + { + "epoch": 2.4244856999498245, + "grad_norm": 1.4142135623730951, + "learning_rate": 1.0779753402341379e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 4530 + }, + { + "epoch": 2.427161732731226, + "grad_norm": 1.4142135623730951, + "learning_rate": 1.0683346061420157e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 4535 + }, + { + "epoch": 2.4298377655126275, + "grad_norm": 1.4142135623730951, + "learning_rate": 1.0587320170847874e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 4540 + }, + { + "epoch": 2.4325137982940293, + "grad_norm": 1.4142135623730951, + "learning_rate": 1.0491676662268156e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 4545 + }, + { + "epoch": 2.4351898310754305, + "grad_norm": 1.4142135623730951, + "learning_rate": 1.0396416463614732e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 4550 + }, + { + "epoch": 2.4378658638568322, + "grad_norm": 1.4142135623730951, + "learning_rate": 1.0301540499102479e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 4555 + }, + { + "epoch": 2.440541896638234, + "grad_norm": 1.4142135623730951, + "learning_rate": 1.0207049689218405e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 4560 + }, + { + "epoch": 2.4432179294196352, + "grad_norm": 1.4142135623730951, + "learning_rate": 1.0112944950712782e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 4565 + }, + { + "epoch": 2.445893962201037, + "grad_norm": 1.4142135623730951, + "learning_rate": 1.0019227196590174e-07, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 4570 + }, + { + "epoch": 2.4485699949824387, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.925897336100664e-08, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 4575 + }, + { + "epoch": 2.45124602776384, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.832956274730946e-08, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 4580 + }, + { + "epoch": 2.4539220605452416, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.740404914195633e-08, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 4585 + }, + { + "epoch": 2.4565980933266434, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.648244152428392e-08, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 4590 + }, + { + "epoch": 2.4592741261080446, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.556474883573379e-08, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 4595 + }, + { + "epoch": 2.4619501588894463, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.465097997976412e-08, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 4600 + }, + { + "epoch": 2.464626191670848, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.374114382176457e-08, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 4605 + }, + { + "epoch": 2.46730222445225, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.283524918896945e-08, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 4610 + }, + { + "epoch": 2.469978257233651, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.193330487037232e-08, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 4615 + }, + { + "epoch": 2.4726542900150528, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.103531961664118e-08, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 4620 + }, + { + "epoch": 2.475330322796454, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.014130214003269e-08, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 4625 + }, + { + "epoch": 2.4780063555778558, + "grad_norm": 1.4142135623730951, + "learning_rate": 8.925126111430848e-08, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 4630 + }, + { + "epoch": 2.4806823883592575, + "grad_norm": 1.4142135623730951, + "learning_rate": 8.83652051746504e-08, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 4635 + }, + { + "epoch": 2.483358421140659, + "grad_norm": 1.4142135623730951, + "learning_rate": 8.748314291757696e-08, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 4640 + }, + { + "epoch": 2.4860344539220605, + "grad_norm": 1.4142135623730951, + "learning_rate": 8.660508290086032e-08, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 4645 + }, + { + "epoch": 2.488710486703462, + "grad_norm": 1.4142135623730951, + "learning_rate": 8.573103364344231e-08, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 4650 + }, + { + "epoch": 2.4913865194848634, + "grad_norm": 1.4142135623730951, + "learning_rate": 8.486100362535292e-08, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 4655 + }, + { + "epoch": 2.494062552266265, + "grad_norm": 1.4142135623730951, + "learning_rate": 8.399500128762693e-08, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 4660 + }, + { + "epoch": 2.496738585047667, + "grad_norm": 1.4142135623730951, + "learning_rate": 8.313303503222313e-08, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 4665 + }, + { + "epoch": 2.4994146178290686, + "grad_norm": 1.4142135623730951, + "learning_rate": 8.227511322194164e-08, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 4670 + }, + { + "epoch": 2.50209065061047, + "grad_norm": 1.4142135623730951, + "learning_rate": 8.142124418034385e-08, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 4675 + }, + { + "epoch": 2.5047666833918716, + "grad_norm": 1.4142135623730951, + "learning_rate": 8.057143619167073e-08, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 4680 + }, + { + "epoch": 2.507442716173273, + "grad_norm": 1.4142135623730951, + "learning_rate": 7.97256975007633e-08, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 4685 + }, + { + "epoch": 2.5101187489546746, + "grad_norm": 1.4142135623730951, + "learning_rate": 7.888403631298186e-08, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 4690 + }, + { + "epoch": 2.5127947817360763, + "grad_norm": 1.4142135623730951, + "learning_rate": 7.804646079412719e-08, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 4695 + }, + { + "epoch": 2.515470814517478, + "grad_norm": 1.4142135623730951, + "learning_rate": 7.72129790703604e-08, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 4700 + }, + { + "epoch": 2.5181468472988793, + "grad_norm": 1.4142135623730951, + "learning_rate": 7.638359922812504e-08, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 4705 + }, + { + "epoch": 2.520822880080281, + "grad_norm": 1.4142135623730951, + "learning_rate": 7.555832931406774e-08, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 4710 + }, + { + "epoch": 2.5234989128616827, + "grad_norm": 1.4142135623730951, + "learning_rate": 7.47371773349611e-08, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 4715 + }, + { + "epoch": 2.526174945643084, + "grad_norm": 1.4142135623730951, + "learning_rate": 7.392015125762496e-08, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 4720 + }, + { + "epoch": 2.5288509784244857, + "grad_norm": 1.4142135623730951, + "learning_rate": 7.310725900885018e-08, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 4725 + }, + { + "epoch": 2.5315270112058874, + "grad_norm": 1.4142135623730951, + "learning_rate": 7.229850847532076e-08, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 4730 + }, + { + "epoch": 2.5342030439872887, + "grad_norm": 1.4142135623730951, + "learning_rate": 7.149390750353779e-08, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 4735 + }, + { + "epoch": 2.5368790767686904, + "grad_norm": 1.4142135623730951, + "learning_rate": 7.069346389974374e-08, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 4740 + }, + { + "epoch": 2.539555109550092, + "grad_norm": 1.4142135623730951, + "learning_rate": 6.989718542984563e-08, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 4745 + }, + { + "epoch": 2.5422311423314934, + "grad_norm": 1.4142135623730951, + "learning_rate": 6.9105079819341e-08, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 4750 + }, + { + "epoch": 2.544907175112895, + "grad_norm": 1.4142135623730951, + "learning_rate": 6.831715475324163e-08, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 4755 + }, + { + "epoch": 2.547583207894297, + "grad_norm": 1.4142135623730951, + "learning_rate": 6.753341787600026e-08, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 4760 + }, + { + "epoch": 2.5502592406756985, + "grad_norm": 1.4142135623730951, + "learning_rate": 6.67538767914353e-08, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 4765 + }, + { + "epoch": 2.5529352734571, + "grad_norm": 1.4142135623730951, + "learning_rate": 6.597853906265793e-08, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 4770 + }, + { + "epoch": 2.5556113062385015, + "grad_norm": 1.4142135623730951, + "learning_rate": 6.5207412211998e-08, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 4775 + }, + { + "epoch": 2.558287339019903, + "grad_norm": 1.4142135623730951, + "learning_rate": 6.444050372093186e-08, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 4780 + }, + { + "epoch": 2.5609633718013045, + "grad_norm": 1.4142135623730951, + "learning_rate": 6.367782103000873e-08, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 4785 + }, + { + "epoch": 2.5636394045827062, + "grad_norm": 1.4142135623730951, + "learning_rate": 6.29193715387798e-08, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 4790 + }, + { + "epoch": 2.566315437364108, + "grad_norm": 1.4142135623730951, + "learning_rate": 6.216516260572502e-08, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 4795 + }, + { + "epoch": 2.568991470145509, + "grad_norm": 1.4142135623730951, + "learning_rate": 6.141520154818297e-08, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 4800 + }, + { + "epoch": 2.568991470145509, + "eval_logits/chosen": NaN, + "eval_logits/rejected": NaN, + "eval_logps/chosen": NaN, + "eval_logps/rejected": NaN, + "eval_loss": NaN, + "eval_rewards/accuracies": 0.0, + "eval_rewards/chosen": NaN, + "eval_rewards/margins": NaN, + "eval_rewards/rejected": NaN, + "eval_runtime": 34.7803, + "eval_samples_per_second": 38.671, + "eval_semantic_entropy": NaN, + "eval_steps_per_second": 9.689, + "step": 4800 + }, + { + "epoch": 2.571667502926911, + "grad_norm": 1.4142135623730951, + "learning_rate": 6.066949564227897e-08, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 4805 + }, + { + "epoch": 2.574343535708312, + "grad_norm": 1.4142135623730951, + "learning_rate": 5.992805212285523e-08, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 4810 + }, + { + "epoch": 2.577019568489714, + "grad_norm": 1.4142135623730951, + "learning_rate": 5.9190878183399684e-08, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 4815 + }, + { + "epoch": 2.5796956012711156, + "grad_norm": 1.4142135623730951, + "learning_rate": 5.845798097597748e-08, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 4820 + }, + { + "epoch": 2.5823716340525174, + "grad_norm": 1.4142135623730951, + "learning_rate": 5.772936761116026e-08, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 4825 + }, + { + "epoch": 2.5850476668339186, + "grad_norm": 1.4142135623730951, + "learning_rate": 5.700504515795829e-08, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 4830 + }, + { + "epoch": 2.5877236996153203, + "grad_norm": 1.4142135623730951, + "learning_rate": 5.628502064375101e-08, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 4835 + }, + { + "epoch": 2.5903997323967216, + "grad_norm": 1.4142135623730951, + "learning_rate": 5.55693010542197e-08, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 4840 + }, + { + "epoch": 2.5930757651781233, + "grad_norm": 1.4142135623730951, + "learning_rate": 5.485789333327856e-08, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 4845 + }, + { + "epoch": 2.595751797959525, + "grad_norm": 1.4142135623730951, + "learning_rate": 5.4150804383008675e-08, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 4850 + }, + { + "epoch": 2.5984278307409268, + "grad_norm": 1.4142135623730951, + "learning_rate": 5.344804106359002e-08, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 4855 + }, + { + "epoch": 2.601103863522328, + "grad_norm": 1.4142135623730951, + "learning_rate": 5.274961019323559e-08, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 4860 + }, + { + "epoch": 2.6037798963037297, + "grad_norm": 1.4142135623730951, + "learning_rate": 5.205551854812451e-08, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 4865 + }, + { + "epoch": 2.606455929085131, + "grad_norm": 1.4142135623730951, + "learning_rate": 5.1365772862337177e-08, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 4870 + }, + { + "epoch": 2.6091319618665327, + "grad_norm": 1.4142135623730951, + "learning_rate": 5.068037982778905e-08, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 4875 + }, + { + "epoch": 2.6118079946479344, + "grad_norm": 1.4142135623730951, + "learning_rate": 4.999934609416656e-08, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 4880 + }, + { + "epoch": 2.614484027429336, + "grad_norm": 1.4142135623730951, + "learning_rate": 4.932267826886183e-08, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 4885 + }, + { + "epoch": 2.6171600602107374, + "grad_norm": 1.4142135623730951, + "learning_rate": 4.8650382916909206e-08, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 4890 + }, + { + "epoch": 2.619836092992139, + "grad_norm": 1.4142135623730951, + "learning_rate": 4.7982466560920976e-08, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 4895 + }, + { + "epoch": 2.622512125773541, + "grad_norm": 1.4142135623730951, + "learning_rate": 4.7318935681024685e-08, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 4900 + }, + { + "epoch": 2.625188158554942, + "grad_norm": 1.4142135623730951, + "learning_rate": 4.6659796714799745e-08, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 4905 + }, + { + "epoch": 2.627864191336344, + "grad_norm": 1.4142135623730951, + "learning_rate": 4.60050560572155e-08, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 4910 + }, + { + "epoch": 2.6305402241177456, + "grad_norm": 1.4142135623730951, + "learning_rate": 4.535472006056834e-08, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 4915 + }, + { + "epoch": 2.6332162568991473, + "grad_norm": 1.4142135623730951, + "learning_rate": 4.470879503442132e-08, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 4920 + }, + { + "epoch": 2.6358922896805486, + "grad_norm": 1.4142135623730951, + "learning_rate": 4.406728724554154e-08, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 4925 + }, + { + "epoch": 2.6385683224619503, + "grad_norm": 1.4142135623730951, + "learning_rate": 4.3430202917840664e-08, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 4930 + }, + { + "epoch": 2.6412443552433515, + "grad_norm": 1.4142135623730951, + "learning_rate": 4.279754823231346e-08, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 4935 + }, + { + "epoch": 2.6439203880247533, + "grad_norm": 1.4142135623730951, + "learning_rate": 4.216932932697859e-08, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 4940 + }, + { + "epoch": 2.646596420806155, + "grad_norm": 1.4142135623730951, + "learning_rate": 4.154555229681844e-08, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 4945 + }, + { + "epoch": 2.6492724535875567, + "grad_norm": 1.4142135623730951, + "learning_rate": 4.092622319372069e-08, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 4950 + }, + { + "epoch": 2.651948486368958, + "grad_norm": 1.4142135623730951, + "learning_rate": 4.031134802641889e-08, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 4955 + }, + { + "epoch": 2.6546245191503597, + "grad_norm": 1.4142135623730951, + "learning_rate": 3.970093276043468e-08, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 4960 + }, + { + "epoch": 2.657300551931761, + "grad_norm": 1.4142135623730951, + "learning_rate": 3.9094983318019584e-08, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 4965 + }, + { + "epoch": 2.6599765847131627, + "grad_norm": 1.4142135623730951, + "learning_rate": 3.849350557809789e-08, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 4970 + }, + { + "epoch": 2.6626526174945644, + "grad_norm": 1.4142135623730951, + "learning_rate": 3.789650537620903e-08, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 4975 + }, + { + "epoch": 2.665328650275966, + "grad_norm": 1.4142135623730951, + "learning_rate": 3.730398850445182e-08, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 4980 + }, + { + "epoch": 2.6680046830573674, + "grad_norm": 1.4142135623730951, + "learning_rate": 3.671596071142735e-08, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 4985 + }, + { + "epoch": 2.670680715838769, + "grad_norm": 1.4142135623730951, + "learning_rate": 3.6132427702183996e-08, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 4990 + }, + { + "epoch": 2.6733567486201704, + "grad_norm": 1.4142135623730951, + "learning_rate": 3.555339513816147e-08, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 4995 + }, + { + "epoch": 2.676032781401572, + "grad_norm": 1.4142135623730951, + "learning_rate": 3.497886863713639e-08, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 5000 + }, + { + "epoch": 2.678708814182974, + "grad_norm": 1.4142135623730951, + "learning_rate": 3.440885377316721e-08, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 5005 + }, + { + "epoch": 2.6813848469643755, + "grad_norm": 1.4142135623730951, + "learning_rate": 3.384335607654082e-08, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 5010 + }, + { + "epoch": 2.684060879745777, + "grad_norm": 1.4142135623730951, + "learning_rate": 3.328238103371811e-08, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 5015 + }, + { + "epoch": 2.6867369125271785, + "grad_norm": 1.4142135623730951, + "learning_rate": 3.272593408728169e-08, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 5020 + }, + { + "epoch": 2.6894129453085798, + "grad_norm": 1.4142135623730951, + "learning_rate": 3.217402063588204e-08, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 5025 + }, + { + "epoch": 2.6920889780899815, + "grad_norm": 1.4142135623730951, + "learning_rate": 3.162664603418608e-08, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 5030 + }, + { + "epoch": 2.694765010871383, + "grad_norm": 1.4142135623730951, + "learning_rate": 3.1083815592824416e-08, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 5035 + }, + { + "epoch": 2.697441043652785, + "grad_norm": 1.4142135623730951, + "learning_rate": 3.054553457834053e-08, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 5040 + }, + { + "epoch": 2.700117076434186, + "grad_norm": 1.4142135623730951, + "learning_rate": 3.0011808213139036e-08, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 5045 + }, + { + "epoch": 2.702793109215588, + "grad_norm": 1.4142135623730951, + "learning_rate": 2.948264167543568e-08, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 5050 + }, + { + "epoch": 2.7054691419969896, + "grad_norm": 1.4142135623730951, + "learning_rate": 2.8958040099206216e-08, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 5055 + }, + { + "epoch": 2.708145174778391, + "grad_norm": 1.4142135623730951, + "learning_rate": 2.843800857413775e-08, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 5060 + }, + { + "epoch": 2.7108212075597926, + "grad_norm": 1.4142135623730951, + "learning_rate": 2.7922552145578203e-08, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 5065 + }, + { + "epoch": 2.7134972403411943, + "grad_norm": 1.4142135623730951, + "learning_rate": 2.7411675814488277e-08, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 5070 + }, + { + "epoch": 2.7161732731225956, + "grad_norm": 1.4142135623730951, + "learning_rate": 2.690538453739216e-08, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 5075 + }, + { + "epoch": 2.7188493059039973, + "grad_norm": 1.4142135623730951, + "learning_rate": 2.6403683226330298e-08, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 5080 + }, + { + "epoch": 2.721525338685399, + "grad_norm": 1.4142135623730951, + "learning_rate": 2.5906576748810804e-08, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 5085 + }, + { + "epoch": 2.7242013714668003, + "grad_norm": 1.4142135623730951, + "learning_rate": 2.5414069927763016e-08, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 5090 + }, + { + "epoch": 2.726877404248202, + "grad_norm": 1.4142135623730951, + "learning_rate": 2.4926167541490185e-08, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 5095 + }, + { + "epoch": 2.7295534370296037, + "grad_norm": 1.4142135623730951, + "learning_rate": 2.4442874323623574e-08, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 5100 + }, + { + "epoch": 2.7322294698110055, + "grad_norm": 1.4142135623730951, + "learning_rate": 2.396419496307589e-08, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 5105 + }, + { + "epoch": 2.7349055025924067, + "grad_norm": 1.4142135623730951, + "learning_rate": 2.349013410399653e-08, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 5110 + }, + { + "epoch": 2.7375815353738084, + "grad_norm": 1.4142135623730951, + "learning_rate": 2.3020696345725954e-08, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 5115 + }, + { + "epoch": 2.7402575681552097, + "grad_norm": 1.4142135623730951, + "learning_rate": 2.2555886242751398e-08, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 5120 + }, + { + "epoch": 2.7429336009366114, + "grad_norm": 1.4142135623730951, + "learning_rate": 2.2095708304662453e-08, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 5125 + }, + { + "epoch": 2.745609633718013, + "grad_norm": 1.4142135623730951, + "learning_rate": 2.16401669961076e-08, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 5130 + }, + { + "epoch": 2.748285666499415, + "grad_norm": 1.4142135623730951, + "learning_rate": 2.1189266736750532e-08, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 5135 + }, + { + "epoch": 2.750961699280816, + "grad_norm": 1.4142135623730951, + "learning_rate": 2.0743011901227623e-08, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 5140 + }, + { + "epoch": 2.753637732062218, + "grad_norm": 1.4142135623730951, + "learning_rate": 2.030140681910508e-08, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 5145 + }, + { + "epoch": 2.756313764843619, + "grad_norm": 1.4142135623730951, + "learning_rate": 1.986445577483753e-08, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 5150 + }, + { + "epoch": 2.758989797625021, + "grad_norm": 1.4142135623730951, + "learning_rate": 1.9432163007725765e-08, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 5155 + }, + { + "epoch": 2.7616658304064226, + "grad_norm": 1.4142135623730951, + "learning_rate": 1.9004532711876297e-08, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 5160 + }, + { + "epoch": 2.7643418631878243, + "grad_norm": 1.4142135623730951, + "learning_rate": 1.8581569036159928e-08, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 5165 + }, + { + "epoch": 2.7670178959692255, + "grad_norm": 1.4142135623730951, + "learning_rate": 1.8163276084172285e-08, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 5170 + }, + { + "epoch": 2.7696939287506273, + "grad_norm": 1.4142135623730951, + "learning_rate": 1.7749657914193194e-08, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 5175 + }, + { + "epoch": 2.7723699615320285, + "grad_norm": 1.4142135623730951, + "learning_rate": 1.7340718539148203e-08, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 5180 + }, + { + "epoch": 2.7750459943134302, + "grad_norm": 1.4142135623730951, + "learning_rate": 1.6936461926568724e-08, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 5185 + }, + { + "epoch": 2.777722027094832, + "grad_norm": 1.4142135623730951, + "learning_rate": 1.6536891998554346e-08, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 5190 + }, + { + "epoch": 2.7803980598762337, + "grad_norm": 1.4142135623730951, + "learning_rate": 1.6142012631734093e-08, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 5195 + }, + { + "epoch": 2.783074092657635, + "grad_norm": 1.4142135623730951, + "learning_rate": 1.575182765722949e-08, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 5200 + }, + { + "epoch": 2.783074092657635, + "eval_logits/chosen": NaN, + "eval_logits/rejected": NaN, + "eval_logps/chosen": NaN, + "eval_logps/rejected": NaN, + "eval_loss": NaN, + "eval_rewards/accuracies": 0.0, + "eval_rewards/chosen": NaN, + "eval_rewards/margins": NaN, + "eval_rewards/rejected": NaN, + "eval_runtime": 34.8842, + "eval_samples_per_second": 38.556, + "eval_semantic_entropy": NaN, + "eval_steps_per_second": 9.661, + "step": 5200 + }, + { + "epoch": 2.7857501254390367, + "grad_norm": 1.4142135623730951, + "learning_rate": 1.536634086061672e-08, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 5205 + }, + { + "epoch": 2.788426158220438, + "grad_norm": 1.4142135623730951, + "learning_rate": 1.4985555981890495e-08, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 5210 + }, + { + "epoch": 2.7911021910018396, + "grad_norm": 1.4142135623730951, + "learning_rate": 1.4609476715427226e-08, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 5215 + }, + { + "epoch": 2.7937782237832414, + "grad_norm": 1.4142135623730951, + "learning_rate": 1.4238106709949792e-08, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 5220 + }, + { + "epoch": 2.796454256564643, + "grad_norm": 1.4142135623730951, + "learning_rate": 1.3871449568491511e-08, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 5225 + }, + { + "epoch": 2.7991302893460444, + "grad_norm": 1.4142135623730951, + "learning_rate": 1.3509508848361606e-08, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 5230 + }, + { + "epoch": 2.801806322127446, + "grad_norm": 1.4142135623730951, + "learning_rate": 1.3152288061110517e-08, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 5235 + }, + { + "epoch": 2.804482354908848, + "grad_norm": 1.4142135623730951, + "learning_rate": 1.2799790672495814e-08, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 5240 + }, + { + "epoch": 2.807158387690249, + "grad_norm": 1.4142135623730951, + "learning_rate": 1.2452020102448835e-08, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 5245 + }, + { + "epoch": 2.8098344204716508, + "grad_norm": 1.4142135623730951, + "learning_rate": 1.2108979725041103e-08, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 5250 + }, + { + "epoch": 2.8125104532530525, + "grad_norm": 1.4142135623730951, + "learning_rate": 1.1770672868451958e-08, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 5255 + }, + { + "epoch": 2.8151864860344538, + "grad_norm": 1.4142135623730951, + "learning_rate": 1.1437102814935872e-08, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 5260 + }, + { + "epoch": 2.8178625188158555, + "grad_norm": 1.4142135623730951, + "learning_rate": 1.1108272800791018e-08, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 5265 + }, + { + "epoch": 2.820538551597257, + "grad_norm": 1.4142135623730951, + "learning_rate": 1.078418601632769e-08, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 5270 + }, + { + "epoch": 2.8232145843786585, + "grad_norm": 1.4142135623730951, + "learning_rate": 1.0464845605837159e-08, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 5275 + }, + { + "epoch": 2.82589061716006, + "grad_norm": 1.4142135623730951, + "learning_rate": 1.0150254667561642e-08, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 5280 + }, + { + "epoch": 2.828566649941462, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.840416253663719e-09, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 5285 + }, + { + "epoch": 2.8312426827228636, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.535333370197074e-09, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 5290 + }, + { + "epoch": 2.833918715504265, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.23500897707713e-09, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 5295 + }, + { + "epoch": 2.8365947482856666, + "grad_norm": 1.4142135623730951, + "learning_rate": 8.939445988052574e-09, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 5300 + }, + { + "epoch": 2.839270781067068, + "grad_norm": 1.4142135623730951, + "learning_rate": 8.648647270676656e-09, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 5305 + }, + { + "epoch": 2.8419468138484696, + "grad_norm": 1.4142135623730951, + "learning_rate": 8.362615646279991e-09, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 5310 + }, + { + "epoch": 2.8446228466298713, + "grad_norm": 1.4142135623730951, + "learning_rate": 8.081353889942466e-09, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 5315 + }, + { + "epoch": 2.847298879411273, + "grad_norm": 1.4142135623730951, + "learning_rate": 7.804864730467042e-09, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 5320 + }, + { + "epoch": 2.8499749121926743, + "grad_norm": 1.4142135623730951, + "learning_rate": 7.533150850352665e-09, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 5325 + }, + { + "epoch": 2.852650944974076, + "grad_norm": 1.4142135623730951, + "learning_rate": 7.2662148857686175e-09, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 5330 + }, + { + "epoch": 2.8553269777554773, + "grad_norm": 1.4142135623730951, + "learning_rate": 7.0040594265287635e-09, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 5335 + }, + { + "epoch": 2.858003010536879, + "grad_norm": 1.4142135623730951, + "learning_rate": 6.746687016066566e-09, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 5340 + }, + { + "epoch": 2.8606790433182807, + "grad_norm": 1.4142135623730951, + "learning_rate": 6.494100151410276e-09, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 5345 + }, + { + "epoch": 2.8633550760996824, + "grad_norm": 1.4142135623730951, + "learning_rate": 6.246301283158728e-09, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 5350 + }, + { + "epoch": 2.8660311088810837, + "grad_norm": 1.4142135623730951, + "learning_rate": 6.0032928154576944e-09, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 5355 + }, + { + "epoch": 2.8687071416624854, + "grad_norm": 1.4142135623730951, + "learning_rate": 5.76507710597629e-09, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 5360 + }, + { + "epoch": 2.8713831744438867, + "grad_norm": 1.4142135623730951, + "learning_rate": 5.531656465884438e-09, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 5365 + }, + { + "epoch": 2.8740592072252884, + "grad_norm": 1.4142135623730951, + "learning_rate": 5.303033159830217e-09, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 5370 + }, + { + "epoch": 2.87673524000669, + "grad_norm": 1.4142135623730951, + "learning_rate": 5.079209405917939e-09, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 5375 + }, + { + "epoch": 2.879411272788092, + "grad_norm": 1.4142135623730951, + "learning_rate": 4.860187375686664e-09, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 5380 + }, + { + "epoch": 2.882087305569493, + "grad_norm": 1.4142135623730951, + "learning_rate": 4.64596919408905e-09, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 5385 + }, + { + "epoch": 2.884763338350895, + "grad_norm": 1.4142135623730951, + "learning_rate": 4.436556939470814e-09, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 5390 + }, + { + "epoch": 2.887439371132296, + "grad_norm": 1.4142135623730951, + "learning_rate": 4.23195264355064e-09, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 5395 + }, + { + "epoch": 2.890115403913698, + "grad_norm": 1.4142135623730951, + "learning_rate": 4.032158291400245e-09, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 5400 + }, + { + "epoch": 2.8927914366950995, + "grad_norm": 1.4142135623730951, + "learning_rate": 3.837175821425398e-09, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 5405 + }, + { + "epoch": 2.8954674694765012, + "grad_norm": 1.4142135623730951, + "learning_rate": 3.6470071253467683e-09, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 5410 + }, + { + "epoch": 2.8981435022579025, + "grad_norm": 1.4142135623730951, + "learning_rate": 3.461654048181939e-09, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 5415 + }, + { + "epoch": 2.9008195350393042, + "grad_norm": 1.4142135623730951, + "learning_rate": 3.281118388227255e-09, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 5420 + }, + { + "epoch": 2.903495567820706, + "grad_norm": 1.4142135623730951, + "learning_rate": 3.1054018970405048e-09, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 5425 + }, + { + "epoch": 2.906171600602107, + "grad_norm": 1.4142135623730951, + "learning_rate": 2.9345062794238207e-09, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 5430 + }, + { + "epoch": 2.908847633383509, + "grad_norm": 1.4142135623730951, + "learning_rate": 2.7684331934072492e-09, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 5435 + }, + { + "epoch": 2.9115236661649107, + "grad_norm": 1.4142135623730951, + "learning_rate": 2.6071842502326526e-09, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 5440 + }, + { + "epoch": 2.9141996989463124, + "grad_norm": 1.4142135623730951, + "learning_rate": 2.450761014337888e-09, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 5445 + }, + { + "epoch": 2.9168757317277136, + "grad_norm": 1.4142135623730951, + "learning_rate": 2.299165003341985e-09, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 5450 + }, + { + "epoch": 2.9195517645091154, + "grad_norm": 1.4142135623730951, + "learning_rate": 2.1523976880299945e-09, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 5455 + }, + { + "epoch": 2.9222277972905166, + "grad_norm": 1.4142135623730951, + "learning_rate": 2.010460492339161e-09, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 5460 + }, + { + "epoch": 2.9249038300719183, + "grad_norm": 1.4142135623730951, + "learning_rate": 1.8733547933446614e-09, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 5465 + }, + { + "epoch": 2.92757986285332, + "grad_norm": 1.4142135623730951, + "learning_rate": 1.7410819212467231e-09, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 5470 + }, + { + "epoch": 2.9302558956347218, + "grad_norm": 1.4142135623730951, + "learning_rate": 1.613643159357192e-09, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 5475 + }, + { + "epoch": 2.932931928416123, + "grad_norm": 1.4142135623730951, + "learning_rate": 1.4910397440875967e-09, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 5480 + }, + { + "epoch": 2.9356079611975248, + "grad_norm": 1.4142135623730951, + "learning_rate": 1.3732728649368253e-09, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 5485 + }, + { + "epoch": 2.938283993978926, + "grad_norm": 1.4142135623730951, + "learning_rate": 1.260343664479524e-09, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 5490 + }, + { + "epoch": 2.9409600267603278, + "grad_norm": 1.4142135623730951, + "learning_rate": 1.1522532383554384e-09, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 5495 + }, + { + "epoch": 2.9436360595417295, + "grad_norm": 1.4142135623730951, + "learning_rate": 1.049002635258256e-09, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 5500 + }, + { + "epoch": 2.946312092323131, + "grad_norm": 1.4142135623730951, + "learning_rate": 9.505928569258358e-10, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 5505 + }, + { + "epoch": 2.9489881251045325, + "grad_norm": 1.4142135623730951, + "learning_rate": 8.57024858130273e-10, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 5510 + }, + { + "epoch": 2.951664157885934, + "grad_norm": 1.4142135623730951, + "learning_rate": 7.682995466686826e-10, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 5515 + }, + { + "epoch": 2.9543401906673354, + "grad_norm": 1.4142135623730951, + "learning_rate": 6.844177833543741e-10, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 5520 + }, + { + "epoch": 2.957016223448737, + "grad_norm": 1.4142135623730951, + "learning_rate": 6.053803820087467e-10, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 5525 + }, + { + "epoch": 2.959692256230139, + "grad_norm": 1.4142135623730951, + "learning_rate": 5.311881094528514e-10, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 5530 + }, + { + "epoch": 2.9623682890115406, + "grad_norm": 1.4142135623730951, + "learning_rate": 4.6184168550050806e-10, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 5535 + }, + { + "epoch": 2.965044321792942, + "grad_norm": 1.4142135623730951, + "learning_rate": 3.973417829510328e-10, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 5540 + }, + { + "epoch": 2.9677203545743436, + "grad_norm": 1.4142135623730951, + "learning_rate": 3.3768902758274377e-10, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 5545 + }, + { + "epoch": 2.970396387355745, + "grad_norm": 1.4142135623730951, + "learning_rate": 2.8288399814691e-10, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 5550 + }, + { + "epoch": 2.9730724201371466, + "grad_norm": 1.4142135623730951, + "learning_rate": 2.3292722636220066e-10, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 5555 + }, + { + "epoch": 2.9757484529185483, + "grad_norm": 1.4142135623730951, + "learning_rate": 1.8781919690946668e-10, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 5560 + }, + { + "epoch": 2.97842448569995, + "grad_norm": 1.4142135623730951, + "learning_rate": 1.4756034742696711e-10, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 5565 + }, + { + "epoch": 2.9811005184813513, + "grad_norm": 1.4142135623730951, + "learning_rate": 1.12151068506261e-10, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 5570 + }, + { + "epoch": 2.983776551262753, + "grad_norm": 1.4142135623730951, + "learning_rate": 8.159170368826629e-11, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 5575 + }, + { + "epoch": 2.9864525840441547, + "grad_norm": 1.4142135623730951, + "learning_rate": 5.588254946015114e-11, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 5580 + }, + { + "epoch": 2.989128616825556, + "grad_norm": 1.4142135623730951, + "learning_rate": 3.502385525216978e-11, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 5585 + }, + { + "epoch": 2.9918046496069577, + "grad_norm": 1.4142135623730951, + "learning_rate": 1.901582343555308e-11, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 5590 + }, + { + "epoch": 2.9944806823883594, + "grad_norm": 1.4142135623730951, + "learning_rate": 7.858609320232634e-12, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 5595 + }, + { + "epoch": 2.9971567151697607, + "grad_norm": 1.4142135623730951, + "learning_rate": 1.5523211535639624e-12, + "logits/chosen": NaN, + "logits/rejected": NaN, + "logps/chosen": NaN, + "logps/rejected": NaN, + "loss": 0.0, + "rewards/accuracies": 0.0, + "rewards/chosen": NaN, + "rewards/margins": NaN, + "rewards/rejected": NaN, + "semantic_entropy": NaN, + "step": 5600 + }, + { + "epoch": 2.9971567151697607, + "eval_logits/chosen": NaN, + "eval_logits/rejected": NaN, + "eval_logps/chosen": NaN, + "eval_logps/rejected": NaN, + "eval_loss": NaN, + "eval_rewards/accuracies": 0.0, + "eval_rewards/chosen": NaN, + "eval_rewards/margins": NaN, + "eval_rewards/rejected": NaN, + "eval_runtime": 34.7507, + "eval_samples_per_second": 38.704, + "eval_semantic_entropy": NaN, + "eval_steps_per_second": 9.698, + "step": 5600 + }, + { + "epoch": 2.999297541394882, + "step": 5604, + "total_flos": 0.0, + "train_loss": 0.024124226235730746, + "train_runtime": 28949.9436, + "train_samples_per_second": 6.196, + "train_steps_per_second": 0.194 + } + ], + "logging_steps": 5, + "max_steps": 5604, + "num_input_tokens_seen": 0, + "num_train_epochs": 3, + "save_steps": 1000000, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": true + }, + "attributes": {} + } + }, + "total_flos": 0.0, + "train_batch_size": 2, + "trial_name": null, + "trial_params": null +}