diff --git "a/trainer_state.json" "b/trainer_state.json"
--- "a/trainer_state.json"
+++ "b/trainer_state.json"
@@ -10,18170 +10,17036 @@
   "log_history": [
     {
       "epoch": 0.002676032781401572,
-      "grad_norm": 6.283021946330416,
+      "grad_norm": 4.174769404980618,
       "learning_rate": 8.9126559714795e-09,
-      "logits/chosen": -0.05665075033903122,
-      "logits/rejected": 0.15200912952423096,
-      "logps/chosen": -1.7162984609603882,
-      "logps/rejected": -1.889739990234375,
-      "loss": 0.5728,
+      "logits/chosen": -0.056673564016819,
+      "logits/rejected": 0.15200209617614746,
+      "logps/chosen": -1.7160017490386963,
+      "logps/rejected": -1.8894321918487549,
+      "loss": 0.2436,
       "rewards/accuracies": 0.5687500238418579,
-      "rewards/chosen": -1.7162984609603882,
-      "rewards/margins": 0.17344175279140472,
-      "rewards/rejected": -1.889739990234375,
-      "semantic_entropy": 0.6583643555641174,
+      "rewards/chosen": -1.7160017490386963,
+      "rewards/margins": 0.17343027889728546,
+      "rewards/rejected": -1.8894321918487549,
       "step": 5
     },
     {
       "epoch": 0.005352065562803144,
-      "grad_norm": 5.209798699126062,
+      "grad_norm": 2.8626364007215304,
       "learning_rate": 1.7825311942959e-08,
-      "logits/chosen": 0.010867366567254066,
-      "logits/rejected": 0.13097676634788513,
-      "logps/chosen": -1.8017022609710693,
-      "logps/rejected": -1.846153974533081,
-      "loss": 0.5698,
-      "rewards/accuracies": 0.5562499761581421,
-      "rewards/chosen": -1.8017022609710693,
-      "rewards/margins": 0.04445166885852814,
-      "rewards/rejected": -1.846153974533081,
-      "semantic_entropy": 0.6395805478096008,
+      "logits/chosen": 0.015492829494178295,
+      "logits/rejected": 0.1357966512441635,
+      "logps/chosen": -1.8026939630508423,
+      "logps/rejected": -1.846070647239685,
+      "loss": 0.25,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -1.8026939630508423,
+      "rewards/margins": 0.043376728892326355,
+      "rewards/rejected": -1.846070647239685,
       "step": 10
     },
     {
       "epoch": 0.008028098344204716,
-      "grad_norm": 5.586761639648108,
+      "grad_norm": 3.044428420168379,
       "learning_rate": 2.67379679144385e-08,
-      "logits/chosen": -0.03902262821793556,
-      "logits/rejected": 0.05713306739926338,
-      "logps/chosen": -1.6349042654037476,
-      "logps/rejected": -1.7645162343978882,
-      "loss": 0.6338,
+      "logits/chosen": -0.03407303988933563,
+      "logits/rejected": 0.06202689930796623,
+      "logps/chosen": -1.6353429555892944,
+      "logps/rejected": -1.765667200088501,
+      "loss": 0.287,
       "rewards/accuracies": 0.4937500059604645,
-      "rewards/chosen": -1.6349042654037476,
-      "rewards/margins": 0.1296120136976242,
-      "rewards/rejected": -1.7645162343978882,
-      "semantic_entropy": 0.6933382749557495,
+      "rewards/chosen": -1.6353429555892944,
+      "rewards/margins": 0.13032427430152893,
+      "rewards/rejected": -1.765667200088501,
       "step": 15
     },
     {
       "epoch": 0.010704131125606288,
-      "grad_norm": 6.56499151474477,
+      "grad_norm": 3.8909506975846084,
       "learning_rate": 3.5650623885918e-08,
-      "logits/chosen": -0.0506710521876812,
-      "logits/rejected": 0.031738776713609695,
-      "logps/chosen": -1.7259511947631836,
-      "logps/rejected": -1.8071794509887695,
-      "loss": 0.6122,
-      "rewards/accuracies": 0.48750001192092896,
-      "rewards/chosen": -1.7259511947631836,
-      "rewards/margins": 0.08122824132442474,
-      "rewards/rejected": -1.8071794509887695,
-      "semantic_entropy": 0.6682224869728088,
+      "logits/chosen": -0.049756430089473724,
+      "logits/rejected": 0.0334530733525753,
+      "logps/chosen": -1.7236230373382568,
+      "logps/rejected": -1.8058974742889404,
+      "loss": 0.2784,
+      "rewards/accuracies": 0.4937500059604645,
+      "rewards/chosen": -1.7236230373382568,
+      "rewards/margins": 0.08227436244487762,
+      "rewards/rejected": -1.8058974742889404,
       "step": 20
     },
     {
       "epoch": 0.013380163907007862,
-      "grad_norm": 6.873589227199821,
+      "grad_norm": 3.7378465518883943,
       "learning_rate": 4.45632798573975e-08,
-      "logits/chosen": -0.049850888550281525,
-      "logits/rejected": 0.03644446283578873,
-      "logps/chosen": -1.8693091869354248,
-      "logps/rejected": -1.779761552810669,
-      "loss": 0.6281,
-      "rewards/accuracies": 0.375,
-      "rewards/chosen": -1.8693091869354248,
-      "rewards/margins": -0.08954761922359467,
-      "rewards/rejected": -1.779761552810669,
-      "semantic_entropy": 0.6430644392967224,
+      "logits/chosen": -0.05973156541585922,
+      "logits/rejected": 0.024282705038785934,
+      "logps/chosen": -1.8695319890975952,
+      "logps/rejected": -1.7784847021102905,
+      "loss": 0.3067,
+      "rewards/accuracies": 0.3812499940395355,
+      "rewards/chosen": -1.8695319890975952,
+      "rewards/margins": -0.09104712307453156,
+      "rewards/rejected": -1.7784847021102905,
       "step": 25
     },
     {
       "epoch": 0.016056196688409432,
-      "grad_norm": 6.0582854700474815,
+      "grad_norm": 2.9924518409881413,
       "learning_rate": 5.3475935828877e-08,
-      "logits/chosen": -0.09021981805562973,
-      "logits/rejected": 0.00014630556688643992,
-      "logps/chosen": -1.9092113971710205,
-      "logps/rejected": -1.832843542098999,
-      "loss": 0.5563,
+      "logits/chosen": -0.07078413665294647,
+      "logits/rejected": 0.02136463299393654,
+      "logps/chosen": -1.907697319984436,
+      "logps/rejected": -1.8313062191009521,
+      "loss": 0.2477,
       "rewards/accuracies": 0.4375,
-      "rewards/chosen": -1.9092113971710205,
-      "rewards/margins": -0.07636785507202148,
-      "rewards/rejected": -1.832843542098999,
-      "semantic_entropy": 0.6176734566688538,
+      "rewards/chosen": -1.907697319984436,
+      "rewards/margins": -0.07639118283987045,
+      "rewards/rejected": -1.8313062191009521,
       "step": 30
     },
     {
       "epoch": 0.018732229469811006,
-      "grad_norm": 6.953077360616749,
+      "grad_norm": 3.9037635937417243,
       "learning_rate": 6.23885918003565e-08,
-      "logits/chosen": -0.058985866606235504,
-      "logits/rejected": 0.09778233617544174,
-      "logps/chosen": -1.8485629558563232,
-      "logps/rejected": -1.9991296529769897,
-      "loss": 0.5799,
-      "rewards/accuracies": 0.48750001192092896,
-      "rewards/chosen": -1.8485629558563232,
-      "rewards/margins": 0.15056660771369934,
-      "rewards/rejected": -1.9991296529769897,
-      "semantic_entropy": 0.6344074010848999,
+      "logits/chosen": -0.05222669243812561,
+      "logits/rejected": 0.10753919184207916,
+      "logps/chosen": -1.8471834659576416,
+      "logps/rejected": -1.9986279010772705,
+      "loss": 0.2626,
+      "rewards/accuracies": 0.4937500059604645,
+      "rewards/chosen": -1.8471834659576416,
+      "rewards/margins": 0.15144455432891846,
+      "rewards/rejected": -1.9986279010772705,
       "step": 35
     },
     {
       "epoch": 0.021408262251212576,
-      "grad_norm": 5.95861907781093,
+      "grad_norm": 3.260061146078127,
       "learning_rate": 7.1301247771836e-08,
-      "logits/chosen": 0.028498858213424683,
-      "logits/rejected": 0.20293541252613068,
-      "logps/chosen": -1.8856210708618164,
-      "logps/rejected": -1.7465416193008423,
-      "loss": 0.599,
+      "logits/chosen": 0.04427720606327057,
+      "logits/rejected": 0.21839460730552673,
+      "logps/chosen": -1.886055588722229,
+      "logps/rejected": -1.7470966577529907,
+      "loss": 0.2781,
       "rewards/accuracies": 0.45625001192092896,
-      "rewards/chosen": -1.8856210708618164,
-      "rewards/margins": -0.13907942175865173,
-      "rewards/rejected": -1.7465416193008423,
-      "semantic_entropy": 0.6422087550163269,
+      "rewards/chosen": -1.886055588722229,
+      "rewards/margins": -0.1389588713645935,
+      "rewards/rejected": -1.7470966577529907,
       "step": 40
     },
     {
       "epoch": 0.02408429503261415,
-      "grad_norm": 6.724726630089442,
+      "grad_norm": 3.7651938269189533,
       "learning_rate": 8.021390374331551e-08,
-      "logits/chosen": 0.014616099186241627,
-      "logits/rejected": 0.2100570946931839,
-      "logps/chosen": -1.8413318395614624,
-      "logps/rejected": -1.8763275146484375,
-      "loss": 0.5937,
-      "rewards/accuracies": 0.48750001192092896,
-      "rewards/chosen": -1.8413318395614624,
-      "rewards/margins": 0.03499563783407211,
-      "rewards/rejected": -1.8763275146484375,
-      "semantic_entropy": 0.6482739448547363,
+      "logits/chosen": 0.019146930426359177,
+      "logits/rejected": 0.21615441143512726,
+      "logps/chosen": -1.8423458337783813,
+      "logps/rejected": -1.8769302368164062,
+      "loss": 0.2696,
+      "rewards/accuracies": 0.4937500059604645,
+      "rewards/chosen": -1.8423458337783813,
+      "rewards/margins": 0.03458467125892639,
+      "rewards/rejected": -1.8769302368164062,
       "step": 45
     },
     {
       "epoch": 0.026760327814015723,
-      "grad_norm": 7.060246294252313,
+      "grad_norm": 3.566135479934334,
       "learning_rate": 8.9126559714795e-08,
-      "logits/chosen": -0.03206641972064972,
-      "logits/rejected": 0.11953778564929962,
-      "logps/chosen": -1.9085075855255127,
-      "logps/rejected": -1.7868010997772217,
-      "loss": 0.5802,
-      "rewards/accuracies": 0.512499988079071,
-      "rewards/chosen": -1.9085075855255127,
-      "rewards/margins": -0.12170647084712982,
-      "rewards/rejected": -1.7868010997772217,
-      "semantic_entropy": 0.6314837336540222,
+      "logits/chosen": -0.050375960767269135,
+      "logits/rejected": 0.09876345098018646,
+      "logps/chosen": -1.9080318212509155,
+      "logps/rejected": -1.7855631113052368,
+      "loss": 0.2647,
+      "rewards/accuracies": 0.5062500238418579,
+      "rewards/chosen": -1.9080318212509155,
+      "rewards/margins": -0.12246876955032349,
+      "rewards/rejected": -1.7855631113052368,
       "step": 50
     },
     {
       "epoch": 0.029436360595417294,
-      "grad_norm": 5.9296298405276495,
+      "grad_norm": 3.2091482239918308,
       "learning_rate": 9.80392156862745e-08,
-      "logits/chosen": -0.11400948464870453,
-      "logits/rejected": 0.10222546011209488,
-      "logps/chosen": -1.8476674556732178,
-      "logps/rejected": -1.880591630935669,
-      "loss": 0.572,
-      "rewards/accuracies": 0.5562499761581421,
-      "rewards/chosen": -1.8476674556732178,
-      "rewards/margins": 0.03292415291070938,
-      "rewards/rejected": -1.880591630935669,
-      "semantic_entropy": 0.6409212946891785,
+      "logits/chosen": -0.10335006564855576,
+      "logits/rejected": 0.11803705990314484,
+      "logps/chosen": -1.8465286493301392,
+      "logps/rejected": -1.8804638385772705,
+      "loss": 0.2517,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -1.8465286493301392,
+      "rewards/margins": 0.03393523767590523,
+      "rewards/rejected": -1.8804638385772705,
       "step": 55
     },
     {
       "epoch": 0.032112393376818864,
-      "grad_norm": 6.192609903998256,
+      "grad_norm": 3.3346635910473887,
       "learning_rate": 1.06951871657754e-07,
-      "logits/chosen": -0.06294813007116318,
-      "logits/rejected": 0.12941911816596985,
-      "logps/chosen": -1.8072830438613892,
-      "logps/rejected": -1.9127168655395508,
-      "loss": 0.5598,
-      "rewards/accuracies": 0.5249999761581421,
-      "rewards/chosen": -1.8072830438613892,
-      "rewards/margins": 0.10543377697467804,
-      "rewards/rejected": -1.9127168655395508,
-      "semantic_entropy": 0.6318107843399048,
+      "logits/chosen": -0.09414264559745789,
+      "logits/rejected": 0.09447915107011795,
+      "logps/chosen": -1.8043931722640991,
+      "logps/rejected": -1.9090369939804077,
+      "loss": 0.2443,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": -1.8043931722640991,
+      "rewards/margins": 0.10464388132095337,
+      "rewards/rejected": -1.9090369939804077,
       "step": 60
     },
     {
       "epoch": 0.03478842615822044,
-      "grad_norm": 5.840367536650179,
+      "grad_norm": 3.1364454101958548,
       "learning_rate": 1.158645276292335e-07,
-      "logits/chosen": -0.031631387770175934,
-      "logits/rejected": 0.11730644851922989,
-      "logps/chosen": -1.6562246084213257,
-      "logps/rejected": -1.7897517681121826,
-      "loss": 0.6131,
-      "rewards/accuracies": 0.518750011920929,
-      "rewards/chosen": -1.6562246084213257,
-      "rewards/margins": 0.13352727890014648,
-      "rewards/rejected": -1.7897517681121826,
-      "semantic_entropy": 0.691199004650116,
+      "logits/chosen": -0.02966163121163845,
+      "logits/rejected": 0.11632075160741806,
+      "logps/chosen": -1.6551532745361328,
+      "logps/rejected": -1.788581132888794,
+      "loss": 0.2675,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -1.6551532745361328,
+      "rewards/margins": 0.13342775404453278,
+      "rewards/rejected": -1.788581132888794,
       "step": 65
     },
     {
       "epoch": 0.03746445893962201,
-      "grad_norm": 7.339621823064037,
+      "grad_norm": 4.538924165904268,
       "learning_rate": 1.24777183600713e-07,
-      "logits/chosen": -0.07941316813230515,
-      "logits/rejected": 0.07183218747377396,
-      "logps/chosen": -1.793076753616333,
-      "logps/rejected": -1.8430589437484741,
-      "loss": 0.5936,
-      "rewards/accuracies": 0.42500001192092896,
-      "rewards/chosen": -1.793076753616333,
-      "rewards/margins": 0.04998219758272171,
-      "rewards/rejected": -1.8430589437484741,
-      "semantic_entropy": 0.6468250155448914,
+      "logits/chosen": -0.06675219535827637,
+      "logits/rejected": 0.085882268846035,
+      "logps/chosen": -1.7921323776245117,
+      "logps/rejected": -1.8404977321624756,
+      "loss": 0.2704,
+      "rewards/accuracies": 0.41874998807907104,
+      "rewards/chosen": -1.7921323776245117,
+      "rewards/margins": 0.04836542159318924,
+      "rewards/rejected": -1.8404977321624756,
       "step": 70
     },
     {
       "epoch": 0.04014049172102358,
-      "grad_norm": 5.431450354661162,
+      "grad_norm": 2.8899566254010987,
       "learning_rate": 1.3368983957219251e-07,
-      "logits/chosen": -0.0522337444126606,
-      "logits/rejected": 0.13029292225837708,
-      "logps/chosen": -1.8287160396575928,
-      "logps/rejected": -2.097831964492798,
-      "loss": 0.5472,
+      "logits/chosen": -0.04987819120287895,
+      "logits/rejected": 0.1338050663471222,
+      "logps/chosen": -1.822100043296814,
+      "logps/rejected": -2.090280771255493,
+      "loss": 0.238,
       "rewards/accuracies": 0.5249999761581421,
-      "rewards/chosen": -1.8287160396575928,
-      "rewards/margins": 0.2691158354282379,
-      "rewards/rejected": -2.097831964492798,
-      "semantic_entropy": 0.6201809048652649,
+      "rewards/chosen": -1.822100043296814,
+      "rewards/margins": 0.26818060874938965,
+      "rewards/rejected": -2.090280771255493,
       "step": 75
     },
     {
       "epoch": 0.04281652450242515,
-      "grad_norm": 5.120322608884211,
+      "grad_norm": 2.7037524466902587,
       "learning_rate": 1.42602495543672e-07,
-      "logits/chosen": 0.00244101881980896,
-      "logits/rejected": 0.10615000873804092,
-      "logps/chosen": -1.7778211832046509,
-      "logps/rejected": -1.809617280960083,
-      "loss": 0.5924,
-      "rewards/accuracies": 0.512499988079071,
-      "rewards/chosen": -1.7778211832046509,
-      "rewards/margins": 0.03179623559117317,
-      "rewards/rejected": -1.809617280960083,
-      "semantic_entropy": 0.6536020040512085,
+      "logits/chosen": 0.008540439419448376,
+      "logits/rejected": 0.11213047802448273,
+      "logps/chosen": -1.7725322246551514,
+      "logps/rejected": -1.8043262958526611,
+      "loss": 0.2662,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": -1.7725322246551514,
+      "rewards/margins": 0.03179407864809036,
+      "rewards/rejected": -1.8043262958526611,
       "step": 80
     },
     {
       "epoch": 0.04549255728382673,
-      "grad_norm": 4.807337181522284,
+      "grad_norm": 2.753588303143848,
       "learning_rate": 1.5151515151515152e-07,
-      "logits/chosen": -0.14471609890460968,
-      "logits/rejected": 0.10519156605005264,
-      "logps/chosen": -1.8619205951690674,
-      "logps/rejected": -2.0526270866394043,
-      "loss": 0.5784,
+      "logits/chosen": -0.14115747809410095,
+      "logits/rejected": 0.11033763736486435,
+      "logps/chosen": -1.8558228015899658,
+      "logps/rejected": -2.0468955039978027,
+      "loss": 0.2628,
       "rewards/accuracies": 0.5062500238418579,
-      "rewards/chosen": -1.8619205951690674,
-      "rewards/margins": 0.19070668518543243,
-      "rewards/rejected": -2.0526270866394043,
-      "semantic_entropy": 0.6318272352218628,
+      "rewards/chosen": -1.8558228015899658,
+      "rewards/margins": 0.19107282161712646,
+      "rewards/rejected": -2.0468955039978027,
       "step": 85
     },
     {
       "epoch": 0.0481685900652283,
-      "grad_norm": 5.097287930626076,
+      "grad_norm": 2.4810036120937773,
       "learning_rate": 1.6042780748663102e-07,
-      "logits/chosen": 0.09117968380451202,
-      "logits/rejected": 0.05077249929308891,
-      "logps/chosen": -1.8373451232910156,
-      "logps/rejected": -1.8451576232910156,
-      "loss": 0.5988,
-      "rewards/accuracies": 0.4749999940395355,
-      "rewards/chosen": -1.8373451232910156,
-      "rewards/margins": 0.007812516763806343,
-      "rewards/rejected": -1.8451576232910156,
-      "semantic_entropy": 0.6506969332695007,
+      "logits/chosen": 0.09177587926387787,
+      "logits/rejected": 0.05044828727841377,
+      "logps/chosen": -1.8309383392333984,
+      "logps/rejected": -1.8402868509292603,
+      "loss": 0.2741,
+      "rewards/accuracies": 0.46875,
+      "rewards/chosen": -1.8309383392333984,
+      "rewards/margins": 0.009348212741315365,
+      "rewards/rejected": -1.8402868509292603,
       "step": 90
     },
     {
       "epoch": 0.05084462284662987,
-      "grad_norm": 4.648043327968244,
+      "grad_norm": 2.5214005774541346,
       "learning_rate": 1.693404634581105e-07,
-      "logits/chosen": -0.06766053289175034,
-      "logits/rejected": 0.08066412061452866,
-      "logps/chosen": -1.9248930215835571,
-      "logps/rejected": -2.0033318996429443,
-      "loss": 0.5561,
+      "logits/chosen": -0.05386476591229439,
+      "logits/rejected": 0.09711028635501862,
+      "logps/chosen": -1.915281891822815,
+      "logps/rejected": -1.996042251586914,
+      "loss": 0.2471,
       "rewards/accuracies": 0.581250011920929,
-      "rewards/chosen": -1.9248930215835571,
-      "rewards/margins": 0.07843902707099915,
-      "rewards/rejected": -2.0033318996429443,
-      "semantic_entropy": 0.6197448968887329,
+      "rewards/chosen": -1.915281891822815,
+      "rewards/margins": 0.08076085150241852,
+      "rewards/rejected": -1.996042251586914,
       "step": 95
     },
     {
       "epoch": 0.05352065562803145,
-      "grad_norm": 4.21712461471473,
+      "grad_norm": 2.386081290272195,
       "learning_rate": 1.7825311942959e-07,
-      "logits/chosen": -0.04044175148010254,
-      "logits/rejected": 0.021123576909303665,
-      "logps/chosen": -1.8103749752044678,
-      "logps/rejected": -1.9236066341400146,
-      "loss": 0.567,
-      "rewards/accuracies": 0.5,
-      "rewards/chosen": -1.8103749752044678,
-      "rewards/margins": 0.11323173344135284,
-      "rewards/rejected": -1.9236066341400146,
-      "semantic_entropy": 0.6433764696121216,
+      "logits/chosen": -0.04361366480588913,
+      "logits/rejected": 0.017345154657959938,
+      "logps/chosen": -1.7915188074111938,
+      "logps/rejected": -1.9044063091278076,
+      "loss": 0.2475,
+      "rewards/accuracies": 0.5062500238418579,
+      "rewards/chosen": -1.7915188074111938,
+      "rewards/margins": 0.11288763582706451,
+      "rewards/rejected": -1.9044063091278076,
       "step": 100
     },
     {
       "epoch": 0.05619668840943302,
-      "grad_norm": 4.152220901781052,
+      "grad_norm": 2.3355814398098738,
       "learning_rate": 1.8716577540106952e-07,
-      "logits/chosen": 0.061221588402986526,
-      "logits/rejected": 0.08823797106742859,
-      "logps/chosen": -1.7954018115997314,
-      "logps/rejected": -1.9569883346557617,
-      "loss": 0.5718,
+      "logits/chosen": 0.05401003360748291,
+      "logits/rejected": 0.08016423881053925,
+      "logps/chosen": -1.775671362876892,
+      "logps/rejected": -1.9383106231689453,
+      "loss": 0.2496,
       "rewards/accuracies": 0.5687500238418579,
-      "rewards/chosen": -1.7954018115997314,
-      "rewards/margins": 0.16158629953861237,
-      "rewards/rejected": -1.9569883346557617,
-      "semantic_entropy": 0.648250937461853,
+      "rewards/chosen": -1.775671362876892,
+      "rewards/margins": 0.16263923048973083,
+      "rewards/rejected": -1.9383106231689453,
       "step": 105
     },
     {
       "epoch": 0.05887272119083459,
-      "grad_norm": 4.253659296532382,
+      "grad_norm": 2.5055549833177158,
       "learning_rate": 1.96078431372549e-07,
-      "logits/chosen": 0.019205275923013687,
-      "logits/rejected": 0.11396453529596329,
-      "logps/chosen": -1.8869092464447021,
-      "logps/rejected": -1.9481251239776611,
-      "loss": 0.5738,
-      "rewards/accuracies": 0.512499988079071,
-      "rewards/chosen": -1.8869092464447021,
-      "rewards/margins": 0.06121576577425003,
-      "rewards/rejected": -1.9481251239776611,
-      "semantic_entropy": 0.630905032157898,
+      "logits/chosen": 0.026605969294905663,
+      "logits/rejected": 0.12203983962535858,
+      "logps/chosen": -1.8630073070526123,
+      "logps/rejected": -1.9225571155548096,
+      "loss": 0.2607,
+      "rewards/accuracies": 0.5062500238418579,
+      "rewards/chosen": -1.8630073070526123,
+      "rewards/margins": 0.05954958125948906,
+      "rewards/rejected": -1.9225571155548096,
       "step": 110
     },
     {
       "epoch": 0.06154875397223616,
-      "grad_norm": 5.0490119418622,
+      "grad_norm": 2.42093904789979,
       "learning_rate": 2.049910873440285e-07,
-      "logits/chosen": 0.050636060535907745,
-      "logits/rejected": 0.26533403992652893,
-      "logps/chosen": -1.8734686374664307,
-      "logps/rejected": -2.212010383605957,
-      "loss": 0.5106,
-      "rewards/accuracies": 0.612500011920929,
-      "rewards/chosen": -1.8734686374664307,
-      "rewards/margins": 0.33854159712791443,
-      "rewards/rejected": -2.212010383605957,
-      "semantic_entropy": 0.5976813435554504,
+      "logits/chosen": 0.04268549010157585,
+      "logits/rejected": 0.2566087543964386,
+      "logps/chosen": -1.8545005321502686,
+      "logps/rejected": -2.1896767616271973,
+      "loss": 0.2134,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -1.8545005321502686,
+      "rewards/margins": 0.3351760804653168,
+      "rewards/rejected": -2.1896767616271973,
       "step": 115
     },
     {
       "epoch": 0.06422478675363773,
-      "grad_norm": 3.6994949372266035,
+      "grad_norm": 1.8206023092375123,
       "learning_rate": 2.13903743315508e-07,
-      "logits/chosen": -0.07818116992712021,
-      "logits/rejected": 0.09761984646320343,
-      "logps/chosen": -2.0244534015655518,
-      "logps/rejected": -2.173408269882202,
-      "loss": 0.5293,
+      "logits/chosen": -0.06773766130208969,
+      "logits/rejected": 0.1087050586938858,
+      "logps/chosen": -2.0017330646514893,
+      "logps/rejected": -2.148353099822998,
+      "loss": 0.2308,
       "rewards/accuracies": 0.4937500059604645,
-      "rewards/chosen": -2.0244534015655518,
-      "rewards/margins": 0.14895491302013397,
-      "rewards/rejected": -2.173408269882202,
-      "semantic_entropy": 0.6009461879730225,
+      "rewards/chosen": -2.0017330646514893,
+      "rewards/margins": 0.14662012457847595,
+      "rewards/rejected": -2.148353099822998,
       "step": 120
     },
     {
       "epoch": 0.0669008195350393,
-      "grad_norm": 4.7177100362781434,
+      "grad_norm": 2.8040060365210007,
       "learning_rate": 2.2281639928698751e-07,
-      "logits/chosen": -0.060094915330410004,
-      "logits/rejected": 0.07375667989253998,
-      "logps/chosen": -1.9164215326309204,
-      "logps/rejected": -1.8263633251190186,
-      "loss": 0.5804,
+      "logits/chosen": -0.0807386040687561,
+      "logits/rejected": 0.05026230961084366,
+      "logps/chosen": -1.89572274684906,
+      "logps/rejected": -1.8105024099349976,
+      "loss": 0.2647,
       "rewards/accuracies": 0.4749999940395355,
-      "rewards/chosen": -1.9164215326309204,
-      "rewards/margins": -0.09005783498287201,
-      "rewards/rejected": -1.8263633251190186,
-      "semantic_entropy": 0.6356278657913208,
+      "rewards/chosen": -1.89572274684906,
+      "rewards/margins": -0.08522024005651474,
+      "rewards/rejected": -1.8105024099349976,
       "step": 125
     },
     {
       "epoch": 0.06957685231644088,
-      "grad_norm": 4.3470043435073356,
+      "grad_norm": 2.6288404494949145,
       "learning_rate": 2.31729055258467e-07,
-      "logits/chosen": 0.06565021723508835,
-      "logits/rejected": 0.2070547640323639,
-      "logps/chosen": -1.9880714416503906,
-      "logps/rejected": -2.111767530441284,
-      "loss": 0.5254,
-      "rewards/accuracies": 0.48124998807907104,
-      "rewards/chosen": -1.9880714416503906,
-      "rewards/margins": 0.12369606643915176,
-      "rewards/rejected": -2.111767530441284,
-      "semantic_entropy": 0.5927776098251343,
+      "logits/chosen": 0.03656303137540817,
+      "logits/rejected": 0.1726987063884735,
+      "logps/chosen": -1.9655530452728271,
+      "logps/rejected": -2.089103937149048,
+      "loss": 0.231,
+      "rewards/accuracies": 0.46875,
+      "rewards/chosen": -1.9655530452728271,
+      "rewards/margins": 0.12355075031518936,
+      "rewards/rejected": -2.089103937149048,
       "step": 130
     },
     {
       "epoch": 0.07225288509784245,
-      "grad_norm": 3.6305162217659905,
+      "grad_norm": 2.087737893476425,
       "learning_rate": 2.406417112299465e-07,
-      "logits/chosen": -0.030332911759614944,
-      "logits/rejected": 0.08955554664134979,
-      "logps/chosen": -2.080584764480591,
-      "logps/rejected": -2.0448110103607178,
-      "loss": 0.5326,
-      "rewards/accuracies": 0.518750011920929,
-      "rewards/chosen": -2.080584764480591,
-      "rewards/margins": -0.035773664712905884,
-      "rewards/rejected": -2.0448110103607178,
-      "semantic_entropy": 0.5878274440765381,
+      "logits/chosen": -0.019556980580091476,
+      "logits/rejected": 0.10337207466363907,
+      "logps/chosen": -2.0496957302093506,
+      "logps/rejected": -2.0199496746063232,
+      "loss": 0.241,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": -2.0496957302093506,
+      "rewards/margins": -0.02974613569676876,
+      "rewards/rejected": -2.0199496746063232,
       "step": 135
     },
     {
       "epoch": 0.07492891787924402,
-      "grad_norm": 4.577194794071166,
+      "grad_norm": 2.9049109285511663,
       "learning_rate": 2.49554367201426e-07,
-      "logits/chosen": -0.024137228727340698,
-      "logits/rejected": 0.14822633564472198,
-      "logps/chosen": -2.0620641708374023,
-      "logps/rejected": -2.340951919555664,
-      "loss": 0.4846,
-      "rewards/accuracies": 0.5249999761581421,
-      "rewards/chosen": -2.0620641708374023,
-      "rewards/margins": 0.2788878381252289,
-      "rewards/rejected": -2.340951919555664,
-      "semantic_entropy": 0.5582477450370789,
+      "logits/chosen": -0.025853702798485756,
+      "logits/rejected": 0.14452826976776123,
+      "logps/chosen": -2.0190365314483643,
+      "logps/rejected": -2.2830276489257812,
+      "loss": 0.2095,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -2.0190365314483643,
+      "rewards/margins": 0.2639915347099304,
+      "rewards/rejected": -2.2830276489257812,
       "step": 140
     },
     {
       "epoch": 0.0776049506606456,
-      "grad_norm": 3.5772189076002534,
+      "grad_norm": 2.156959064207746,
       "learning_rate": 2.5846702317290554e-07,
-      "logits/chosen": -0.0062357052229344845,
-      "logits/rejected": 0.15561768412590027,
-      "logps/chosen": -2.054307699203491,
-      "logps/rejected": -2.20831298828125,
-      "loss": 0.5148,
-      "rewards/accuracies": 0.46875,
-      "rewards/chosen": -2.054307699203491,
-      "rewards/margins": 0.15400531888008118,
-      "rewards/rejected": -2.20831298828125,
-      "semantic_entropy": 0.5854038000106812,
+      "logits/chosen": -0.011653195135295391,
+      "logits/rejected": 0.14680761098861694,
+      "logps/chosen": -1.9980818033218384,
+      "logps/rejected": -2.149132251739502,
+      "loss": 0.2268,
+      "rewards/accuracies": 0.48124998807907104,
+      "rewards/chosen": -1.9980818033218384,
+      "rewards/margins": 0.1510503739118576,
+      "rewards/rejected": -2.149132251739502,
       "step": 145
     },
     {
       "epoch": 0.08028098344204716,
-      "grad_norm": 3.320147224300046,
+      "grad_norm": 2.3619427872695433,
       "learning_rate": 2.6737967914438503e-07,
-      "logits/chosen": -0.034813135862350464,
-      "logits/rejected": 0.1387842893600464,
-      "logps/chosen": -2.0534462928771973,
-      "logps/rejected": -2.0570671558380127,
-      "loss": 0.5563,
-      "rewards/accuracies": 0.512499988079071,
-      "rewards/chosen": -2.0534462928771973,
-      "rewards/margins": 0.0036209137178957462,
-      "rewards/rejected": -2.0570671558380127,
-      "semantic_entropy": 0.625429093837738,
+      "logits/chosen": -0.034545183181762695,
+      "logits/rejected": 0.13397535681724548,
+      "logps/chosen": -1.9922664165496826,
+      "logps/rejected": -1.99588143825531,
+      "loss": 0.2493,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": -1.9922664165496826,
+      "rewards/margins": 0.003614726709201932,
+      "rewards/rejected": -1.99588143825531,
       "step": 150
     },
     {
       "epoch": 0.08295701622344874,
-      "grad_norm": 3.471479341023171,
+      "grad_norm": 2.2327996873434297,
       "learning_rate": 2.762923351158645e-07,
-      "logits/chosen": -0.029315322637557983,
-      "logits/rejected": 0.018080363050103188,
-      "logps/chosen": -2.1867306232452393,
-      "logps/rejected": -2.2244086265563965,
-      "loss": 0.5212,
+      "logits/chosen": -0.0200195349752903,
+      "logits/rejected": 0.026785830035805702,
+      "logps/chosen": -2.1182286739349365,
+      "logps/rejected": -2.162487030029297,
+      "loss": 0.2351,
       "rewards/accuracies": 0.48750001192092896,
-      "rewards/chosen": -2.1867306232452393,
-      "rewards/margins": 0.037678077816963196,
-      "rewards/rejected": -2.2244086265563965,
-      "semantic_entropy": 0.5816220045089722,
+      "rewards/chosen": -2.1182286739349365,
+      "rewards/margins": 0.0442582368850708,
+      "rewards/rejected": -2.162487030029297,
       "step": 155
     },
     {
       "epoch": 0.0856330490048503,
-      "grad_norm": 3.7011763753323144,
+      "grad_norm": 2.1191289655384895,
       "learning_rate": 2.85204991087344e-07,
-      "logits/chosen": -0.10797729343175888,
-      "logits/rejected": 0.044836580753326416,
-      "logps/chosen": -2.4026219844818115,
-      "logps/rejected": -2.3765602111816406,
-      "loss": 0.4831,
+      "logits/chosen": -0.12646076083183289,
+      "logits/rejected": 0.01917138509452343,
+      "logps/chosen": -2.3306081295013428,
+      "logps/rejected": -2.306840181350708,
+      "loss": 0.2229,
       "rewards/accuracies": 0.48750001192092896,
-      "rewards/chosen": -2.4026219844818115,
-      "rewards/margins": -0.026061728596687317,
-      "rewards/rejected": -2.3765602111816406,
-      "semantic_entropy": 0.5305178761482239,
+      "rewards/chosen": -2.3306081295013428,
+      "rewards/margins": -0.023767992854118347,
+      "rewards/rejected": -2.306840181350708,
       "step": 160
     },
     {
       "epoch": 0.08830908178625188,
-      "grad_norm": 3.4705911589732112,
+      "grad_norm": 2.04617266810806,
       "learning_rate": 2.941176470588235e-07,
-      "logits/chosen": -0.01804346963763237,
-      "logits/rejected": 0.16947805881500244,
-      "logps/chosen": -2.112272024154663,
-      "logps/rejected": -2.4508056640625,
-      "loss": 0.5079,
+      "logits/chosen": -0.04032761603593826,
+      "logits/rejected": 0.14231903851032257,
+      "logps/chosen": -2.0537381172180176,
+      "logps/rejected": -2.376739978790283,
+      "loss": 0.2268,
       "rewards/accuracies": 0.5687500238418579,
-      "rewards/chosen": -2.112272024154663,
-      "rewards/margins": 0.3385334312915802,
-      "rewards/rejected": -2.4508056640625,
-      "semantic_entropy": 0.5715000033378601,
+      "rewards/chosen": -2.0537381172180176,
+      "rewards/margins": 0.32300177216529846,
+      "rewards/rejected": -2.376739978790283,
       "step": 165
     },
     {
       "epoch": 0.09098511456765346,
-      "grad_norm": 3.413708707373631,
+      "grad_norm": 2.009380146648297,
       "learning_rate": 3.0303030303030305e-07,
-      "logits/chosen": -0.06662870943546295,
-      "logits/rejected": -0.008535407483577728,
-      "logps/chosen": -2.417525291442871,
-      "logps/rejected": -2.3798742294311523,
-      "loss": 0.4764,
-      "rewards/accuracies": 0.45625001192092896,
-      "rewards/chosen": -2.417525291442871,
-      "rewards/margins": -0.0376511812210083,
-      "rewards/rejected": -2.3798742294311523,
-      "semantic_entropy": 0.540223240852356,
+      "logits/chosen": -0.05518675968050957,
+      "logits/rejected": 0.0020436719059944153,
+      "logps/chosen": -2.344125509262085,
+      "logps/rejected": -2.3139758110046387,
+      "loss": 0.2106,
+      "rewards/accuracies": 0.44999998807907104,
+      "rewards/chosen": -2.344125509262085,
+      "rewards/margins": -0.030149567872285843,
+      "rewards/rejected": -2.3139758110046387,
       "step": 170
     },
     {
       "epoch": 0.09366114734905502,
-      "grad_norm": 3.543204898829903,
+      "grad_norm": 2.018083114120056,
       "learning_rate": 3.1194295900178254e-07,
-      "logits/chosen": 0.09488913416862488,
-      "logits/rejected": 0.09268101304769516,
-      "logps/chosen": -2.30230450630188,
-      "logps/rejected": -2.363039255142212,
-      "loss": 0.5165,
+      "logits/chosen": 0.09154976904392242,
+      "logits/rejected": 0.09093359112739563,
+      "logps/chosen": -2.2235960960388184,
+      "logps/rejected": -2.286865711212158,
+      "loss": 0.2469,
       "rewards/accuracies": 0.46875,
-      "rewards/chosen": -2.30230450630188,
-      "rewards/margins": 0.06073470786213875,
-      "rewards/rejected": -2.363039255142212,
-      "semantic_entropy": 0.5503964424133301,
+      "rewards/chosen": -2.2235960960388184,
+      "rewards/margins": 0.06326953321695328,
+      "rewards/rejected": -2.286865711212158,
       "step": 175
     },
     {
       "epoch": 0.0963371801304566,
-      "grad_norm": 3.882410336806739,
+      "grad_norm": 1.701656302291715,
       "learning_rate": 3.2085561497326203e-07,
-      "logits/chosen": 0.025393879041075706,
-      "logits/rejected": 0.028031084686517715,
-      "logps/chosen": -2.42897367477417,
-      "logps/rejected": -2.349623203277588,
-      "loss": 0.4847,
+      "logits/chosen": 0.03326406702399254,
+      "logits/rejected": 0.03350166231393814,
+      "logps/chosen": -2.326598644256592,
+      "logps/rejected": -2.278094530105591,
+      "loss": 0.2179,
       "rewards/accuracies": 0.512499988079071,
-      "rewards/chosen": -2.42897367477417,
-      "rewards/margins": -0.079350546002388,
-      "rewards/rejected": -2.349623203277588,
-      "semantic_entropy": 0.5456860661506653,
+      "rewards/chosen": -2.326598644256592,
+      "rewards/margins": -0.048504143953323364,
+      "rewards/rejected": -2.278094530105591,
       "step": 180
     },
     {
       "epoch": 0.09901321291185818,
-      "grad_norm": 6.489830693027923,
+      "grad_norm": 2.064670625121251,
       "learning_rate": 3.297682709447415e-07,
-      "logits/chosen": -0.1303943544626236,
-      "logits/rejected": -0.034635625779628754,
-      "logps/chosen": -2.4026060104370117,
-      "logps/rejected": -2.4866461753845215,
-      "loss": 0.5071,
-      "rewards/accuracies": 0.512499988079071,
-      "rewards/chosen": -2.4026060104370117,
-      "rewards/margins": 0.08404017984867096,
-      "rewards/rejected": -2.4866461753845215,
-      "semantic_entropy": 0.563964307308197,
+      "logits/chosen": -0.10991539061069489,
+      "logits/rejected": -0.01808272674679756,
+      "logps/chosen": -2.2595760822296143,
+      "logps/rejected": -2.352756977081299,
+      "loss": 0.2346,
+      "rewards/accuracies": 0.5062500238418579,
+      "rewards/chosen": -2.2595760822296143,
+      "rewards/margins": 0.09318114817142487,
+      "rewards/rejected": -2.352756977081299,
       "step": 185
     },
     {
       "epoch": 0.10168924569325974,
-      "grad_norm": 5.983301960644024,
+      "grad_norm": 2.8550580867543345,
       "learning_rate": 3.38680926916221e-07,
-      "logits/chosen": -0.03423957899212837,
-      "logits/rejected": 0.10569383203983307,
-      "logps/chosen": -3.298650026321411,
-      "logps/rejected": -3.035168409347534,
-      "loss": 0.4047,
-      "rewards/accuracies": 0.48750001192092896,
-      "rewards/chosen": -3.298650026321411,
-      "rewards/margins": -0.2634817659854889,
-      "rewards/rejected": -3.035168409347534,
-      "semantic_entropy": 0.45990005135536194,
+      "logits/chosen": -0.016482654958963394,
+      "logits/rejected": 0.12248317152261734,
+      "logps/chosen": -2.8278183937072754,
+      "logps/rejected": -2.660628318786621,
+      "loss": 0.194,
+      "rewards/accuracies": 0.4937500059604645,
+      "rewards/chosen": -2.8278183937072754,
+      "rewards/margins": -0.16718989610671997,
+      "rewards/rejected": -2.660628318786621,
       "step": 190
     },
     {
       "epoch": 0.10436527847466132,
-      "grad_norm": 4.991429794604448,
+      "grad_norm": 2.4580704082500793,
       "learning_rate": 3.475935828877005e-07,
-      "logits/chosen": 0.02364252880215645,
-      "logits/rejected": 0.19518662989139557,
-      "logps/chosen": -2.553497076034546,
-      "logps/rejected": -2.5629844665527344,
-      "loss": 0.4404,
-      "rewards/accuracies": 0.5062500238418579,
-      "rewards/chosen": -2.553497076034546,
-      "rewards/margins": 0.009487425908446312,
-      "rewards/rejected": -2.5629844665527344,
-      "semantic_entropy": 0.5125027894973755,
+      "logits/chosen": 0.03471102565526962,
+      "logits/rejected": 0.20238180458545685,
+      "logps/chosen": -2.1727375984191895,
+      "logps/rejected": -2.2292609214782715,
+      "loss": 0.2117,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": -2.1727375984191895,
+      "rewards/margins": 0.05652397871017456,
+      "rewards/rejected": -2.2292609214782715,
       "step": 195
     },
     {
       "epoch": 0.1070413112560629,
-      "grad_norm": 4.42975674257316,
+      "grad_norm": 3.151318863169166,
       "learning_rate": 3.5650623885918e-07,
-      "logits/chosen": -0.04811898618936539,
-      "logits/rejected": 0.10957398265600204,
-      "logps/chosen": -3.3466639518737793,
-      "logps/rejected": -2.866375684738159,
-      "loss": 0.4166,
-      "rewards/accuracies": 0.5249999761581421,
-      "rewards/chosen": -3.3466639518737793,
-      "rewards/margins": -0.4802883267402649,
-      "rewards/rejected": -2.866375684738159,
-      "semantic_entropy": 0.47037285566329956,
+      "logits/chosen": -0.037284087389707565,
+      "logits/rejected": 0.11781023442745209,
+      "logps/chosen": -2.721318244934082,
+      "logps/rejected": -2.402465343475342,
+      "loss": 0.2105,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": -2.721318244934082,
+      "rewards/margins": -0.31885290145874023,
+      "rewards/rejected": -2.402465343475342,
       "step": 200
     },
     {
       "epoch": 0.10971734403746446,
-      "grad_norm": 3.7097199469373354,
+      "grad_norm": 2.6550912450394533,
       "learning_rate": 3.654188948306595e-07,
-      "logits/chosen": -0.0347520187497139,
-      "logits/rejected": 0.13235226273536682,
-      "logps/chosen": -4.012370586395264,
-      "logps/rejected": -3.4062774181365967,
-      "loss": 0.3824,
-      "rewards/accuracies": 0.512499988079071,
-      "rewards/chosen": -4.012370586395264,
-      "rewards/margins": -0.6060926914215088,
-      "rewards/rejected": -3.4062774181365967,
-      "semantic_entropy": 0.44439277052879333,
+      "logits/chosen": -0.013735070824623108,
+      "logits/rejected": 0.15475720167160034,
+      "logps/chosen": -3.2313952445983887,
+      "logps/rejected": -2.8162760734558105,
+      "loss": 0.1895,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -3.2313952445983887,
+      "rewards/margins": -0.415119469165802,
+      "rewards/rejected": -2.8162760734558105,
       "step": 205
     },
     {
       "epoch": 0.11239337681886603,
-      "grad_norm": 3.006727006054674,
+      "grad_norm": 2.2808856180721637,
       "learning_rate": 3.7433155080213904e-07,
-      "logits/chosen": -0.12415327876806259,
-      "logits/rejected": 0.09854158014059067,
-      "logps/chosen": -3.691174030303955,
-      "logps/rejected": -4.26821756362915,
-      "loss": 0.3329,
-      "rewards/accuracies": 0.606249988079071,
-      "rewards/chosen": -3.691174030303955,
-      "rewards/margins": 0.5770431756973267,
-      "rewards/rejected": -4.26821756362915,
-      "semantic_entropy": 0.41728726029396057,
+      "logits/chosen": -0.09457657486200333,
+      "logits/rejected": 0.12909016013145447,
+      "logps/chosen": -3.0543413162231445,
+      "logps/rejected": -3.4971511363983154,
+      "loss": 0.1495,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -3.0543413162231445,
+      "rewards/margins": 0.4428097605705261,
+      "rewards/rejected": -3.4971511363983154,
       "step": 210
     },
     {
       "epoch": 0.1150694096002676,
-      "grad_norm": 3.284414971810367,
+      "grad_norm": 2.1192106274005464,
       "learning_rate": 3.8324420677361853e-07,
-      "logits/chosen": -0.1523238718509674,
-      "logits/rejected": 0.121431365609169,
-      "logps/chosen": -3.3081634044647217,
-      "logps/rejected": -3.495288133621216,
-      "loss": 0.3532,
-      "rewards/accuracies": 0.5625,
-      "rewards/chosen": -3.3081634044647217,
-      "rewards/margins": 0.18712471425533295,
-      "rewards/rejected": -3.495288133621216,
-      "semantic_entropy": 0.4320215582847595,
+      "logits/chosen": -0.13070939481258392,
+      "logits/rejected": 0.14487504959106445,
+      "logps/chosen": -2.8309102058410645,
+      "logps/rejected": -2.9786882400512695,
+      "loss": 0.1608,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -2.8309102058410645,
+      "rewards/margins": 0.14777803421020508,
+      "rewards/rejected": -2.9786882400512695,
       "step": 215
     },
     {
       "epoch": 0.11774544238166917,
-      "grad_norm": 4.517057249275933,
+      "grad_norm": 2.117697527140775,
       "learning_rate": 3.92156862745098e-07,
-      "logits/chosen": 0.08584829419851303,
-      "logits/rejected": 0.20471997559070587,
-      "logps/chosen": -4.001864433288574,
-      "logps/rejected": -4.645659923553467,
-      "loss": 0.314,
-      "rewards/accuracies": 0.6000000238418579,
-      "rewards/chosen": -4.001864433288574,
-      "rewards/margins": 0.6437958478927612,
-      "rewards/rejected": -4.645659923553467,
-      "semantic_entropy": 0.3854829668998718,
+      "logits/chosen": 0.06672084331512451,
+      "logits/rejected": 0.1787821650505066,
+      "logps/chosen": -3.4120097160339355,
+      "logps/rejected": -3.9355578422546387,
+      "loss": 0.14,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -3.4120097160339355,
+      "rewards/margins": 0.5235484838485718,
+      "rewards/rejected": -3.9355578422546387,
       "step": 220
     },
     {
       "epoch": 0.12042147516307075,
-      "grad_norm": 5.140563516027854,
+      "grad_norm": 1.9630938588222873,
       "learning_rate": 4.010695187165775e-07,
-      "logits/chosen": -0.06603296846151352,
-      "logits/rejected": 0.13016948103904724,
-      "logps/chosen": -3.7264492511749268,
-      "logps/rejected": -3.8007068634033203,
-      "loss": 0.313,
+      "logits/chosen": -0.09861567616462708,
+      "logits/rejected": 0.0825483575463295,
+      "logps/chosen": -3.199176788330078,
+      "logps/rejected": -3.2739837169647217,
+      "loss": 0.141,
       "rewards/accuracies": 0.5562499761581421,
-      "rewards/chosen": -3.7264492511749268,
-      "rewards/margins": 0.07425764948129654,
-      "rewards/rejected": -3.8007068634033203,
-      "semantic_entropy": 0.3880666494369507,
+      "rewards/chosen": -3.199176788330078,
+      "rewards/margins": 0.07480701804161072,
+      "rewards/rejected": -3.2739837169647217,
       "step": 225
     },
     {
       "epoch": 0.12309750794447231,
-      "grad_norm": 4.465321116954656,
+      "grad_norm": 2.0932688158739277,
       "learning_rate": 4.09982174688057e-07,
-      "logits/chosen": 0.06401034444570541,
-      "logits/rejected": 0.1629628837108612,
-      "logps/chosen": -4.803007125854492,
-      "logps/rejected": -4.7263898849487305,
-      "loss": 0.2807,
-      "rewards/accuracies": 0.512499988079071,
-      "rewards/chosen": -4.803007125854492,
-      "rewards/margins": -0.07661716639995575,
-      "rewards/rejected": -4.7263898849487305,
-      "semantic_entropy": 0.3396259844303131,
+      "logits/chosen": 0.01902366243302822,
+      "logits/rejected": 0.10696601867675781,
+      "logps/chosen": -3.955059766769409,
+      "logps/rejected": -3.9199607372283936,
+      "loss": 0.1372,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -3.955059766769409,
+      "rewards/margins": -0.03509901836514473,
+      "rewards/rejected": -3.9199607372283936,
       "step": 230
     },
     {
       "epoch": 0.1257735407258739,
-      "grad_norm": 2.7042614308205235,
+      "grad_norm": 2.249728441941806,
       "learning_rate": 4.188948306595365e-07,
-      "logits/chosen": 0.08987166732549667,
-      "logits/rejected": 0.27392953634262085,
-      "logps/chosen": -4.6403398513793945,
-      "logps/rejected": -5.046909332275391,
-      "loss": 0.2585,
-      "rewards/accuracies": 0.5874999761581421,
-      "rewards/chosen": -4.6403398513793945,
-      "rewards/margins": 0.406569242477417,
-      "rewards/rejected": -5.046909332275391,
-      "semantic_entropy": 0.3378155827522278,
+      "logits/chosen": 0.04213883727788925,
+      "logits/rejected": 0.20603123307228088,
+      "logps/chosen": -3.6573188304901123,
+      "logps/rejected": -4.0042901039123535,
+      "loss": 0.1164,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -3.6573188304901123,
+      "rewards/margins": 0.346971333026886,
+      "rewards/rejected": -4.0042901039123535,
       "step": 235
     },
     {
       "epoch": 0.12844957350727546,
-      "grad_norm": 2.753676038637165,
+      "grad_norm": 2.273064270761011,
       "learning_rate": 4.27807486631016e-07,
-      "logits/chosen": 0.042416222393512726,
-      "logits/rejected": 0.18578985333442688,
-      "logps/chosen": -5.076837539672852,
-      "logps/rejected": -4.993945121765137,
-      "loss": 0.2619,
+      "logits/chosen": 0.013959121890366077,
+      "logits/rejected": 0.15524466335773468,
+      "logps/chosen": -4.024859428405762,
+      "logps/rejected": -4.038453578948975,
+      "loss": 0.1286,
       "rewards/accuracies": 0.518750011920929,
-      "rewards/chosen": -5.076837539672852,
-      "rewards/margins": -0.08289314806461334,
-      "rewards/rejected": -4.993945121765137,
-      "semantic_entropy": 0.31904274225234985,
+      "rewards/chosen": -4.024859428405762,
+      "rewards/margins": 0.013593924231827259,
+      "rewards/rejected": -4.038453578948975,
       "step": 240
     },
     {
       "epoch": 0.13112560628867703,
-      "grad_norm": 2.4582120018377376,
+      "grad_norm": 1.8055229717224743,
       "learning_rate": 4.3672014260249554e-07,
-      "logits/chosen": 0.10127206891775131,
-      "logits/rejected": 0.2557363510131836,
-      "logps/chosen": -4.425154685974121,
-      "logps/rejected": -5.560978889465332,
-      "loss": 0.2595,
-      "rewards/accuracies": 0.5874999761581421,
-      "rewards/chosen": -4.425154685974121,
-      "rewards/margins": 1.13582444190979,
-      "rewards/rejected": -5.560978889465332,
-      "semantic_entropy": 0.32832127809524536,
+      "logits/chosen": 0.10808676481246948,
+      "logits/rejected": 0.25572407245635986,
+      "logps/chosen": -3.702683925628662,
+      "logps/rejected": -4.566058158874512,
+      "loss": 0.1141,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -3.702683925628662,
+      "rewards/margins": 0.8633737564086914,
+      "rewards/rejected": -4.566058158874512,
       "step": 245
     },
     {
       "epoch": 0.1338016390700786,
-      "grad_norm": 2.575075964822789,
+      "grad_norm": 1.6074368965988135,
       "learning_rate": 4.4563279857397503e-07,
-      "logits/chosen": 0.08739835023880005,
-      "logits/rejected": 0.2964261472225189,
-      "logps/chosen": -6.85397481918335,
-      "logps/rejected": -6.661816596984863,
-      "loss": 0.2524,
-      "rewards/accuracies": 0.5249999761581421,
-      "rewards/chosen": -6.85397481918335,
-      "rewards/margins": -0.19215914607048035,
-      "rewards/rejected": -6.661816596984863,
-      "semantic_entropy": 0.31161242723464966,
+      "logits/chosen": 0.03807227686047554,
+      "logits/rejected": 0.23177461326122284,
+      "logps/chosen": -5.78887414932251,
+      "logps/rejected": -5.649636268615723,
+      "loss": 0.1132,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -5.78887414932251,
+      "rewards/margins": -0.13923706114292145,
+      "rewards/rejected": -5.649636268615723,
       "step": 250
     },
     {
       "epoch": 0.1364776718514802,
-      "grad_norm": 3.1444323203134688,
+      "grad_norm": 1.251947569890329,
       "learning_rate": 4.545454545454545e-07,
-      "logits/chosen": 0.06549052894115448,
-      "logits/rejected": 0.27386870980262756,
-      "logps/chosen": -4.550482749938965,
-      "logps/rejected": -5.5093584060668945,
-      "loss": 0.2604,
-      "rewards/accuracies": 0.5625,
-      "rewards/chosen": -4.550482749938965,
-      "rewards/margins": 0.9588750600814819,
-      "rewards/rejected": -5.5093584060668945,
-      "semantic_entropy": 0.32709717750549316,
+      "logits/chosen": 0.06387828290462494,
+      "logits/rejected": 0.24852073192596436,
+      "logps/chosen": -4.003514289855957,
+      "logps/rejected": -4.843715667724609,
+      "loss": 0.111,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -4.003514289855957,
+      "rewards/margins": 0.8402018547058105,
+      "rewards/rejected": -4.843715667724609,
       "step": 255
     },
     {
       "epoch": 0.13915370463288176,
-      "grad_norm": 3.3065425131874067,
+      "grad_norm": 1.0557755605880161,
       "learning_rate": 4.63458110516934e-07,
-      "logits/chosen": -0.11962959915399551,
-      "logits/rejected": 0.018381217494606972,
-      "logps/chosen": -6.10165548324585,
-      "logps/rejected": -5.083194732666016,
-      "loss": 0.2118,
-      "rewards/accuracies": 0.5375000238418579,
-      "rewards/chosen": -6.10165548324585,
-      "rewards/margins": -1.0184614658355713,
-      "rewards/rejected": -5.083194732666016,
-      "semantic_entropy": 0.27624058723449707,
+      "logits/chosen": -0.09168213605880737,
+      "logits/rejected": 0.03592577204108238,
+      "logps/chosen": -5.3444318771362305,
+      "logps/rejected": -4.5207133293151855,
+      "loss": 0.0874,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -5.3444318771362305,
+      "rewards/margins": -0.8237188458442688,
+      "rewards/rejected": -4.5207133293151855,
       "step": 260
     },
     {
       "epoch": 0.1418297374142833,
-      "grad_norm": 2.4937567619586947,
+      "grad_norm": 0.948002934749555,
       "learning_rate": 4.723707664884135e-07,
-      "logits/chosen": 0.10635049641132355,
-      "logits/rejected": 0.20212987065315247,
-      "logps/chosen": -7.334754943847656,
-      "logps/rejected": -5.710747241973877,
-      "loss": 0.1978,
-      "rewards/accuracies": 0.5062500238418579,
-      "rewards/chosen": -7.334754943847656,
-      "rewards/margins": -1.6240085363388062,
-      "rewards/rejected": -5.710747241973877,
-      "semantic_entropy": 0.23839779198169708,
+      "logits/chosen": 0.09713621437549591,
+      "logits/rejected": 0.1900687962770462,
+      "logps/chosen": -6.329071998596191,
+      "logps/rejected": -4.980845928192139,
+      "loss": 0.0961,
+      "rewards/accuracies": 0.48750001192092896,
+      "rewards/chosen": -6.329071998596191,
+      "rewards/margins": -1.3482263088226318,
+      "rewards/rejected": -4.980845928192139,
       "step": 265
     },
     {
       "epoch": 0.1445057701956849,
-      "grad_norm": 6.634798485239572,
+      "grad_norm": 0.9982119135432448,
       "learning_rate": 4.81283422459893e-07,
-      "logits/chosen": 0.04447219520807266,
-      "logits/rejected": 0.23252050578594208,
-      "logps/chosen": -5.903593063354492,
-      "logps/rejected": -6.297295093536377,
-      "loss": 0.1997,
-      "rewards/accuracies": 0.512499988079071,
-      "rewards/chosen": -5.903593063354492,
-      "rewards/margins": 0.3937019407749176,
-      "rewards/rejected": -6.297295093536377,
-      "semantic_entropy": 0.24874810874462128,
+      "logits/chosen": 0.031049340963363647,
+      "logits/rejected": 0.20883283019065857,
+      "logps/chosen": -4.965585708618164,
+      "logps/rejected": -5.328085422515869,
+      "loss": 0.0955,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -4.965585708618164,
+      "rewards/margins": 0.3624994158744812,
+      "rewards/rejected": -5.328085422515869,
       "step": 270
     },
     {
       "epoch": 0.14718180297708647,
-      "grad_norm": 12.381758710987581,
+      "grad_norm": 1.6297908390461078,
       "learning_rate": 4.901960784313725e-07,
-      "logits/chosen": 0.30282384157180786,
-      "logits/rejected": 0.38939711451530457,
-      "logps/chosen": -7.172441005706787,
-      "logps/rejected": -7.899168491363525,
-      "loss": 0.182,
-      "rewards/accuracies": 0.59375,
-      "rewards/chosen": -7.172441005706787,
-      "rewards/margins": 0.7267270684242249,
-      "rewards/rejected": -7.899168491363525,
-      "semantic_entropy": 0.22203528881072998,
+      "logits/chosen": 0.14937344193458557,
+      "logits/rejected": 0.25307050347328186,
+      "logps/chosen": -5.577644348144531,
+      "logps/rejected": -6.263503074645996,
+      "loss": 0.1076,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -5.577644348144531,
+      "rewards/margins": 0.685858428478241,
+      "rewards/rejected": -6.263503074645996,
       "step": 275
     },
     {
       "epoch": 0.14985783575848804,
-      "grad_norm": 6.239053192073585,
+      "grad_norm": 1.5496631366124152,
       "learning_rate": 4.99108734402852e-07,
-      "logits/chosen": 0.38646355271339417,
-      "logits/rejected": 0.6254279613494873,
-      "logps/chosen": -9.66862964630127,
-      "logps/rejected": -10.103830337524414,
-      "loss": 0.0771,
-      "rewards/accuracies": 0.574999988079071,
-      "rewards/chosen": -9.66862964630127,
-      "rewards/margins": 0.435200035572052,
-      "rewards/rejected": -10.103830337524414,
-      "semantic_entropy": 0.10753343254327774,
+      "logits/chosen": 0.014246372506022453,
+      "logits/rejected": 0.21954336762428284,
+      "logps/chosen": -6.520092010498047,
+      "logps/rejected": -6.146762847900391,
+      "loss": 0.0769,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -6.520092010498047,
+      "rewards/margins": -0.3733291029930115,
+      "rewards/rejected": -6.146762847900391,
       "step": 280
     },
     {
       "epoch": 0.15253386853988962,
-      "grad_norm": 4.836784229202216,
+      "grad_norm": 1.64411911013092,
       "learning_rate": 5.080213903743315e-07,
-      "logits/chosen": 0.8410245180130005,
-      "logits/rejected": 1.0995079278945923,
-      "logps/chosen": -11.102112770080566,
-      "logps/rejected": -11.977313041687012,
-      "loss": 0.0248,
-      "rewards/accuracies": 0.5,
-      "rewards/chosen": -11.102112770080566,
-      "rewards/margins": 0.8752012252807617,
-      "rewards/rejected": -11.977313041687012,
-      "semantic_entropy": 0.03501223772764206,
+      "logits/chosen": 0.06115398555994034,
+      "logits/rejected": 0.2414974868297577,
+      "logps/chosen": -6.048588752746582,
+      "logps/rejected": -5.88077974319458,
+      "loss": 0.0897,
+      "rewards/accuracies": 0.48124998807907104,
+      "rewards/chosen": -6.048588752746582,
+      "rewards/margins": -0.16780927777290344,
+      "rewards/rejected": -5.88077974319458,
       "step": 285
     },
     {
       "epoch": 0.1552099013212912,
-      "grad_norm": 0.9201296423061804,
+      "grad_norm": 1.4460128553048879,
       "learning_rate": 5.169340463458111e-07,
-      "logits/chosen": 1.6455614566802979,
-      "logits/rejected": 1.8774665594100952,
-      "logps/chosen": -14.004234313964844,
-      "logps/rejected": -16.590394973754883,
-      "loss": 0.0067,
+      "logits/chosen": 0.08946628123521805,
+      "logits/rejected": 0.4639100432395935,
+      "logps/chosen": -5.004059791564941,
+      "logps/rejected": -6.311199188232422,
+      "loss": 0.0526,
       "rewards/accuracies": 0.606249988079071,
-      "rewards/chosen": -14.004234313964844,
-      "rewards/margins": 2.5861611366271973,
-      "rewards/rejected": -16.590394973754883,
-      "semantic_entropy": 0.01032390259206295,
+      "rewards/chosen": -5.004059791564941,
+      "rewards/margins": 1.3071396350860596,
+      "rewards/rejected": -6.311199188232422,
       "step": 290
     },
     {
       "epoch": 0.15788593410269275,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.091528892126652,
       "learning_rate": 5.258467023172905e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 1.0269,
-      "rewards/accuracies": 0.38749998807907104,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 0.1946166455745697,
+      "logits/rejected": 0.2628430426120758,
+      "logps/chosen": -7.0160675048828125,
+      "logps/rejected": -6.312521934509277,
+      "loss": 0.0651,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -7.0160675048828125,
+      "rewards/margins": -0.7035457491874695,
+      "rewards/rejected": -6.312521934509277,
       "step": 295
     },
     {
       "epoch": 0.16056196688409433,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.919579238293798,
       "learning_rate": 5.347593582887701e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 0.3639879822731018,
+      "logits/rejected": 0.5850847363471985,
+      "logps/chosen": -8.438272476196289,
+      "logps/rejected": -8.288660049438477,
+      "loss": 0.041,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -8.438272476196289,
+      "rewards/margins": -0.14961174130439758,
+      "rewards/rejected": -8.288660049438477,
       "step": 300
     },
     {
       "epoch": 0.1632379996654959,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.7385314438585548,
       "learning_rate": 5.436720142602496e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 0.8661595582962036,
+      "logits/rejected": 0.966326117515564,
+      "logps/chosen": -10.178471565246582,
+      "logps/rejected": -9.510921478271484,
+      "loss": 0.0148,
+      "rewards/accuracies": 0.48124998807907104,
+      "rewards/chosen": -10.178471565246582,
+      "rewards/margins": -0.6675509214401245,
+      "rewards/rejected": -9.510921478271484,
       "step": 305
     },
     {
       "epoch": 0.16591403244689748,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.2433192138162146,
       "learning_rate": 5.52584670231729e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 1.4416261911392212,
+      "logits/rejected": 1.5715261697769165,
+      "logps/chosen": -13.690839767456055,
+      "logps/rejected": -14.78709888458252,
+      "loss": 0.0053,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -13.690839767456055,
+      "rewards/margins": 1.0962581634521484,
+      "rewards/rejected": -14.78709888458252,
       "step": 310
     },
     {
       "epoch": 0.16859006522829906,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 0.43788855250304975,
       "learning_rate": 5.614973262032086e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 2.143500804901123,
+      "logits/rejected": 2.398965358734131,
+      "logps/chosen": -16.440887451171875,
+      "logps/rejected": -18.3090877532959,
+      "loss": 0.0041,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -16.440887451171875,
+      "rewards/margins": 1.8682035207748413,
+      "rewards/rejected": -18.3090877532959,
       "step": 315
     },
     {
       "epoch": 0.1712660980097006,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 0.1515220476027461,
       "learning_rate": 5.70409982174688e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 2.4473185539245605,
+      "logits/rejected": 2.6145071983337402,
+      "logps/chosen": -17.95746421813965,
+      "logps/rejected": -18.43484115600586,
+      "loss": 0.0006,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -17.95746421813965,
+      "rewards/margins": 0.4773777425289154,
+      "rewards/rejected": -18.43484115600586,
       "step": 320
     },
     {
       "epoch": 0.17394213079110218,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 0.0934587649699255,
       "learning_rate": 5.793226381461676e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 3.1694912910461426,
+      "logits/rejected": 3.363656997680664,
+      "logps/chosen": -21.727306365966797,
+      "logps/rejected": -23.835086822509766,
+      "loss": 0.0012,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -21.727306365966797,
+      "rewards/margins": 2.107780933380127,
+      "rewards/rejected": -23.835086822509766,
       "step": 325
     },
     {
       "epoch": 0.17661816357250376,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 0.04289188015821034,
       "learning_rate": 5.88235294117647e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 3.4928061962127686,
+      "logits/rejected": 3.451262950897217,
+      "logps/chosen": -21.600542068481445,
+      "logps/rejected": -24.803955078125,
+      "loss": 0.0009,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -21.600542068481445,
+      "rewards/margins": 3.203413486480713,
+      "rewards/rejected": -24.803955078125,
       "step": 330
     },
     {
       "epoch": 0.17929419635390534,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.4645623935559486,
       "learning_rate": 5.971479500891266e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 4.0196733474731445,
+      "logits/rejected": 4.058861255645752,
+      "logps/chosen": -24.990421295166016,
+      "logps/rejected": -25.708927154541016,
+      "loss": 0.0007,
+      "rewards/accuracies": 0.4749999940395355,
+      "rewards/chosen": -24.990421295166016,
+      "rewards/margins": 0.7185049653053284,
+      "rewards/rejected": -25.708927154541016,
       "step": 335
     },
     {
       "epoch": 0.18197022913530692,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 0.04214895746629256,
       "learning_rate": 6.060606060606061e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 4.27630615234375,
+      "logits/rejected": 4.299677848815918,
+      "logps/chosen": -25.054115295410156,
+      "logps/rejected": -27.55368423461914,
+      "loss": 0.0,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -25.054115295410156,
+      "rewards/margins": 2.4995665550231934,
+      "rewards/rejected": -27.55368423461914,
       "step": 340
     },
     {
       "epoch": 0.1846462619167085,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 0.04112378062024114,
       "learning_rate": 6.149732620320855e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 4.434001445770264,
+      "logits/rejected": 4.55509090423584,
+      "logps/chosen": -25.489980697631836,
+      "logps/rejected": -26.87412452697754,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -25.489980697631836,
+      "rewards/margins": 1.3841450214385986,
+      "rewards/rejected": -26.87412452697754,
       "step": 345
     },
     {
       "epoch": 0.18732229469811004,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 0.00042743414594228086,
       "learning_rate": 6.238859180035651e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 4.505887031555176,
+      "logits/rejected": 4.699957847595215,
+      "logps/chosen": -25.814910888671875,
+      "logps/rejected": -28.64125633239746,
+      "loss": 0.001,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -25.814910888671875,
+      "rewards/margins": 2.8263463973999023,
+      "rewards/rejected": -28.64125633239746,
       "step": 350
     },
     {
       "epoch": 0.18999832747951162,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 0.035879226433014506,
       "learning_rate": 6.327985739750445e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 5.051446914672852,
+      "logits/rejected": 4.953315734863281,
+      "logps/chosen": -27.82820701599121,
+      "logps/rejected": -29.5991268157959,
+      "loss": 0.0,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -27.82820701599121,
+      "rewards/margins": 1.7709205150604248,
+      "rewards/rejected": -29.5991268157959,
       "step": 355
     },
     {
       "epoch": 0.1926743602609132,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 0.0003175469168009553,
       "learning_rate": 6.417112299465241e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 5.0589118003845215,
+      "logits/rejected": 5.075278282165527,
+      "logps/chosen": -29.5541934967041,
+      "logps/rejected": -30.4432430267334,
+      "loss": 0.0002,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -29.5541934967041,
+      "rewards/margins": 0.8890513181686401,
+      "rewards/rejected": -30.4432430267334,
       "step": 360
     },
     {
       "epoch": 0.19535039304231477,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 0.0051673080346850316,
       "learning_rate": 6.506238859180035e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 4.929441928863525,
+      "logits/rejected": 4.989992141723633,
+      "logps/chosen": -28.256805419921875,
+      "logps/rejected": -28.80402183532715,
+      "loss": 0.0005,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -28.256805419921875,
+      "rewards/margins": 0.5472115278244019,
+      "rewards/rejected": -28.80402183532715,
       "step": 365
     },
     {
       "epoch": 0.19802642582371635,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 0.01031437573004315,
       "learning_rate": 6.59536541889483e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 5.048266410827637,
+      "logits/rejected": 4.988327980041504,
+      "logps/chosen": -27.683841705322266,
+      "logps/rejected": -28.48501968383789,
+      "loss": 0.0001,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -27.683841705322266,
+      "rewards/margins": 0.801181435585022,
+      "rewards/rejected": -28.48501968383789,
       "step": 370
     },
     {
       "epoch": 0.2007024586051179,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 0.04705777109675603,
       "learning_rate": 6.684491978609626e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 5.017124176025391,
+      "logits/rejected": 4.993551731109619,
+      "logps/chosen": -27.910104751586914,
+      "logps/rejected": -30.440393447875977,
+      "loss": 0.0,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -27.910104751586914,
+      "rewards/margins": 2.530287504196167,
+      "rewards/rejected": -30.440393447875977,
       "step": 375
     },
     {
       "epoch": 0.20337849138651948,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.116720502400147e-05,
       "learning_rate": 6.77361853832442e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 5.353625297546387,
+      "logits/rejected": 5.556002616882324,
+      "logps/chosen": -29.577198028564453,
+      "logps/rejected": -32.58041000366211,
+      "loss": 0.0,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -29.577198028564453,
+      "rewards/margins": 3.003213405609131,
+      "rewards/rejected": -32.58041000366211,
       "step": 380
     },
     {
       "epoch": 0.20605452416792105,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 0.00010713210858484258,
       "learning_rate": 6.862745098039216e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 5.450345516204834,
+      "logits/rejected": 5.39117431640625,
+      "logps/chosen": -29.253742218017578,
+      "logps/rejected": -31.753101348876953,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -29.253742218017578,
+      "rewards/margins": 2.4993581771850586,
+      "rewards/rejected": -31.753101348876953,
       "step": 385
     },
     {
       "epoch": 0.20873055694932263,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 0.00553465724178475,
       "learning_rate": 6.95187165775401e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 5.020052433013916,
+      "logits/rejected": 4.734940052032471,
+      "logps/chosen": -29.95306968688965,
+      "logps/rejected": -31.805065155029297,
+      "loss": 0.0017,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -29.95306968688965,
+      "rewards/margins": 1.8519980907440186,
+      "rewards/rejected": -31.805065155029297,
       "step": 390
     },
     {
       "epoch": 0.2114065897307242,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 0.004926924435489586,
       "learning_rate": 7.040998217468806e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 5.617483615875244,
+      "logits/rejected": 5.613303184509277,
+      "logps/chosen": -29.747699737548828,
+      "logps/rejected": -32.044166564941406,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -29.747699737548828,
+      "rewards/margins": 2.296464204788208,
+      "rewards/rejected": -32.044166564941406,
       "step": 395
     },
     {
       "epoch": 0.2140826225121258,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 0.10980056588215964,
       "learning_rate": 7.1301247771836e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 5.197115421295166,
+      "logits/rejected": 5.246896266937256,
+      "logps/chosen": -29.11849021911621,
+      "logps/rejected": -31.373455047607422,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -29.11849021911621,
+      "rewards/margins": 2.2549638748168945,
+      "rewards/rejected": -31.373455047607422,
       "step": 400
     },
     {
       "epoch": 0.2140826225121258,
-      "eval_logits/chosen": NaN,
-      "eval_logits/rejected": NaN,
-      "eval_logps/chosen": NaN,
-      "eval_logps/rejected": NaN,
-      "eval_loss": NaN,
-      "eval_rewards/accuracies": 0.0,
-      "eval_rewards/chosen": NaN,
-      "eval_rewards/margins": NaN,
-      "eval_rewards/rejected": NaN,
-      "eval_runtime": 36.0123,
-      "eval_samples_per_second": 37.348,
-      "eval_semantic_entropy": NaN,
-      "eval_steps_per_second": 9.358,
+      "eval_logits/chosen": 4.508347988128662,
+      "eval_logits/rejected": 4.525824546813965,
+      "eval_logps/chosen": -29.474035263061523,
+      "eval_logps/rejected": -31.223838806152344,
+      "eval_loss": 0.00014844455290585756,
+      "eval_rewards/accuracies": 0.5689911246299744,
+      "eval_rewards/chosen": -29.474035263061523,
+      "eval_rewards/margins": 1.749806523323059,
+      "eval_rewards/rejected": -31.223838806152344,
+      "eval_runtime": 41.4755,
+      "eval_samples_per_second": 32.429,
+      "eval_steps_per_second": 8.125,
       "step": 400
     },
     {
       "epoch": 0.21675865529352734,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.0460074400129147e-07,
       "learning_rate": 7.219251336898395e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 5.426613807678223,
+      "logits/rejected": 5.434569358825684,
+      "logps/chosen": -31.552480697631836,
+      "logps/rejected": -33.021236419677734,
+      "loss": 0.0001,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": -31.552480697631836,
+      "rewards/margins": 1.468758225440979,
+      "rewards/rejected": -33.021236419677734,
       "step": 405
     },
     {
       "epoch": 0.2194346880749289,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 0.0018394903367872774,
       "learning_rate": 7.30837789661319e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 5.585395812988281,
+      "logits/rejected": 5.505690574645996,
+      "logps/chosen": -30.765277862548828,
+      "logps/rejected": -33.54123306274414,
+      "loss": 0.0,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -30.765277862548828,
+      "rewards/margins": 2.7759573459625244,
+      "rewards/rejected": -33.54123306274414,
       "step": 410
     },
     {
       "epoch": 0.2221107208563305,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 9.519517465463827e-07,
       "learning_rate": 7.397504456327985e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 5.678066730499268,
+      "logits/rejected": 5.77048397064209,
+      "logps/chosen": -32.274635314941406,
+      "logps/rejected": -33.142452239990234,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -32.274635314941406,
+      "rewards/margins": 0.8678176999092102,
+      "rewards/rejected": -33.142452239990234,
       "step": 415
     },
     {
       "epoch": 0.22478675363773207,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 0.028954290644148215,
       "learning_rate": 7.486631016042781e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 5.4329633712768555,
+      "logits/rejected": 5.234847068786621,
+      "logps/chosen": -30.790943145751953,
+      "logps/rejected": -33.33657455444336,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -30.790943145751953,
+      "rewards/margins": 2.545635938644409,
+      "rewards/rejected": -33.33657455444336,
       "step": 420
     },
     {
       "epoch": 0.22746278641913364,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 0.0006987982490430461,
       "learning_rate": 7.575757575757575e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 5.261269569396973,
+      "logits/rejected": 5.335232257843018,
+      "logps/chosen": -30.87042808532715,
+      "logps/rejected": -33.61421203613281,
+      "loss": 0.0,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -30.87042808532715,
+      "rewards/margins": 2.743783950805664,
+      "rewards/rejected": -33.61421203613281,
       "step": 425
     },
     {
       "epoch": 0.2301388192005352,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 0.008281334251306848,
       "learning_rate": 7.664884135472371e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 5.641324043273926,
+      "logits/rejected": 5.514430522918701,
+      "logps/chosen": -28.550350189208984,
+      "logps/rejected": -32.46430587768555,
+      "loss": 0.0,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -28.550350189208984,
+      "rewards/margins": 3.9139564037323,
+      "rewards/rejected": -32.46430587768555,
       "step": 430
     },
     {
       "epoch": 0.23281485198193677,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 0.0001335339701865778,
       "learning_rate": 7.754010695187165e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 5.681117057800293,
+      "logits/rejected": 5.728180408477783,
+      "logps/chosen": -30.91018295288086,
+      "logps/rejected": -31.73850440979004,
+      "loss": 0.0001,
+      "rewards/accuracies": 0.48124998807907104,
+      "rewards/chosen": -30.91018295288086,
+      "rewards/margins": 0.8283231854438782,
+      "rewards/rejected": -31.73850440979004,
       "step": 435
     },
     {
       "epoch": 0.23549088476333835,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 0.0003905120367096067,
       "learning_rate": 7.84313725490196e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 5.308198928833008,
+      "logits/rejected": 5.242469310760498,
+      "logps/chosen": -28.735559463500977,
+      "logps/rejected": -30.480484008789062,
+      "loss": 0.0,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -28.735559463500977,
+      "rewards/margins": 1.7449262142181396,
+      "rewards/rejected": -30.480484008789062,
       "step": 440
     },
     {
       "epoch": 0.23816691754473993,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 0.035724152997041105,
       "learning_rate": 7.932263814616755e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 5.429460048675537,
+      "logits/rejected": 5.417057037353516,
+      "logps/chosen": -29.862049102783203,
+      "logps/rejected": -31.693466186523438,
+      "loss": 0.0001,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -29.862049102783203,
+      "rewards/margins": 1.8314167261123657,
+      "rewards/rejected": -31.693466186523438,
       "step": 445
     },
     {
       "epoch": 0.2408429503261415,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 0.0008963971978067854,
       "learning_rate": 8.02139037433155e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 5.698996543884277,
+      "logits/rejected": 5.760076522827148,
+      "logps/chosen": -31.261394500732422,
+      "logps/rejected": -33.20148468017578,
+      "loss": 0.0,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -31.261394500732422,
+      "rewards/margins": 1.9400899410247803,
+      "rewards/rejected": -33.20148468017578,
       "step": 450
     },
     {
       "epoch": 0.24351898310754308,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 0.0002816622127148177,
       "learning_rate": 8.110516934046346e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 5.82552433013916,
+      "logits/rejected": 5.86575174331665,
+      "logps/chosen": -29.802892684936523,
+      "logps/rejected": -31.987655639648438,
+      "loss": 0.0,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -29.802892684936523,
+      "rewards/margins": 2.184760570526123,
+      "rewards/rejected": -31.987655639648438,
       "step": 455
     },
     {
       "epoch": 0.24619501588894463,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 0.0007354757513253456,
       "learning_rate": 8.19964349376114e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 5.799552917480469,
+      "logits/rejected": 5.78856897354126,
+      "logps/chosen": -30.004852294921875,
+      "logps/rejected": -32.112491607666016,
+      "loss": 0.0,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -30.004852294921875,
+      "rewards/margins": 2.1076369285583496,
+      "rewards/rejected": -32.112491607666016,
       "step": 460
     },
     {
       "epoch": 0.2488710486703462,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 0.00249200164181285,
       "learning_rate": 8.288770053475936e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 5.848760604858398,
+      "logits/rejected": 5.7724995613098145,
+      "logps/chosen": -32.0567512512207,
+      "logps/rejected": -33.11432647705078,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -32.0567512512207,
+      "rewards/margins": 1.057578444480896,
+      "rewards/rejected": -33.11432647705078,
       "step": 465
     },
     {
       "epoch": 0.2515470814517478,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 0.008788564272295995,
       "learning_rate": 8.37789661319073e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 6.024893760681152,
+      "logits/rejected": 6.152130126953125,
+      "logps/chosen": -33.38634490966797,
+      "logps/rejected": -33.031341552734375,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -33.38634490966797,
+      "rewards/margins": -0.3550027012825012,
+      "rewards/rejected": -33.031341552734375,
       "step": 470
     },
     {
       "epoch": 0.25422311423314936,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 0.008045023749322238,
       "learning_rate": 8.467023172905525e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 6.149905204772949,
+      "logits/rejected": 6.1068315505981445,
+      "logps/chosen": -31.69088363647461,
+      "logps/rejected": -35.666839599609375,
+      "loss": 0.0,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -31.69088363647461,
+      "rewards/margins": 3.9759507179260254,
+      "rewards/rejected": -35.666839599609375,
       "step": 475
     },
     {
       "epoch": 0.2568991470145509,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 0.24161562793150843,
       "learning_rate": 8.55614973262032e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 6.131672382354736,
+      "logits/rejected": 6.029187202453613,
+      "logps/chosen": -31.79244041442871,
+      "logps/rejected": -34.317020416259766,
+      "loss": 0.0001,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -31.79244041442871,
+      "rewards/margins": 2.5245766639709473,
+      "rewards/rejected": -34.317020416259766,
       "step": 480
     },
     {
       "epoch": 0.2595751797959525,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 0.0006102102592899613,
       "learning_rate": 8.645276292335115e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 6.5127692222595215,
+      "logits/rejected": 6.600030422210693,
+      "logps/chosen": -33.49677276611328,
+      "logps/rejected": -35.524227142333984,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -33.49677276611328,
+      "rewards/margins": 2.027453899383545,
+      "rewards/rejected": -35.524227142333984,
       "step": 485
     },
     {
       "epoch": 0.26225121257735406,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 0.0009493708808510832,
       "learning_rate": 8.734402852049911e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 5.9582061767578125,
+      "logits/rejected": 6.046202182769775,
+      "logps/chosen": -33.23215866088867,
+      "logps/rejected": -34.372230529785156,
+      "loss": 0.0002,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": -33.23215866088867,
+      "rewards/margins": 1.140074610710144,
+      "rewards/rejected": -34.372230529785156,
       "step": 490
     },
     {
       "epoch": 0.26492724535875567,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 0.00022864888927709976,
       "learning_rate": 8.823529411764705e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 6.260375022888184,
+      "logits/rejected": 6.266873836517334,
+      "logps/chosen": -35.20757293701172,
+      "logps/rejected": -34.25310134887695,
+      "loss": 0.0,
+      "rewards/accuracies": 0.48750001192092896,
+      "rewards/chosen": -35.20757293701172,
+      "rewards/margins": -0.9544719457626343,
+      "rewards/rejected": -34.25310134887695,
       "step": 495
     },
     {
       "epoch": 0.2676032781401572,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 0.00019236693309136163,
       "learning_rate": 8.912655971479501e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 6.197518348693848,
+      "logits/rejected": 6.117464065551758,
+      "logps/chosen": -33.92963790893555,
+      "logps/rejected": -34.88914108276367,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5062500238418579,
+      "rewards/chosen": -33.92963790893555,
+      "rewards/margins": 0.9595043063163757,
+      "rewards/rejected": -34.88914108276367,
       "step": 500
     },
     {
       "epoch": 0.27027931092155877,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 0.11369246832499683,
       "learning_rate": 9.001782531194295e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 6.148838043212891,
+      "logits/rejected": 6.055180549621582,
+      "logps/chosen": -32.2354850769043,
+      "logps/rejected": -33.38620376586914,
+      "loss": 0.0014,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -32.2354850769043,
+      "rewards/margins": 1.1507171392440796,
+      "rewards/rejected": -33.38620376586914,
       "step": 505
     },
     {
       "epoch": 0.2729553437029604,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 0.00024122807731692076,
       "learning_rate": 9.09090909090909e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 6.060546875,
+      "logits/rejected": 6.003035068511963,
+      "logps/chosen": -33.80747985839844,
+      "logps/rejected": -34.501033782958984,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -33.80747985839844,
+      "rewards/margins": 0.6935546398162842,
+      "rewards/rejected": -34.501033782958984,
       "step": 510
     },
     {
       "epoch": 0.2756313764843619,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 0.0017872707280551482,
       "learning_rate": 9.180035650623885e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 5.7357988357543945,
+      "logits/rejected": 5.852870941162109,
+      "logps/chosen": -31.31736183166504,
+      "logps/rejected": -33.037017822265625,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -31.31736183166504,
+      "rewards/margins": 1.7196519374847412,
+      "rewards/rejected": -33.037017822265625,
       "step": 515
     },
     {
       "epoch": 0.27830740926576353,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.6238419454377426e-05,
       "learning_rate": 9.26916221033868e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 6.473957061767578,
+      "logits/rejected": 6.2191596031188965,
+      "logps/chosen": -34.885589599609375,
+      "logps/rejected": -35.81165313720703,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -34.885589599609375,
+      "rewards/margins": 0.9260598421096802,
+      "rewards/rejected": -35.81165313720703,
       "step": 520
     },
     {
       "epoch": 0.2809834420471651,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 0.0014621432044564487,
       "learning_rate": 9.358288770053476e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 6.131948947906494,
+      "logits/rejected": 6.05927038192749,
+      "logps/chosen": -33.809661865234375,
+      "logps/rejected": -35.10086441040039,
+      "loss": 0.0,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -33.809661865234375,
+      "rewards/margins": 1.2912040948867798,
+      "rewards/rejected": -35.10086441040039,
       "step": 525
     },
     {
       "epoch": 0.2836594748285666,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 0.035805550797865744,
       "learning_rate": 9.44741532976827e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 6.354075908660889,
+      "logits/rejected": 6.551912784576416,
+      "logps/chosen": -33.002723693847656,
+      "logps/rejected": -34.30055618286133,
+      "loss": 0.0,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -33.002723693847656,
+      "rewards/margins": 1.2978332042694092,
+      "rewards/rejected": -34.30055618286133,
       "step": 530
     },
     {
       "epoch": 0.28633550760996823,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.2210176107842878e-05,
       "learning_rate": 9.536541889483066e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 6.335618019104004,
+      "logits/rejected": 5.982363700866699,
+      "logps/chosen": -32.45991134643555,
+      "logps/rejected": -36.38939666748047,
+      "loss": 0.0,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -32.45991134643555,
+      "rewards/margins": 3.929482936859131,
+      "rewards/rejected": -36.38939666748047,
       "step": 535
     },
     {
       "epoch": 0.2890115403913698,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 0.0014578279830980442,
       "learning_rate": 9.62566844919786e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 6.4163498878479,
+      "logits/rejected": 6.378708362579346,
+      "logps/chosen": -32.91603469848633,
+      "logps/rejected": -34.10979461669922,
+      "loss": 0.0,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -32.91603469848633,
+      "rewards/margins": 1.193760871887207,
+      "rewards/rejected": -34.10979461669922,
       "step": 540
     },
     {
       "epoch": 0.2916875731727714,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 0.009891179319808723,
       "learning_rate": 9.714795008912655e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 6.257334232330322,
+      "logits/rejected": 6.111904144287109,
+      "logps/chosen": -32.97081756591797,
+      "logps/rejected": -36.07368087768555,
+      "loss": 0.0,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -32.97081756591797,
+      "rewards/margins": 3.102867841720581,
+      "rewards/rejected": -36.07368087768555,
       "step": 545
     },
     {
       "epoch": 0.29436360595417294,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 0.00010441327591347197,
       "learning_rate": 9.80392156862745e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 6.489959716796875,
+      "logits/rejected": 6.449127197265625,
+      "logps/chosen": -33.431461334228516,
+      "logps/rejected": -34.821311950683594,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -33.431461334228516,
+      "rewards/margins": 1.389845371246338,
+      "rewards/rejected": -34.821311950683594,
       "step": 550
     },
     {
       "epoch": 0.2970396387355745,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 0.006376648448372838,
       "learning_rate": 9.893048128342244e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 6.40233850479126,
+      "logits/rejected": 6.086165904998779,
+      "logps/chosen": -34.14889907836914,
+      "logps/rejected": -34.164825439453125,
+      "loss": 0.0,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -34.14889907836914,
+      "rewards/margins": 0.01592273637652397,
+      "rewards/rejected": -34.164825439453125,
       "step": 555
     },
     {
       "epoch": 0.2997156715169761,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 0.017814151085457603,
       "learning_rate": 9.98217468805704e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 6.360291957855225,
+      "logits/rejected": 6.425871849060059,
+      "logps/chosen": -33.9993896484375,
+      "logps/rejected": -34.31085968017578,
+      "loss": 0.0,
+      "rewards/accuracies": 0.45625001192092896,
+      "rewards/chosen": -33.9993896484375,
+      "rewards/margins": 0.3114696443080902,
+      "rewards/rejected": -34.31085968017578,
       "step": 560
     },
     {
       "epoch": 0.30239170429837764,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 0.0005208945254706291,
       "learning_rate": 9.999984476788462e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 6.604605197906494,
+      "logits/rejected": 6.542850494384766,
+      "logps/chosen": -33.41400909423828,
+      "logps/rejected": -35.229984283447266,
+      "loss": 0.0,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -33.41400909423828,
+      "rewards/margins": 1.8159730434417725,
+      "rewards/rejected": -35.229984283447266,
       "step": 565
     },
     {
       "epoch": 0.30506773707977924,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.1017687150351856e-05,
       "learning_rate": 9.999921413906797e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 6.328225135803223,
+      "logits/rejected": 6.258444786071777,
+      "logps/chosen": -32.51350784301758,
+      "logps/rejected": -35.06706237792969,
+      "loss": 0.0,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -32.51350784301758,
+      "rewards/margins": 2.553555488586426,
+      "rewards/rejected": -35.06706237792969,
       "step": 570
     },
     {
       "epoch": 0.3077437698611808,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.325106964892115e-06,
       "learning_rate": 9.999809841765644e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 6.846964359283447,
+      "logits/rejected": 7.237646579742432,
+      "logps/chosen": -34.89623260498047,
+      "logps/rejected": -36.382240295410156,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -34.89623260498047,
+      "rewards/margins": 1.4860029220581055,
+      "rewards/rejected": -36.382240295410156,
       "step": 575
     },
     {
       "epoch": 0.3104198026425824,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 9.052197419586339e-06,
       "learning_rate": 9.999649761447477e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 7.036424160003662,
+      "logits/rejected": 6.716401100158691,
+      "logps/chosen": -34.93789291381836,
+      "logps/rejected": -36.21113204956055,
+      "loss": 0.0,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -34.93789291381836,
+      "rewards/margins": 1.273238182067871,
+      "rewards/rejected": -36.21113204956055,
       "step": 580
     },
     {
       "epoch": 0.31309583542398395,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 0.0013050613006307422,
       "learning_rate": 9.999441174505398e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 6.6325883865356445,
+      "logits/rejected": 6.768698215484619,
+      "logps/chosen": -34.55119323730469,
+      "logps/rejected": -36.57561492919922,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -34.55119323730469,
+      "rewards/margins": 2.0244240760803223,
+      "rewards/rejected": -36.57561492919922,
       "step": 585
     },
     {
       "epoch": 0.3157718682053855,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.135330629953316e-07,
       "learning_rate": 9.999184082963116e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 6.797358512878418,
+      "logits/rejected": 6.864867210388184,
+      "logps/chosen": -34.69548034667969,
+      "logps/rejected": -36.387203216552734,
+      "loss": 0.0,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -34.69548034667969,
+      "rewards/margins": 1.6917203664779663,
+      "rewards/rejected": -36.387203216552734,
       "step": 590
     },
     {
       "epoch": 0.3184479009867871,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 0.0011523503667480658,
       "learning_rate": 9.998878489314937e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 6.9756598472595215,
+      "logits/rejected": 6.642075538635254,
+      "logps/chosen": -34.88782501220703,
+      "logps/rejected": -37.1215705871582,
+      "loss": 0.0,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -34.88782501220703,
+      "rewards/margins": 2.2337491512298584,
+      "rewards/rejected": -37.1215705871582,
       "step": 595
     },
     {
       "epoch": 0.32112393376818865,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 0.0008748957514337547,
       "learning_rate": 9.99852439652573e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 6.799266815185547,
+      "logits/rejected": 6.709418296813965,
+      "logps/chosen": -34.214942932128906,
+      "logps/rejected": -35.7734260559082,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -34.214942932128906,
+      "rewards/margins": 1.5584805011749268,
+      "rewards/rejected": -35.7734260559082,
       "step": 600
     },
     {
       "epoch": 0.32379996654959026,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.320098313797268e-06,
       "learning_rate": 9.998121808030904e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 6.866464138031006,
+      "logits/rejected": 6.8782806396484375,
+      "logps/chosen": -35.04458999633789,
+      "logps/rejected": -36.22504806518555,
+      "loss": 0.0,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -35.04458999633789,
+      "rewards/margins": 1.1804614067077637,
+      "rewards/rejected": -36.22504806518555,
       "step": 605
     },
     {
       "epoch": 0.3264759993309918,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 0.0014950732437660522,
       "learning_rate": 9.997670727736379e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 6.629367828369141,
+      "logits/rejected": 6.501309394836426,
+      "logps/chosen": -34.2564582824707,
+      "logps/rejected": -36.680213928222656,
+      "loss": 0.0,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -34.2564582824707,
+      "rewards/margins": 2.4237563610076904,
+      "rewards/rejected": -36.680213928222656,
       "step": 610
     },
     {
       "epoch": 0.32915203211239336,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.893172143427414e-05,
       "learning_rate": 9.99717116001853e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 6.801131248474121,
+      "logits/rejected": 6.845280647277832,
+      "logps/chosen": -33.56496047973633,
+      "logps/rejected": -36.0307731628418,
+      "loss": 0.0001,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -33.56496047973633,
+      "rewards/margins": 2.465816020965576,
+      "rewards/rejected": -36.0307731628418,
       "step": 615
     },
     {
       "epoch": 0.33182806489379496,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 6.719738160788306e-07,
       "learning_rate": 9.996623109724173e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 6.665761470794678,
+      "logits/rejected": 6.588541507720947,
+      "logps/chosen": -34.28596878051758,
+      "logps/rejected": -35.750144958496094,
+      "loss": 0.0001,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -34.28596878051758,
+      "rewards/margins": 1.4641757011413574,
+      "rewards/rejected": -35.750144958496094,
       "step": 620
     },
     {
       "epoch": 0.3345040976751965,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.4929370534075512e-06,
       "learning_rate": 9.996026582170488e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 6.752054691314697,
+      "logits/rejected": 6.4877800941467285,
+      "logps/chosen": -34.279685974121094,
+      "logps/rejected": -36.55657196044922,
+      "loss": 0.0,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -34.279685974121094,
+      "rewards/margins": 2.276893138885498,
+      "rewards/rejected": -36.55657196044922,
       "step": 625
     },
     {
       "epoch": 0.3371801304565981,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.105902258468344e-06,
       "learning_rate": 9.995381583144996e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 6.7286577224731445,
+      "logits/rejected": 6.735819339752197,
+      "logps/chosen": -33.57708740234375,
+      "logps/rejected": -35.56078338623047,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -33.57708740234375,
+      "rewards/margins": 1.9836965799331665,
+      "rewards/rejected": -35.56078338623047,
       "step": 630
     },
     {
       "epoch": 0.33985616323799966,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 0.0001693453599894774,
       "learning_rate": 9.994688118905471e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 7.191317558288574,
+      "logits/rejected": 6.851290225982666,
+      "logps/chosen": -34.74304962158203,
+      "logps/rejected": -37.620323181152344,
+      "loss": 0.0,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -34.74304962158203,
+      "rewards/margins": 2.877272367477417,
+      "rewards/rejected": -37.620323181152344,
       "step": 635
     },
     {
       "epoch": 0.3425321960194012,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 6.510103094074435e-05,
       "learning_rate": 9.993946196179912e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 6.8470869064331055,
+      "logits/rejected": 6.5663275718688965,
+      "logps/chosen": -35.054527282714844,
+      "logps/rejected": -36.634910583496094,
+      "loss": 0.0,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": -35.054527282714844,
+      "rewards/margins": 1.5803860425949097,
+      "rewards/rejected": -36.634910583496094,
       "step": 640
     },
     {
       "epoch": 0.3452082288008028,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 7.303451703777848e-06,
       "learning_rate": 9.993155822166455e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 7.290508270263672,
+      "logits/rejected": 7.090371131896973,
+      "logps/chosen": -35.157752990722656,
+      "logps/rejected": -36.928775787353516,
+      "loss": 0.0001,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -35.157752990722656,
+      "rewards/margins": 1.77102530002594,
+      "rewards/rejected": -36.928775787353516,
       "step": 645
     },
     {
       "epoch": 0.34788426158220437,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.0119487349165119e-05,
       "learning_rate": 9.992317004533313e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 7.054312229156494,
+      "logits/rejected": 7.178700923919678,
+      "logps/chosen": -34.85392379760742,
+      "logps/rejected": -36.34272003173828,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -34.85392379760742,
+      "rewards/margins": 1.4887895584106445,
+      "rewards/rejected": -36.34272003173828,
       "step": 650
     },
     {
       "epoch": 0.350560294363606,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 0.00014489568910877493,
       "learning_rate": 9.991429751418696e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 6.902565956115723,
+      "logits/rejected": 7.105119228363037,
+      "logps/chosen": -35.772254943847656,
+      "logps/rejected": -36.574886322021484,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -35.772254943847656,
+      "rewards/margins": 0.8026283383369446,
+      "rewards/rejected": -36.574886322021484,
       "step": 655
     },
     {
       "epoch": 0.3532363271450075,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 0.011996029600377949,
       "learning_rate": 9.99049407143074e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 7.235396385192871,
+      "logits/rejected": 6.983212471008301,
+      "logps/chosen": -35.8619499206543,
+      "logps/rejected": -36.84368133544922,
+      "loss": 0.0,
+      "rewards/accuracies": 0.4937500059604645,
+      "rewards/chosen": -35.8619499206543,
+      "rewards/margins": 0.981733500957489,
+      "rewards/rejected": -36.84368133544922,
       "step": 660
     },
     {
       "epoch": 0.35591235992640907,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.19207999024116e-07,
       "learning_rate": 9.989509973647416e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 6.924723148345947,
+      "logits/rejected": 6.78842830657959,
+      "logps/chosen": -35.5040283203125,
+      "logps/rejected": -36.55635070800781,
+      "loss": 0.0,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -35.5040283203125,
+      "rewards/margins": 1.0523169040679932,
+      "rewards/rejected": -36.55635070800781,
       "step": 665
     },
     {
       "epoch": 0.3585883927078107,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.938915632090982e-05,
       "learning_rate": 9.988477467616445e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 7.461147308349609,
+      "logits/rejected": 7.200994968414307,
+      "logps/chosen": -36.98133850097656,
+      "logps/rejected": -39.15379333496094,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -36.98133850097656,
+      "rewards/margins": 2.172454357147217,
+      "rewards/rejected": -39.15379333496094,
       "step": 670
     },
     {
       "epoch": 0.3612644254892122,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 0.009116824378483615,
       "learning_rate": 9.987396563355205e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 7.330562591552734,
+      "logits/rejected": 7.280575752258301,
+      "logps/chosen": -36.07804870605469,
+      "logps/rejected": -37.525672912597656,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -36.07804870605469,
+      "rewards/margins": 1.4476271867752075,
+      "rewards/rejected": -37.525672912597656,
       "step": 675
     },
     {
       "epoch": 0.36394045827061383,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 0.0031837750399864926,
       "learning_rate": 9.986267271350631e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 7.121218681335449,
+      "logits/rejected": 6.910025596618652,
+      "logps/chosen": -35.7744026184082,
+      "logps/rejected": -37.34346389770508,
+      "loss": 0.0009,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -35.7744026184082,
+      "rewards/margins": 1.5690600872039795,
+      "rewards/rejected": -37.34346389770508,
       "step": 680
     },
     {
       "epoch": 0.3666164910520154,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.116779140640639e-06,
       "learning_rate": 9.985089602559123e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 7.288196563720703,
+      "logits/rejected": 6.879519462585449,
+      "logps/chosen": -34.97138214111328,
+      "logps/rejected": -37.335426330566406,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -34.97138214111328,
+      "rewards/margins": 2.364042282104492,
+      "rewards/rejected": -37.335426330566406,
       "step": 685
     },
     {
       "epoch": 0.369292523833417,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 0.00010529115571167023,
       "learning_rate": 9.983863568406428e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 7.017849922180176,
+      "logits/rejected": 7.060793399810791,
+      "logps/chosen": -35.998897552490234,
+      "logps/rejected": -36.42427444458008,
+      "loss": 0.0003,
+      "rewards/accuracies": 0.5062500238418579,
+      "rewards/chosen": -35.998897552490234,
+      "rewards/margins": 0.4253758490085602,
+      "rewards/rejected": -36.42427444458008,
       "step": 690
     },
     {
       "epoch": 0.37196855661481854,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 0.002090983438652169,
       "learning_rate": 9.982589180787532e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 7.2100653648376465,
+      "logits/rejected": 7.025811672210693,
+      "logps/chosen": -36.97612380981445,
+      "logps/rejected": -37.301292419433594,
+      "loss": 0.0,
+      "rewards/accuracies": 0.48124998807907104,
+      "rewards/chosen": -36.97612380981445,
+      "rewards/margins": 0.32516971230506897,
+      "rewards/rejected": -37.301292419433594,
       "step": 695
     },
     {
       "epoch": 0.3746445893962201,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.97456526812461e-05,
       "learning_rate": 9.981266452066553e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 7.534338474273682,
+      "logits/rejected": 7.202698707580566,
+      "logps/chosen": -36.010597229003906,
+      "logps/rejected": -37.713661193847656,
+      "loss": 0.0,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -36.010597229003906,
+      "rewards/margins": 1.7030658721923828,
+      "rewards/rejected": -37.713661193847656,
       "step": 700
     },
     {
       "epoch": 0.3773206221776217,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 0.048827650815992704,
       "learning_rate": 9.979895395076608e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 7.625168800354004,
+      "logits/rejected": 7.130354404449463,
+      "logps/chosen": -37.0987434387207,
+      "logps/rejected": -38.473106384277344,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -37.0987434387207,
+      "rewards/margins": 1.3743607997894287,
+      "rewards/rejected": -38.473106384277344,
       "step": 705
     },
     {
       "epoch": 0.37999665495902324,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.5723916928794195e-05,
       "learning_rate": 9.9784760231197e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 7.514035224914551,
+      "logits/rejected": 7.577850341796875,
+      "logps/chosen": -37.40643310546875,
+      "logps/rejected": -39.87279510498047,
+      "loss": 0.0,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -37.40643310546875,
+      "rewards/margins": 2.466355800628662,
+      "rewards/rejected": -39.87279510498047,
       "step": 710
     },
     {
       "epoch": 0.38267268774042484,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 0.02494130431079218,
       "learning_rate": 9.97700834996658e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 7.218267917633057,
+      "logits/rejected": 6.9841156005859375,
+      "logps/chosen": -36.930870056152344,
+      "logps/rejected": -38.65314865112305,
+      "loss": 0.0,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -36.930870056152344,
+      "rewards/margins": 1.7222788333892822,
+      "rewards/rejected": -38.65314865112305,
       "step": 715
     },
     {
       "epoch": 0.3853487205218264,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.227201395744357e-05,
       "learning_rate": 9.97549238985662e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 7.365689754486084,
+      "logits/rejected": 6.939404487609863,
+      "logps/chosen": -36.51023483276367,
+      "logps/rejected": -38.6107292175293,
+      "loss": 0.0,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -36.51023483276367,
+      "rewards/margins": 2.100494861602783,
+      "rewards/rejected": -38.6107292175293,
       "step": 720
     },
     {
       "epoch": 0.38802475330322794,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.113062910748177e-07,
       "learning_rate": 9.973928157497674e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 7.783537864685059,
+      "logits/rejected": 7.40472936630249,
+      "logps/chosen": -37.044673919677734,
+      "logps/rejected": -38.82173156738281,
+      "loss": 0.0,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -37.044673919677734,
+      "rewards/margins": 1.7770541906356812,
+      "rewards/rejected": -38.82173156738281,
       "step": 725
     },
     {
       "epoch": 0.39070078608462955,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 0.00027254899155386783,
       "learning_rate": 9.972315668065927e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 7.683008670806885,
+      "logits/rejected": 7.494215965270996,
+      "logps/chosen": -38.76002502441406,
+      "logps/rejected": -39.1976203918457,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -38.76002502441406,
+      "rewards/margins": 0.4375886023044586,
+      "rewards/rejected": -39.1976203918457,
       "step": 730
     },
     {
       "epoch": 0.3933768188660311,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 0.0019263658636157074,
       "learning_rate": 9.97065493720576e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 7.4932732582092285,
+      "logits/rejected": 7.3577423095703125,
+      "logps/chosen": -35.77894973754883,
+      "logps/rejected": -36.88482666015625,
+      "loss": 0.0,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -35.77894973754883,
+      "rewards/margins": 1.105878233909607,
+      "rewards/rejected": -36.88482666015625,
       "step": 735
     },
     {
       "epoch": 0.3960528516474327,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 8.810174202265158e-07,
       "learning_rate": 9.968945981029594e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 7.506727695465088,
+      "logits/rejected": 7.288453102111816,
+      "logps/chosen": -35.85859298706055,
+      "logps/rejected": -38.346832275390625,
+      "loss": 0.0,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -35.85859298706055,
+      "rewards/margins": 2.488232135772705,
+      "rewards/rejected": -38.346832275390625,
       "step": 740
     },
     {
       "epoch": 0.39872888442883425,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.3527867281810915e-05,
       "learning_rate": 9.967188816117726e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 7.556031227111816,
+      "logits/rejected": 7.525795936584473,
+      "logps/chosen": -36.91010284423828,
+      "logps/rejected": -40.40468978881836,
+      "loss": 0.0,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -36.91010284423828,
+      "rewards/margins": 3.4945881366729736,
+      "rewards/rejected": -40.40468978881836,
       "step": 745
     },
     {
       "epoch": 0.4014049172102358,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 0.00016034383285586597,
       "learning_rate": 9.965383459518179e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 7.278078556060791,
+      "logits/rejected": 7.1918463706970215,
+      "logps/chosen": -35.90861129760742,
+      "logps/rejected": -38.47333526611328,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -35.90861129760742,
+      "rewards/margins": 2.564725875854492,
+      "rewards/rejected": -38.47333526611328,
       "step": 750
     },
     {
       "epoch": 0.4040809499916374,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.2081192706073446e-07,
       "learning_rate": 9.963529928746533e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 7.5855560302734375,
+      "logits/rejected": 7.1510820388793945,
+      "logps/chosen": -35.81825637817383,
+      "logps/rejected": -38.087852478027344,
+      "loss": 0.0,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -35.81825637817383,
+      "rewards/margins": 2.269597291946411,
+      "rewards/rejected": -38.087852478027344,
       "step": 755
     },
     {
       "epoch": 0.40675698277303896,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 0.0012603887909422699,
       "learning_rate": 9.961628241785746e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 7.386877536773682,
+      "logits/rejected": 7.278040885925293,
+      "logps/chosen": -36.343971252441406,
+      "logps/rejected": -37.227210998535156,
+      "loss": 0.0,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -36.343971252441406,
+      "rewards/margins": 0.8832328915596008,
+      "rewards/rejected": -37.227210998535156,
       "step": 760
     },
     {
       "epoch": 0.40943301555444056,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 0.0022033366806607176,
       "learning_rate": 9.959678417085998e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 7.282350063323975,
+      "logits/rejected": 7.231588840484619,
+      "logps/chosen": -36.273895263671875,
+      "logps/rejected": -37.92772674560547,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -36.273895263671875,
+      "rewards/margins": 1.6538302898406982,
+      "rewards/rejected": -37.92772674560547,
       "step": 765
     },
     {
       "epoch": 0.4121090483358421,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.074897623286504e-07,
       "learning_rate": 9.957680473564493e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 7.903887748718262,
+      "logits/rejected": 7.689046382904053,
+      "logps/chosen": -38.436553955078125,
+      "logps/rejected": -41.07474136352539,
+      "loss": 0.0,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -38.436553955078125,
+      "rewards/margins": 2.6381802558898926,
+      "rewards/rejected": -41.07474136352539,
       "step": 770
     },
     {
       "epoch": 0.41478508111724366,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 0.0040569328807811625,
       "learning_rate": 9.95563443060529e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 7.836782932281494,
+      "logits/rejected": 7.382288932800293,
+      "logps/chosen": -36.24165344238281,
+      "logps/rejected": -38.431461334228516,
+      "loss": 0.0,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -36.24165344238281,
+      "rewards/margins": 2.1898043155670166,
+      "rewards/rejected": -38.431461334228516,
       "step": 775
     },
     {
       "epoch": 0.41746111389864526,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.2960539710483852e-05,
       "learning_rate": 9.95354030805911e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 7.578127861022949,
+      "logits/rejected": 7.2394304275512695,
+      "logps/chosen": -38.148136138916016,
+      "logps/rejected": -39.74500274658203,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -38.148136138916016,
+      "rewards/margins": 1.596866488456726,
+      "rewards/rejected": -39.74500274658203,
       "step": 780
     },
     {
       "epoch": 0.4201371466800468,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 0.0002551349932661308,
       "learning_rate": 9.951398126243133e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 7.510928153991699,
+      "logits/rejected": 7.400703430175781,
+      "logps/chosen": -37.220542907714844,
+      "logps/rejected": -38.346168518066406,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -37.220542907714844,
+      "rewards/margins": 1.1256234645843506,
+      "rewards/rejected": -38.346168518066406,
       "step": 785
     },
     {
       "epoch": 0.4228131794614484,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 0.00024306372646533492,
       "learning_rate": 9.94920790594082e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 7.612584114074707,
+      "logits/rejected": 7.527153015136719,
+      "logps/chosen": -37.6441764831543,
+      "logps/rejected": -40.033878326416016,
+      "loss": 0.0,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -37.6441764831543,
+      "rewards/margins": 2.389700412750244,
+      "rewards/rejected": -40.033878326416016,
       "step": 790
     },
     {
       "epoch": 0.42548921224284997,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.1333233573464902e-05,
       "learning_rate": 9.946969668401696e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 7.460398197174072,
+      "logits/rejected": 6.999265193939209,
+      "logps/chosen": -36.37274932861328,
+      "logps/rejected": -37.866981506347656,
+      "loss": 0.0,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": -36.37274932861328,
+      "rewards/margins": 1.494228720664978,
+      "rewards/rejected": -37.866981506347656,
       "step": 795
     },
     {
       "epoch": 0.4281652450242516,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.751260225097647e-10,
       "learning_rate": 9.944683435341155e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 7.543933868408203,
+      "logits/rejected": 7.4145402908325195,
+      "logps/chosen": -38.499019622802734,
+      "logps/rejected": -39.2597541809082,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -38.499019622802734,
+      "rewards/margins": 0.7607344388961792,
+      "rewards/rejected": -39.2597541809082,
       "step": 800
     },
     {
       "epoch": 0.4281652450242516,
-      "eval_logits/chosen": NaN,
-      "eval_logits/rejected": NaN,
-      "eval_logps/chosen": NaN,
-      "eval_logps/rejected": NaN,
-      "eval_loss": NaN,
-      "eval_rewards/accuracies": 0.0,
-      "eval_rewards/chosen": NaN,
-      "eval_rewards/margins": NaN,
-      "eval_rewards/rejected": NaN,
-      "eval_runtime": 35.0379,
-      "eval_samples_per_second": 38.387,
-      "eval_semantic_entropy": NaN,
-      "eval_steps_per_second": 9.618,
+      "eval_logits/chosen": 6.36430549621582,
+      "eval_logits/rejected": 6.230695724487305,
+      "eval_logps/chosen": -37.04652404785156,
+      "eval_logps/rejected": -38.52766418457031,
+      "eval_loss": 1.0129234397027176e-05,
+      "eval_rewards/accuracies": 0.5578634738922119,
+      "eval_rewards/chosen": -37.04652404785156,
+      "eval_rewards/margins": 1.4811420440673828,
+      "eval_rewards/rejected": -38.52766418457031,
+      "eval_runtime": 40.8465,
+      "eval_samples_per_second": 32.928,
+      "eval_steps_per_second": 8.25,
       "step": 800
     },
     {
       "epoch": 0.4308412778056531,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 0.008618508269670479,
       "learning_rate": 9.942349228940236e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 7.601966857910156,
+      "logits/rejected": 7.381627082824707,
+      "logps/chosen": -35.658599853515625,
+      "logps/rejected": -39.303565979003906,
+      "loss": 0.0,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -35.658599853515625,
+      "rewards/margins": 3.644965648651123,
+      "rewards/rejected": -39.303565979003906,
       "step": 805
     },
     {
       "epoch": 0.43351731058705467,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.5419006373766908e-09,
       "learning_rate": 9.939967071845424e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 7.644564628601074,
+      "logits/rejected": 7.591551303863525,
+      "logps/chosen": -37.43498611450195,
+      "logps/rejected": -38.69459533691406,
+      "loss": 0.0,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -37.43498611450195,
+      "rewards/margins": 1.2596056461334229,
+      "rewards/rejected": -38.69459533691406,
       "step": 810
     },
     {
       "epoch": 0.4361933433684563,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 6.957247575724642e-08,
       "learning_rate": 9.937536987168413e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 7.343323707580566,
+      "logits/rejected": 7.149285316467285,
+      "logps/chosen": -38.592342376708984,
+      "logps/rejected": -41.225990295410156,
+      "loss": 0.0,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -38.592342376708984,
+      "rewards/margins": 2.633650064468384,
+      "rewards/rejected": -41.225990295410156,
       "step": 815
     },
     {
       "epoch": 0.4388693761498578,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 9.45959445445422e-06,
       "learning_rate": 9.935058998485896e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 7.62643575668335,
+      "logits/rejected": 7.673081874847412,
+      "logps/chosen": -39.537689208984375,
+      "logps/rejected": -39.9161376953125,
+      "loss": 0.0,
+      "rewards/accuracies": 0.4937500059604645,
+      "rewards/chosen": -39.537689208984375,
+      "rewards/margins": 0.3784486651420593,
+      "rewards/rejected": -39.9161376953125,
       "step": 820
     },
     {
       "epoch": 0.44154540893125943,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.0012381106919758e-05,
       "learning_rate": 9.932533129839333e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.040262222290039,
+      "logits/rejected": 7.694721221923828,
+      "logps/chosen": -38.33998107910156,
+      "logps/rejected": -39.52722930908203,
+      "loss": 0.0,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -38.33998107910156,
+      "rewards/margins": 1.1872460842132568,
+      "rewards/rejected": -39.52722930908203,
       "step": 825
     },
     {
       "epoch": 0.444221441712661,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 0.007977274137504639,
       "learning_rate": 9.929959405734711e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 7.41546106338501,
+      "logits/rejected": 7.280850887298584,
+      "logps/chosen": -36.20171356201172,
+      "logps/rejected": -37.58140563964844,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -36.20171356201172,
+      "rewards/margins": 1.3796923160552979,
+      "rewards/rejected": -37.58140563964844,
       "step": 830
     },
     {
       "epoch": 0.44689747449406253,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.7754891368202017e-05,
       "learning_rate": 9.927337851142314e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 7.8377203941345215,
+      "logits/rejected": 7.608137607574463,
+      "logps/chosen": -38.19658660888672,
+      "logps/rejected": -39.34210205078125,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -38.19658660888672,
+      "rewards/margins": 1.1455156803131104,
+      "rewards/rejected": -39.34210205078125,
       "step": 835
     },
     {
       "epoch": 0.44957350727546413,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.1258031598332168e-08,
       "learning_rate": 9.924668491496474e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 7.384390354156494,
+      "logits/rejected": 6.984945774078369,
+      "logps/chosen": -37.10817337036133,
+      "logps/rejected": -39.376800537109375,
+      "loss": 0.0,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -37.10817337036133,
+      "rewards/margins": 2.268624782562256,
+      "rewards/rejected": -39.376800537109375,
       "step": 840
     },
     {
       "epoch": 0.4522495400568657,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.00140809333623e-05,
       "learning_rate": 9.92195135269533e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 7.49320125579834,
+      "logits/rejected": 7.54547119140625,
+      "logps/chosen": -37.49378204345703,
+      "logps/rejected": -39.536685943603516,
+      "loss": 0.0,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -37.49378204345703,
+      "rewards/margins": 2.0429091453552246,
+      "rewards/rejected": -39.536685943603516,
       "step": 845
     },
     {
       "epoch": 0.4549255728382673,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 7.834505324589983e-07,
       "learning_rate": 9.919186461100574e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 7.840467929840088,
+      "logits/rejected": 7.551217555999756,
+      "logps/chosen": -37.86256790161133,
+      "logps/rejected": -39.80552291870117,
+      "loss": 0.0,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -37.86256790161133,
+      "rewards/margins": 1.9429543018341064,
+      "rewards/rejected": -39.80552291870117,
       "step": 850
     },
     {
       "epoch": 0.45760160561966884,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 0.1767710140313415,
       "learning_rate": 9.9163738435372e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 7.156085014343262,
+      "logits/rejected": 6.969692230224609,
+      "logps/chosen": -35.7013053894043,
+      "logps/rejected": -38.50730514526367,
+      "loss": 0.0007,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -35.7013053894043,
+      "rewards/margins": 2.805997848510742,
+      "rewards/rejected": -38.50730514526367,
       "step": 855
     },
     {
       "epoch": 0.4602776384010704,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.3790709051049524e-05,
       "learning_rate": 9.913513527293234e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 7.813882350921631,
+      "logits/rejected": 7.680117130279541,
+      "logps/chosen": -36.60405731201172,
+      "logps/rejected": -38.678749084472656,
+      "loss": 0.0,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -36.60405731201172,
+      "rewards/margins": 2.074687957763672,
+      "rewards/rejected": -38.678749084472656,
       "step": 860
     },
     {
       "epoch": 0.462953671182472,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.354378354385777e-07,
       "learning_rate": 9.910605540119474e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 7.874215126037598,
+      "logits/rejected": 7.459391117095947,
+      "logps/chosen": -39.80036163330078,
+      "logps/rejected": -40.59810256958008,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -39.80036163330078,
+      "rewards/margins": 0.7977336645126343,
+      "rewards/rejected": -40.59810256958008,
       "step": 865
     },
     {
       "epoch": 0.46562970396387354,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.1259579120069062e-06,
       "learning_rate": 9.907649910229227e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 7.522549629211426,
+      "logits/rejected": 7.291293144226074,
+      "logps/chosen": -36.571571350097656,
+      "logps/rejected": -39.611175537109375,
+      "loss": 0.0,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -36.571571350097656,
+      "rewards/margins": 3.039602756500244,
+      "rewards/rejected": -39.611175537109375,
       "step": 870
     },
     {
       "epoch": 0.46830573674527515,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.177973632904566e-06,
       "learning_rate": 9.90464666629803e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 7.758486747741699,
+      "logits/rejected": 7.561933994293213,
+      "logps/chosen": -37.98600769042969,
+      "logps/rejected": -38.57429122924805,
+      "loss": 0.0,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": -37.98600769042969,
+      "rewards/margins": 0.5882858037948608,
+      "rewards/rejected": -38.57429122924805,
       "step": 875
     },
     {
       "epoch": 0.4709817695266767,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 6.916912006007473e-05,
       "learning_rate": 9.901595837463363e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 7.254601955413818,
+      "logits/rejected": 7.139842987060547,
+      "logps/chosen": -35.60768127441406,
+      "logps/rejected": -38.065185546875,
+      "loss": 0.0,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -35.60768127441406,
+      "rewards/margins": 2.4575068950653076,
+      "rewards/rejected": -38.065185546875,
       "step": 880
     },
     {
       "epoch": 0.47365780230807825,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.853764441528317e-07,
       "learning_rate": 9.898497453324384e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 7.788936614990234,
+      "logits/rejected": 7.788862705230713,
+      "logps/chosen": -38.87586212158203,
+      "logps/rejected": -40.32879638671875,
+      "loss": 0.0,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -38.87586212158203,
+      "rewards/margins": 1.4529345035552979,
+      "rewards/rejected": -40.32879638671875,
       "step": 885
     },
     {
       "epoch": 0.47633383508947985,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 5.812918658075046e-06,
       "learning_rate": 9.895351543941628e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 7.8732171058654785,
+      "logits/rejected": 7.65131139755249,
+      "logps/chosen": -37.66066360473633,
+      "logps/rejected": -39.59817123413086,
+      "loss": 0.0,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -37.66066360473633,
+      "rewards/margins": 1.9375073909759521,
+      "rewards/rejected": -39.59817123413086,
       "step": 890
     },
     {
       "epoch": 0.4790098678708814,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.0758085258701195e-08,
       "learning_rate": 9.892158139836724e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 7.62447452545166,
+      "logits/rejected": 7.486349582672119,
+      "logps/chosen": -38.606773376464844,
+      "logps/rejected": -39.39204788208008,
+      "loss": 0.0,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -38.606773376464844,
+      "rewards/margins": 0.785274863243103,
+      "rewards/rejected": -39.39204788208008,
       "step": 895
     },
     {
       "epoch": 0.481685900652283,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.7952838189141804e-05,
       "learning_rate": 9.88891727199209e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 7.782861232757568,
+      "logits/rejected": 7.703749656677246,
+      "logps/chosen": -38.985172271728516,
+      "logps/rejected": -41.012203216552734,
+      "loss": 0.0,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -38.985172271728516,
+      "rewards/margins": 2.0270309448242188,
+      "rewards/rejected": -41.012203216552734,
       "step": 900
     },
     {
       "epoch": 0.48436193343368455,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 0.0009951232639731184,
       "learning_rate": 9.885628971850641e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 7.884472846984863,
+      "logits/rejected": 7.380224704742432,
+      "logps/chosen": -38.044490814208984,
+      "logps/rejected": -39.506675720214844,
+      "loss": 0.0,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -38.044490814208984,
+      "rewards/margins": 1.4621862173080444,
+      "rewards/rejected": -39.506675720214844,
       "step": 905
     },
     {
       "epoch": 0.48703796621508616,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.2052396655037205e-09,
       "learning_rate": 9.882293271315481e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 7.479225158691406,
+      "logits/rejected": 7.292827606201172,
+      "logps/chosen": -37.35040283203125,
+      "logps/rejected": -38.60327911376953,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -37.35040283203125,
+      "rewards/margins": 1.2528806924819946,
+      "rewards/rejected": -38.60327911376953,
       "step": 910
     },
     {
       "epoch": 0.4897139989964877,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 0.0007963871998727564,
       "learning_rate": 9.878910202749589e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 7.811363220214844,
+      "logits/rejected": 7.522627353668213,
+      "logps/chosen": -36.3570442199707,
+      "logps/rejected": -38.404640197753906,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -36.3570442199707,
+      "rewards/margins": 2.0475997924804688,
+      "rewards/rejected": -38.404640197753906,
       "step": 915
     },
     {
       "epoch": 0.49239003177788926,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 8.615165521933634e-05,
       "learning_rate": 9.875479798975512e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 7.5098395347595215,
+      "logits/rejected": 6.997906684875488,
+      "logps/chosen": -38.46096420288086,
+      "logps/rejected": -40.547576904296875,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -38.46096420288086,
+      "rewards/margins": 2.08661150932312,
+      "rewards/rejected": -40.547576904296875,
       "step": 920
     },
     {
       "epoch": 0.49506606455929086,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 8.375688750634028e-07,
       "learning_rate": 9.87200209327504e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 7.657576084136963,
+      "logits/rejected": 7.23116397857666,
+      "logps/chosen": -38.20526885986328,
+      "logps/rejected": -40.23979949951172,
+      "loss": 0.0,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -38.20526885986328,
+      "rewards/margins": 2.0345311164855957,
+      "rewards/rejected": -40.23979949951172,
       "step": 925
     },
     {
       "epoch": 0.4977420973406924,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.0649943186027018e-09,
       "learning_rate": 9.868477119388894e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 7.509026527404785,
+      "logits/rejected": 7.3842878341674805,
+      "logps/chosen": -37.2547607421875,
+      "logps/rejected": -38.1758918762207,
+      "loss": 0.0,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -37.2547607421875,
+      "rewards/margins": 0.9211313128471375,
+      "rewards/rejected": -38.1758918762207,
       "step": 930
     },
     {
       "epoch": 0.500418130122094,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.323767311271579e-05,
       "learning_rate": 9.864904911516383e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.09333324432373,
+      "logits/rejected": 7.901297092437744,
+      "logps/chosen": -38.77185821533203,
+      "logps/rejected": -40.321510314941406,
+      "loss": 0.0,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -38.77185821533203,
+      "rewards/margins": 1.5496532917022705,
+      "rewards/rejected": -40.321510314941406,
       "step": 935
     },
     {
       "epoch": 0.5030941629034956,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 0.011011078345763153,
       "learning_rate": 9.861285504315084e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 7.495965003967285,
+      "logits/rejected": 7.350213050842285,
+      "logps/chosen": -38.28999710083008,
+      "logps/rejected": -39.809730529785156,
+      "loss": 0.0,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -38.28999710083008,
+      "rewards/margins": 1.5197333097457886,
+      "rewards/rejected": -39.809730529785156,
       "step": 940
     },
     {
       "epoch": 0.5057701956848971,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.5926991049482472e-05,
       "learning_rate": 9.857618932900502e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.081062316894531,
+      "logits/rejected": 7.699568271636963,
+      "logps/chosen": -38.677940368652344,
+      "logps/rejected": -40.96052932739258,
+      "loss": 0.0,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -38.677940368652344,
+      "rewards/margins": 2.2825827598571777,
+      "rewards/rejected": -40.96052932739258,
       "step": 945
     },
     {
       "epoch": 0.5084462284662987,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 0.003033728417356917,
       "learning_rate": 9.853905232845727e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 7.655567169189453,
+      "logits/rejected": 7.4965009689331055,
+      "logps/chosen": -37.157310485839844,
+      "logps/rejected": -39.4751091003418,
+      "loss": 0.0,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -37.157310485839844,
+      "rewards/margins": 2.317800760269165,
+      "rewards/rejected": -39.4751091003418,
       "step": 950
     },
     {
       "epoch": 0.5111222612477003,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.8303040967632359e-06,
       "learning_rate": 9.850144440181095e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 7.543534755706787,
+      "logits/rejected": 7.293342590332031,
+      "logps/chosen": -36.8066291809082,
+      "logps/rejected": -39.609161376953125,
+      "loss": 0.0,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -36.8066291809082,
+      "rewards/margins": 2.802532911300659,
+      "rewards/rejected": -39.609161376953125,
       "step": 955
     },
     {
       "epoch": 0.5137982940291018,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.7089219400119935e-05,
       "learning_rate": 9.846336591393832e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 7.614388942718506,
+      "logits/rejected": 7.3226470947265625,
+      "logps/chosen": -37.35138702392578,
+      "logps/rejected": -39.05803680419922,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -37.35138702392578,
+      "rewards/margins": 1.7066415548324585,
+      "rewards/rejected": -39.05803680419922,
       "step": 960
     },
     {
       "epoch": 0.5164743268105034,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 8.080497424132641e-08,
       "learning_rate": 9.842481723427704e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 7.483859062194824,
+      "logits/rejected": 7.482878684997559,
+      "logps/chosen": -36.816795349121094,
+      "logps/rejected": -38.47077560424805,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -36.816795349121094,
+      "rewards/margins": 1.6539726257324219,
+      "rewards/rejected": -38.47077560424805,
       "step": 965
     },
     {
       "epoch": 0.519150359591905,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 0.00022005626828690273,
       "learning_rate": 9.838579873682658e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 7.405804634094238,
+      "logits/rejected": 7.3831682205200195,
+      "logps/chosen": -36.64482498168945,
+      "logps/rejected": -37.14829635620117,
+      "loss": 0.0,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -36.64482498168945,
+      "rewards/margins": 0.5034673810005188,
+      "rewards/rejected": -37.14829635620117,
       "step": 970
     },
     {
       "epoch": 0.5218263923733065,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.524468606787021e-06,
       "learning_rate": 9.834631080014457e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 7.954876899719238,
+      "logits/rejected": 7.514945030212402,
+      "logps/chosen": -38.008880615234375,
+      "logps/rejected": -38.99338912963867,
+      "loss": 0.0,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -38.008880615234375,
+      "rewards/margins": 0.9845094680786133,
+      "rewards/rejected": -38.99338912963867,
       "step": 975
     },
     {
       "epoch": 0.5245024251547081,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.817397889273523e-05,
       "learning_rate": 9.830635380734312e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 7.911994934082031,
+      "logits/rejected": 7.618377685546875,
+      "logps/chosen": -38.021732330322266,
+      "logps/rejected": -40.397789001464844,
+      "loss": 0.0,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -38.021732330322266,
+      "rewards/margins": 2.3760623931884766,
+      "rewards/rejected": -40.397789001464844,
       "step": 980
     },
     {
       "epoch": 0.5271784579361097,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 0.0003054801353373189,
       "learning_rate": 9.826592814608517e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 7.941254615783691,
+      "logits/rejected": 7.508225440979004,
+      "logps/chosen": -39.09351348876953,
+      "logps/rejected": -40.13032150268555,
+      "loss": 0.0,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -39.09351348876953,
+      "rewards/margins": 1.0368082523345947,
+      "rewards/rejected": -40.13032150268555,
       "step": 985
     },
     {
       "epoch": 0.5298544907175113,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.7692966238449743e-09,
       "learning_rate": 9.822503420858067e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 7.556215763092041,
+      "logits/rejected": 7.690290927886963,
+      "logps/chosen": -38.79026412963867,
+      "logps/rejected": -38.26075744628906,
+      "loss": 0.0,
+      "rewards/accuracies": 0.4937500059604645,
+      "rewards/chosen": -38.79026412963867,
+      "rewards/margins": -0.5295054316520691,
+      "rewards/rejected": -38.26075744628906,
       "step": 990
     },
     {
       "epoch": 0.5325305234989128,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.3164274782238733e-05,
       "learning_rate": 9.818367239158277e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
-      "step": 995
+      "logits/chosen": 7.353630065917969,
+      "logits/rejected": 7.314842224121094,
+      "logps/chosen": -38.19721221923828,
+      "logps/rejected": -38.21575164794922,
+      "loss": 0.0,
+      "rewards/accuracies": 0.48750001192092896,
+      "rewards/chosen": -38.19721221923828,
+      "rewards/margins": 0.018535912036895752,
+      "rewards/rejected": -38.21575164794922,
+      "step": 995
     },
     {
       "epoch": 0.5352065562803144,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 8.117008705392227e-07,
       "learning_rate": 9.8141843096384e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 7.69936990737915,
+      "logits/rejected": 7.194311618804932,
+      "logps/chosen": -37.16504669189453,
+      "logps/rejected": -39.00267791748047,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -37.16504669189453,
+      "rewards/margins": 1.8376274108886719,
+      "rewards/rejected": -39.00267791748047,
       "step": 1000
     },
     {
       "epoch": 0.537882589061716,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 0.009834807349041097,
       "learning_rate": 9.809954672881237e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 7.660292625427246,
+      "logits/rejected": 7.279736518859863,
+      "logps/chosen": -38.558616638183594,
+      "logps/rejected": -39.17066192626953,
+      "loss": 0.0,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -38.558616638183594,
+      "rewards/margins": 0.6120432019233704,
+      "rewards/rejected": -39.17066192626953,
       "step": 1005
     },
     {
       "epoch": 0.5405586218431175,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 0.00012281036588404807,
       "learning_rate": 9.80567836992274e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.064266204833984,
+      "logits/rejected": 7.85525369644165,
+      "logps/chosen": -37.27289581298828,
+      "logps/rejected": -39.949440002441406,
+      "loss": 0.0,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -37.27289581298828,
+      "rewards/margins": 2.676543712615967,
+      "rewards/rejected": -39.949440002441406,
       "step": 1010
     },
     {
       "epoch": 0.5432346546245191,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.660593368360222e-06,
       "learning_rate": 9.801355442251625e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.168622970581055,
+      "logits/rejected": 7.666064262390137,
+      "logps/chosen": -37.37153625488281,
+      "logps/rejected": -38.426536560058594,
+      "loss": 0.0,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -37.37153625488281,
+      "rewards/margins": 1.0550004243850708,
+      "rewards/rejected": -38.426536560058594,
       "step": 1015
     },
     {
       "epoch": 0.5459106874059207,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.49350215950251e-06,
       "learning_rate": 9.796985931808949e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 7.922774314880371,
+      "logits/rejected": 7.670487403869629,
+      "logps/chosen": -38.050071716308594,
+      "logps/rejected": -40.38166809082031,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -38.050071716308594,
+      "rewards/margins": 2.331602096557617,
+      "rewards/rejected": -40.38166809082031,
       "step": 1020
     },
     {
       "epoch": 0.5485867201873222,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 0.00012182959582150638,
       "learning_rate": 9.792569880987724e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 7.931715488433838,
+      "logits/rejected": 7.495892524719238,
+      "logps/chosen": -38.64578628540039,
+      "logps/rejected": -39.6500244140625,
+      "loss": 0.0,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -38.64578628540039,
+      "rewards/margins": 1.004235029220581,
+      "rewards/rejected": -39.6500244140625,
       "step": 1025
     },
     {
       "epoch": 0.5512627529687238,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 0.361254533250141,
       "learning_rate": 9.788107332632493e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 7.760477542877197,
+      "logits/rejected": 8.117753982543945,
+      "logps/chosen": -38.537662506103516,
+      "logps/rejected": -40.69624710083008,
+      "loss": 0.0021,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -38.537662506103516,
+      "rewards/margins": 2.1585824489593506,
+      "rewards/rejected": -40.69624710083008,
       "step": 1030
     },
     {
       "epoch": 0.5539387857501255,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 0.00036584275977218176,
       "learning_rate": 9.783598330038924e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.067626953125,
+      "logits/rejected": 7.869374752044678,
+      "logps/chosen": -37.5782356262207,
+      "logps/rejected": -39.36564254760742,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -37.5782356262207,
+      "rewards/margins": 1.7874081134796143,
+      "rewards/rejected": -39.36564254760742,
       "step": 1035
     },
     {
       "epoch": 0.5566148185315271,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.3050769761167753e-08,
       "learning_rate": 9.779042916953376e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 7.5154266357421875,
+      "logits/rejected": 6.9996843338012695,
+      "logps/chosen": -37.27549362182617,
+      "logps/rejected": -38.62503433227539,
+      "loss": 0.0001,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -37.27549362182617,
+      "rewards/margins": 1.3495433330535889,
+      "rewards/rejected": -38.62503433227539,
       "step": 1040
     },
     {
       "epoch": 0.5592908513129285,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.2266412298239093e-06,
       "learning_rate": 9.774441137572487e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 7.77196741104126,
+      "logits/rejected": 7.535773277282715,
+      "logps/chosen": -38.214019775390625,
+      "logps/rejected": -40.248146057128906,
+      "loss": 0.0,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -38.214019775390625,
+      "rewards/margins": 2.0341203212738037,
+      "rewards/rejected": -40.248146057128906,
       "step": 1045
     },
     {
       "epoch": 0.5619668840943302,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 0.0004249368159004441,
       "learning_rate": 9.76979303654274e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 7.581748962402344,
+      "logits/rejected": 7.272462368011475,
+      "logps/chosen": -37.40267562866211,
+      "logps/rejected": -38.54042053222656,
+      "loss": 0.0,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -37.40267562866211,
+      "rewards/margins": 1.1377480030059814,
+      "rewards/rejected": -38.54042053222656,
       "step": 1050
     },
     {
       "epoch": 0.5646429168757318,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.5354593928172725e-09,
       "learning_rate": 9.765098658960035e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 7.954503536224365,
+      "logits/rejected": 8.149930000305176,
+      "logps/chosen": -38.73603057861328,
+      "logps/rejected": -40.9662971496582,
+      "loss": 0.0,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -38.73603057861328,
+      "rewards/margins": 2.23026704788208,
+      "rewards/rejected": -40.9662971496582,
       "step": 1055
     },
     {
       "epoch": 0.5673189496571333,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 0.16407495049696466,
       "learning_rate": 9.76035805036924e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 7.949182987213135,
+      "logits/rejected": 7.690735816955566,
+      "logps/chosen": -38.31816864013672,
+      "logps/rejected": -40.112491607666016,
+      "loss": 0.0,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -38.31816864013672,
+      "rewards/margins": 1.7943261861801147,
+      "rewards/rejected": -40.112491607666016,
       "step": 1060
     },
     {
       "epoch": 0.5699949824385349,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 0.00029397863166676997,
       "learning_rate": 9.755571256763764e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.006070137023926,
+      "logits/rejected": 7.897732734680176,
+      "logps/chosen": -37.7571907043457,
+      "logps/rejected": -40.086395263671875,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -37.7571907043457,
+      "rewards/margins": 2.3292055130004883,
+      "rewards/rejected": -40.086395263671875,
       "step": 1065
     },
     {
       "epoch": 0.5726710152199365,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.145959972074204e-07,
       "learning_rate": 9.750738324585097e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 7.74841833114624,
+      "logits/rejected": 7.3475165367126465,
+      "logps/chosen": -37.12546920776367,
+      "logps/rejected": -39.638248443603516,
+      "loss": 0.0,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -37.12546920776367,
+      "rewards/margins": 2.5127782821655273,
+      "rewards/rejected": -39.638248443603516,
       "step": 1070
     },
     {
       "epoch": 0.5753470480013381,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.423495572407899e-06,
       "learning_rate": 9.74585930072237e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.111615180969238,
+      "logits/rejected": 7.866713047027588,
+      "logps/chosen": -39.933719635009766,
+      "logps/rejected": -42.261451721191406,
+      "loss": 0.0,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -39.933719635009766,
+      "rewards/margins": 2.3277344703674316,
+      "rewards/rejected": -42.261451721191406,
       "step": 1075
     },
     {
       "epoch": 0.5780230807827396,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 0.009672790038437799,
       "learning_rate": 9.740934232511892e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 7.873568058013916,
+      "logits/rejected": 7.647279262542725,
+      "logps/chosen": -38.52336883544922,
+      "logps/rejected": -39.57231521606445,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -38.52336883544922,
+      "rewards/margins": 1.048934817314148,
+      "rewards/rejected": -39.57231521606445,
       "step": 1080
     },
     {
       "epoch": 0.5806991135641412,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.2426491865256395e-07,
       "learning_rate": 9.735963167736698e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.265195846557617,
+      "logits/rejected": 7.997408866882324,
+      "logps/chosen": -39.147308349609375,
+      "logps/rejected": -40.83783721923828,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -39.147308349609375,
+      "rewards/margins": 1.6905357837677002,
+      "rewards/rejected": -40.83783721923828,
       "step": 1085
     },
     {
       "epoch": 0.5833751463455428,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.984132368887616e-07,
       "learning_rate": 9.730946154626078e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 7.9652204513549805,
+      "logits/rejected": 7.508782386779785,
+      "logps/chosen": -38.911537170410156,
+      "logps/rejected": -39.65976333618164,
+      "loss": 0.0,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": -38.911537170410156,
+      "rewards/margins": 0.7482300400733948,
+      "rewards/rejected": -39.65976333618164,
       "step": 1090
     },
     {
       "epoch": 0.5860511791269443,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.009116813281513e-06,
       "learning_rate": 9.725883241855117e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.442087173461914,
+      "logits/rejected": 8.048315048217773,
+      "logps/chosen": -40.02463150024414,
+      "logps/rejected": -41.45846939086914,
+      "loss": 0.0,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -40.02463150024414,
+      "rewards/margins": 1.4338366985321045,
+      "rewards/rejected": -41.45846939086914,
       "step": 1095
     },
     {
       "epoch": 0.5887272119083459,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 0.6139429115609526,
       "learning_rate": 9.720774478544218e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 7.953616142272949,
+      "logits/rejected": 7.603008270263672,
+      "logps/chosen": -39.673118591308594,
+      "logps/rejected": -40.210662841796875,
+      "loss": 0.0002,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": -39.673118591308594,
+      "rewards/margins": 0.5375412106513977,
+      "rewards/rejected": -40.210662841796875,
       "step": 1100
     },
     {
       "epoch": 0.5914032446897475,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 5.826684640971938e-06,
       "learning_rate": 9.715619914258624e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.497035026550293,
+      "logits/rejected": 8.352646827697754,
+      "logps/chosen": -39.933937072753906,
+      "logps/rejected": -40.52880859375,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -39.933937072753906,
+      "rewards/margins": 0.5948749780654907,
+      "rewards/rejected": -40.52880859375,
       "step": 1105
     },
     {
       "epoch": 0.594079277471149,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 7.44512141444027e-06,
       "learning_rate": 9.710419599007937e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 7.805276393890381,
+      "logits/rejected": 7.424256801605225,
+      "logps/chosen": -38.6500244140625,
+      "logps/rejected": -39.21177673339844,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -38.6500244140625,
+      "rewards/margins": 0.5617539882659912,
+      "rewards/rejected": -39.21177673339844,
       "step": 1110
     },
     {
       "epoch": 0.5967553102525506,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 7.710652805952235e-06,
       "learning_rate": 9.705173583245643e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.37909984588623,
+      "logits/rejected": 7.973823547363281,
+      "logps/chosen": -40.06968307495117,
+      "logps/rejected": -42.461734771728516,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -40.06968307495117,
+      "rewards/margins": 2.392047166824341,
+      "rewards/rejected": -42.461734771728516,
       "step": 1115
     },
     {
       "epoch": 0.5994313430339522,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.4132177908186556e-07,
       "learning_rate": 9.699881917868609e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.056367874145508,
+      "logits/rejected": 7.825897216796875,
+      "logps/chosen": -37.437705993652344,
+      "logps/rejected": -39.452354431152344,
+      "loss": 0.0,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -37.437705993652344,
+      "rewards/margins": 2.0146484375,
+      "rewards/rejected": -39.452354431152344,
       "step": 1120
     },
     {
       "epoch": 0.6021073758153538,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 0.00020303080176862164,
       "learning_rate": 9.694544654216594e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 7.815061092376709,
+      "logits/rejected": 7.363912105560303,
+      "logps/chosen": -39.43842697143555,
+      "logps/rejected": -40.97020721435547,
+      "loss": 0.0,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -39.43842697143555,
+      "rewards/margins": 1.5317813158035278,
+      "rewards/rejected": -40.97020721435547,
       "step": 1125
     },
     {
       "epoch": 0.6047834085967553,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 0.0003991712640830549,
       "learning_rate": 9.689161844071755e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 7.602792263031006,
+      "logits/rejected": 7.502458095550537,
+      "logps/chosen": -38.625186920166016,
+      "logps/rejected": -40.96052932739258,
+      "loss": 0.0,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -38.625186920166016,
+      "rewards/margins": 2.3353395462036133,
+      "rewards/rejected": -40.96052932739258,
       "step": 1130
     },
     {
       "epoch": 0.6074594413781569,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.37903272490884e-05,
       "learning_rate": 9.683733539658138e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 7.982511043548584,
+      "logits/rejected": 7.626869201660156,
+      "logps/chosen": -37.98310089111328,
+      "logps/rejected": -40.76325225830078,
+      "loss": 0.0,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -37.98310089111328,
+      "rewards/margins": 2.7801480293273926,
+      "rewards/rejected": -40.76325225830078,
       "step": 1135
     },
     {
       "epoch": 0.6101354741595585,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.2942584596614336e-05,
       "learning_rate": 9.678259793641178e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.08725643157959,
+      "logits/rejected": 8.020898818969727,
+      "logps/chosen": -39.81681442260742,
+      "logps/rejected": -40.624794006347656,
+      "loss": 0.0,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -39.81681442260742,
+      "rewards/margins": 0.8079781532287598,
+      "rewards/rejected": -40.624794006347656,
       "step": 1140
     },
     {
       "epoch": 0.61281150694096,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.900377238233814e-09,
       "learning_rate": 9.672740659127183e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.23723030090332,
+      "logits/rejected": 7.829659938812256,
+      "logps/chosen": -39.133609771728516,
+      "logps/rejected": -39.844322204589844,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -39.133609771728516,
+      "rewards/margins": 0.7107132077217102,
+      "rewards/rejected": -39.844322204589844,
       "step": 1145
     },
     {
       "epoch": 0.6154875397223616,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.9878388306590226e-10,
       "learning_rate": 9.667176189662818e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.174409866333008,
+      "logits/rejected": 7.807578086853027,
+      "logps/chosen": -39.63080596923828,
+      "logps/rejected": -40.829383850097656,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -39.63080596923828,
+      "rewards/margins": 1.1985732316970825,
+      "rewards/rejected": -40.829383850097656,
       "step": 1150
     },
     {
       "epoch": 0.6181635725037632,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.50688062942319e-05,
       "learning_rate": 9.661566439234592e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 7.922749996185303,
+      "logits/rejected": 7.881862640380859,
+      "logps/chosen": -38.62043762207031,
+      "logps/rejected": -40.07584762573242,
+      "loss": 0.0,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -38.62043762207031,
+      "rewards/margins": 1.4554095268249512,
+      "rewards/rejected": -40.07584762573242,
       "step": 1155
     },
     {
       "epoch": 0.6208396052851648,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.5467628971347876e-07,
       "learning_rate": 9.655911462268327e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 7.79284143447876,
+      "logits/rejected": 7.658049583435059,
+      "logps/chosen": -38.09325408935547,
+      "logps/rejected": -40.65690231323242,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -38.09325408935547,
+      "rewards/margins": 2.563648223876953,
+      "rewards/rejected": -40.65690231323242,
       "step": 1160
     },
     {
       "epoch": 0.6235156380665663,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.793625003254257e-07,
       "learning_rate": 9.650211313628636e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 7.992042541503906,
+      "logits/rejected": 7.954789161682129,
+      "logps/chosen": -38.96379470825195,
+      "logps/rejected": -40.213645935058594,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -38.96379470825195,
+      "rewards/margins": 1.2498466968536377,
+      "rewards/rejected": -40.213645935058594,
       "step": 1165
     },
     {
       "epoch": 0.6261916708479679,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 0.00020843590155329592,
       "learning_rate": 9.644466048618386e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 7.867042541503906,
+      "logits/rejected": 7.492088317871094,
+      "logps/chosen": -38.6095085144043,
+      "logps/rejected": -39.39339065551758,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -38.6095085144043,
+      "rewards/margins": 0.7838778495788574,
+      "rewards/rejected": -39.39339065551758,
       "step": 1170
     },
     {
       "epoch": 0.6288677036293695,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.1773373791217627e-07,
       "learning_rate": 9.63867572297816e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.233621597290039,
+      "logits/rejected": 7.841109275817871,
+      "logps/chosen": -39.95270919799805,
+      "logps/rejected": -41.25849533081055,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -39.95270919799805,
+      "rewards/margins": 1.3057838678359985,
+      "rewards/rejected": -41.25849533081055,
       "step": 1175
     },
     {
       "epoch": 0.631543736410771,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.6626753800657045e-09,
       "learning_rate": 9.632840392885727e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.159266471862793,
+      "logits/rejected": 7.716169834136963,
+      "logps/chosen": -38.42955780029297,
+      "logps/rejected": -40.182743072509766,
+      "loss": 0.0,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -38.42955780029297,
+      "rewards/margins": 1.7531821727752686,
+      "rewards/rejected": -40.182743072509766,
       "step": 1180
     },
     {
       "epoch": 0.6342197691921726,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 0.0018605726208984858,
       "learning_rate": 9.626960114955483e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.13940143585205,
+      "logits/rejected": 7.720973014831543,
+      "logps/chosen": -37.29474639892578,
+      "logps/rejected": -40.07600021362305,
+      "loss": 0.0001,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -37.29474639892578,
+      "rewards/margins": 2.7812552452087402,
+      "rewards/rejected": -40.07600021362305,
       "step": 1185
     },
     {
       "epoch": 0.6368958019735742,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.918177352610572e-09,
       "learning_rate": 9.621034946237909e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.093598365783691,
+      "logits/rejected": 7.705621242523193,
+      "logps/chosen": -39.510284423828125,
+      "logps/rejected": -41.15454864501953,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -39.510284423828125,
+      "rewards/margins": 1.6442687511444092,
+      "rewards/rejected": -41.15454864501953,
       "step": 1190
     },
     {
       "epoch": 0.6395718347549757,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 0.000251170090949831,
       "learning_rate": 9.615064944219021e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.17841911315918,
+      "logits/rejected": 7.797991752624512,
+      "logps/chosen": -40.805179595947266,
+      "logps/rejected": -42.521812438964844,
+      "loss": 0.0,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -40.805179595947266,
+      "rewards/margins": 1.7166383266448975,
+      "rewards/rejected": -42.521812438964844,
       "step": 1195
     },
     {
       "epoch": 0.6422478675363773,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.0366767541922134e-07,
       "learning_rate": 9.609050166819803e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.052102088928223,
+      "logits/rejected": 7.984274387359619,
+      "logps/chosen": -39.85681915283203,
+      "logps/rejected": -40.10163879394531,
+      "loss": 0.001,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -39.85681915283203,
+      "rewards/margins": 0.24481861293315887,
+      "rewards/rejected": -40.10163879394531,
       "step": 1200
     },
     {
       "epoch": 0.6422478675363773,
-      "eval_logits/chosen": NaN,
-      "eval_logits/rejected": NaN,
-      "eval_logps/chosen": NaN,
-      "eval_logps/rejected": NaN,
-      "eval_loss": NaN,
-      "eval_rewards/accuracies": 0.0,
-      "eval_rewards/chosen": NaN,
-      "eval_rewards/margins": NaN,
-      "eval_rewards/rejected": NaN,
-      "eval_runtime": 35.3915,
-      "eval_samples_per_second": 38.004,
-      "eval_semantic_entropy": NaN,
-      "eval_steps_per_second": 9.522,
+      "eval_logits/chosen": 6.700042247772217,
+      "eval_logits/rejected": 6.514923572540283,
+      "eval_logps/chosen": -38.794212341308594,
+      "eval_logps/rejected": -40.12665939331055,
+      "eval_loss": 1.5090794249772443e-06,
+      "eval_rewards/accuracies": 0.5667656064033508,
+      "eval_rewards/chosen": -38.794212341308594,
+      "eval_rewards/margins": 1.3324474096298218,
+      "eval_rewards/rejected": -40.12665939331055,
+      "eval_runtime": 40.2985,
+      "eval_samples_per_second": 33.376,
+      "eval_steps_per_second": 8.363,
       "step": 1200
     },
     {
       "epoch": 0.6449239003177789,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.571025649803181e-07,
       "learning_rate": 9.602990672395653e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.21595287322998,
+      "logits/rejected": 7.726536750793457,
+      "logps/chosen": -39.610347747802734,
+      "logps/rejected": -41.17707443237305,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -39.610347747802734,
+      "rewards/margins": 1.566732406616211,
+      "rewards/rejected": -41.17707443237305,
       "step": 1205
     },
     {
       "epoch": 0.6475999330991805,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.3910108117174059e-09,
       "learning_rate": 9.59688651973581e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.231789588928223,
+      "logits/rejected": 7.744787693023682,
+      "logps/chosen": -39.04650115966797,
+      "logps/rejected": -39.942771911621094,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -39.04650115966797,
+      "rewards/margins": 0.8962669372558594,
+      "rewards/rejected": -39.942771911621094,
       "step": 1210
     },
     {
       "epoch": 0.650275965880582,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 0.0007458526424624522,
       "learning_rate": 9.590737768062792e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.047931671142578,
+      "logits/rejected": 7.630962371826172,
+      "logps/chosen": -38.702293395996094,
+      "logps/rejected": -40.36885452270508,
+      "loss": 0.0,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -38.702293395996094,
+      "rewards/margins": 1.6665655374526978,
+      "rewards/rejected": -40.36885452270508,
       "step": 1215
     },
     {
       "epoch": 0.6529519986619836,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 0.1610317626246477,
       "learning_rate": 9.584544477031816e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 7.843100070953369,
+      "logits/rejected": 7.694161891937256,
+      "logps/chosen": -39.58910369873047,
+      "logps/rejected": -42.216697692871094,
+      "loss": 0.0,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -39.58910369873047,
+      "rewards/margins": 2.6275923252105713,
+      "rewards/rejected": -42.216697692871094,
       "step": 1220
     },
     {
       "epoch": 0.6556280314433852,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.554071556976578e-05,
       "learning_rate": 9.578306706730215e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.172025680541992,
+      "logits/rejected": 7.747583866119385,
+      "logps/chosen": -39.46088409423828,
+      "logps/rejected": -41.235870361328125,
+      "loss": 0.0,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -39.46088409423828,
+      "rewards/margins": 1.774987816810608,
+      "rewards/rejected": -41.235870361328125,
       "step": 1225
     },
     {
       "epoch": 0.6583040642247867,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.6651937292717813e-07,
       "learning_rate": 9.572024517676865e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.174722671508789,
+      "logits/rejected": 7.941262722015381,
+      "logps/chosen": -39.199161529541016,
+      "logps/rejected": -40.506752014160156,
+      "loss": 0.0,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -39.199161529541016,
+      "rewards/margins": 1.307594895362854,
+      "rewards/rejected": -40.506752014160156,
       "step": 1230
     },
     {
       "epoch": 0.6609800970061883,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.0343609113431195e-06,
       "learning_rate": 9.565697970821593e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.244317054748535,
+      "logits/rejected": 7.794415473937988,
+      "logps/chosen": -38.94886016845703,
+      "logps/rejected": -40.545372009277344,
+      "loss": 0.0,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -38.94886016845703,
+      "rewards/margins": 1.5965158939361572,
+      "rewards/rejected": -40.545372009277344,
       "step": 1235
     },
     {
       "epoch": 0.6636561297875899,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.6902023490552281e-06,
       "learning_rate": 9.559327127544585e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.549825668334961,
+      "logits/rejected": 8.43071174621582,
+      "logps/chosen": -41.14146041870117,
+      "logps/rejected": -42.68232345581055,
+      "loss": 0.0,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -41.14146041870117,
+      "rewards/margins": 1.5408649444580078,
+      "rewards/rejected": -42.68232345581055,
       "step": 1240
     },
     {
       "epoch": 0.6663321625689914,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 5.155827998256065e-10,
       "learning_rate": 9.552912049655789e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.480178833007812,
+      "logits/rejected": 8.107583999633789,
+      "logps/chosen": -40.715816497802734,
+      "logps/rejected": -42.54690170288086,
+      "loss": 0.0,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -40.715816497802734,
+      "rewards/margins": 1.8310868740081787,
+      "rewards/rejected": -42.54690170288086,
       "step": 1245
     },
     {
       "epoch": 0.669008195350393,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 0.003137647159854406,
       "learning_rate": 9.546452799394315e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.568636894226074,
+      "logits/rejected": 8.026914596557617,
+      "logps/chosen": -40.63987350463867,
+      "logps/rejected": -42.18573760986328,
+      "loss": 0.0,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -40.63987350463867,
+      "rewards/margins": 1.5458616018295288,
+      "rewards/rejected": -42.18573760986328,
       "step": 1250
     },
     {
       "epoch": 0.6716842281317946,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.967368470942391e-09,
       "learning_rate": 9.539949439427846e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.262842178344727,
+      "logits/rejected": 8.109676361083984,
+      "logps/chosen": -40.08157730102539,
+      "logps/rejected": -41.22216796875,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -40.08157730102539,
+      "rewards/margins": 1.140596628189087,
+      "rewards/rejected": -41.22216796875,
       "step": 1255
     },
     {
       "epoch": 0.6743602609131962,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.654212667263991e-09,
       "learning_rate": 9.533402032852002e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.280901908874512,
+      "logits/rejected": 7.905728340148926,
+      "logps/chosen": -39.38500213623047,
+      "logps/rejected": -41.9587287902832,
+      "loss": 0.0,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -39.38500213623047,
+      "rewards/margins": 2.5737221240997314,
+      "rewards/rejected": -41.9587287902832,
       "step": 1260
     },
     {
       "epoch": 0.6770362936945977,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.0892819823674152e-09,
       "learning_rate": 9.526810643189754e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.520188331604004,
+      "logits/rejected": 7.725737571716309,
+      "logps/chosen": -39.5770378112793,
+      "logps/rejected": -41.63107681274414,
+      "loss": 0.0,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -39.5770378112793,
+      "rewards/margins": 2.0540409088134766,
+      "rewards/rejected": -41.63107681274414,
       "step": 1265
     },
     {
       "epoch": 0.6797123264759993,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.5056855392210347e-06,
       "learning_rate": 9.52017533439079e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.530022621154785,
+      "logits/rejected": 8.410375595092773,
+      "logps/chosen": -40.831825256347656,
+      "logps/rejected": -41.7933235168457,
+      "loss": 0.0,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -40.831825256347656,
+      "rewards/margins": 0.9614920616149902,
+      "rewards/rejected": -41.7933235168457,
       "step": 1270
     },
     {
       "epoch": 0.6823883592574009,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.6898715210826791e-06,
       "learning_rate": 9.513496170830909e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.285804748535156,
+      "logits/rejected": 8.108721733093262,
+      "logps/chosen": -39.77947235107422,
+      "logps/rejected": -40.9272346496582,
+      "loss": 0.0,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -39.77947235107422,
+      "rewards/margins": 1.1477549076080322,
+      "rewards/rejected": -40.9272346496582,
       "step": 1275
     },
     {
       "epoch": 0.6850643920388024,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 9.957210999909553e-07,
       "learning_rate": 9.506773217311382e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 7.924864768981934,
+      "logits/rejected": 7.505575656890869,
+      "logps/chosen": -39.848594665527344,
+      "logps/rejected": -40.96566390991211,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -39.848594665527344,
+      "rewards/margins": 1.1170703172683716,
+      "rewards/rejected": -40.96566390991211,
       "step": 1280
     },
     {
       "epoch": 0.687740424820204,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 9.583693548457569e-08,
       "learning_rate": 9.500006539058334e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.455413818359375,
+      "logits/rejected": 7.947080135345459,
+      "logps/chosen": -40.40897750854492,
+      "logps/rejected": -41.91022491455078,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -40.40897750854492,
+      "rewards/margins": 1.5012447834014893,
+      "rewards/rejected": -41.91022491455078,
       "step": 1285
     },
     {
       "epoch": 0.6904164576016056,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.677881002744384e-06,
       "learning_rate": 9.493196201722109e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.407316207885742,
+      "logits/rejected": 8.076360702514648,
+      "logps/chosen": -39.57369613647461,
+      "logps/rejected": -41.303043365478516,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -39.57369613647461,
+      "rewards/margins": 1.7293485403060913,
+      "rewards/rejected": -41.303043365478516,
       "step": 1290
     },
     {
       "epoch": 0.6930924903830072,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.2516851606508294e-10,
       "learning_rate": 9.486342271376628e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.39858627319336,
+      "logits/rejected": 8.682069778442383,
+      "logps/chosen": -39.82408905029297,
+      "logps/rejected": -41.20485305786133,
+      "loss": 0.0,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -39.82408905029297,
+      "rewards/margins": 1.3807626962661743,
+      "rewards/rejected": -41.20485305786133,
       "step": 1295
     },
     {
       "epoch": 0.6957685231644087,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.524749759736089e-07,
       "learning_rate": 9.479444814518755e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.44395923614502,
+      "logits/rejected": 7.684889316558838,
+      "logps/chosen": -39.279842376708984,
+      "logps/rejected": -41.58313751220703,
+      "loss": 0.0,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -39.279842376708984,
+      "rewards/margins": 2.303298234939575,
+      "rewards/rejected": -41.58313751220703,
       "step": 1300
     },
     {
       "epoch": 0.6984445559458103,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.4551212633776652e-06,
       "learning_rate": 9.472503898067645e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.053625106811523,
+      "logits/rejected": 8.036966323852539,
+      "logps/chosen": -40.34284210205078,
+      "logps/rejected": -40.94997024536133,
+      "loss": 0.0,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": -40.34284210205078,
+      "rewards/margins": 0.6071248650550842,
+      "rewards/rejected": -40.94997024536133,
       "step": 1305
     },
     {
       "epoch": 0.701120588727212,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.1205545159748336e-09,
       "learning_rate": 9.465519589364099e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 7.924736976623535,
+      "logits/rejected": 7.821497440338135,
+      "logps/chosen": -38.628082275390625,
+      "logps/rejected": -40.260337829589844,
+      "loss": 0.0,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -38.628082275390625,
+      "rewards/margins": 1.63225519657135,
+      "rewards/rejected": -40.260337829589844,
       "step": 1310
     },
     {
       "epoch": 0.7037966215086134,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.0614199759997344e-06,
       "learning_rate": 9.458491956169914e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.606477737426758,
+      "logits/rejected": 8.16909408569336,
+      "logps/chosen": -38.722190856933594,
+      "logps/rejected": -41.011512756347656,
+      "loss": 0.0,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -38.722190856933594,
+      "rewards/margins": 2.289320468902588,
+      "rewards/rejected": -41.011512756347656,
       "step": 1315
     },
     {
       "epoch": 0.706472654290015,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.3301969072183396e-07,
       "learning_rate": 9.451421066667215e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.571816444396973,
+      "logits/rejected": 8.233956336975098,
+      "logps/chosen": -39.03949737548828,
+      "logps/rejected": -41.33659744262695,
+      "loss": 0.0,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -39.03949737548828,
+      "rewards/margins": 2.2970995903015137,
+      "rewards/rejected": -41.33659744262695,
       "step": 1320
     },
     {
       "epoch": 0.7091486870714167,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 0.00019004317750762452,
       "learning_rate": 9.444306989457805e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.159810066223145,
+      "logits/rejected": 8.031582832336426,
+      "logps/chosen": -39.55509567260742,
+      "logps/rejected": -41.02944564819336,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -39.55509567260742,
+      "rewards/margins": 1.4743516445159912,
+      "rewards/rejected": -41.02944564819336,
       "step": 1325
     },
     {
       "epoch": 0.7118247198528181,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.594547145170921e-09,
       "learning_rate": 9.437149793562489e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.045509338378906,
+      "logits/rejected": 7.7881035804748535,
+      "logps/chosen": -38.70356369018555,
+      "logps/rejected": -39.1339225769043,
+      "loss": 0.0,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -38.70356369018555,
+      "rewards/margins": 0.4303584098815918,
+      "rewards/rejected": -39.1339225769043,
       "step": 1330
     },
     {
       "epoch": 0.7145007526342197,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.152601177913876e-05,
       "learning_rate": 9.429949548420417e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.452252388000488,
+      "logits/rejected": 8.18472671508789,
+      "logps/chosen": -39.76271057128906,
+      "logps/rejected": -40.83562469482422,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -39.76271057128906,
+      "rewards/margins": 1.0729191303253174,
+      "rewards/rejected": -40.83562469482422,
       "step": 1335
     },
     {
       "epoch": 0.7171767854156214,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.4505040532485936,
       "learning_rate": 9.422706323888396e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.398438453674316,
+      "logits/rejected": 8.44367790222168,
+      "logps/chosen": -38.87419891357422,
+      "logps/rejected": -40.024925231933594,
+      "loss": 0.0003,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -38.87419891357422,
+      "rewards/margins": 1.150724172592163,
+      "rewards/rejected": -40.024925231933594,
       "step": 1340
     },
     {
       "epoch": 0.719852818197023,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 5.692852308569447e-05,
       "learning_rate": 9.415420190240225e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.081521987915039,
+      "logits/rejected": 7.656586647033691,
+      "logps/chosen": -36.452674865722656,
+      "logps/rejected": -40.089290618896484,
+      "loss": 0.0,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -36.452674865722656,
+      "rewards/margins": 3.6366171836853027,
+      "rewards/rejected": -40.089290618896484,
       "step": 1345
     },
     {
       "epoch": 0.7225288509784245,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 0.6167234651032344,
       "learning_rate": 9.408091218166002e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 6.0876383781433105,
+      "logits/rejected": 6.024787902832031,
+      "logps/chosen": -30.829113006591797,
+      "logps/rejected": -31.317214965820312,
+      "loss": 0.0001,
+      "rewards/accuracies": 0.5062500238418579,
+      "rewards/chosen": -30.829113006591797,
+      "rewards/margins": 0.4881015717983246,
+      "rewards/rejected": -31.317214965820312,
       "step": 1350
     },
     {
       "epoch": 0.7252048837598261,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 0.0021453077032664494,
       "learning_rate": 9.400719478771449e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 6.615061283111572,
+      "logits/rejected": 6.28089714050293,
+      "logps/chosen": -33.01981735229492,
+      "logps/rejected": -36.19890594482422,
+      "loss": 0.0,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -33.01981735229492,
+      "rewards/margins": 3.1790847778320312,
+      "rewards/rejected": -36.19890594482422,
       "step": 1355
     },
     {
       "epoch": 0.7278809165412277,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 7.938306104734793e-05,
       "learning_rate": 9.393305043577209e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 7.4625244140625,
+      "logits/rejected": 7.271597385406494,
+      "logps/chosen": -36.03832244873047,
+      "logps/rejected": -36.957035064697266,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -36.03832244873047,
+      "rewards/margins": 0.9187110662460327,
+      "rewards/rejected": -36.957035064697266,
       "step": 1360
     },
     {
       "epoch": 0.7305569493226292,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 0.00015826475228190558,
       "learning_rate": 9.38584798451817e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 7.673105716705322,
+      "logits/rejected": 7.417733192443848,
+      "logps/chosen": -35.800697326660156,
+      "logps/rejected": -38.100189208984375,
+      "loss": 0.0,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -35.800697326660156,
+      "rewards/margins": 2.299489736557007,
+      "rewards/rejected": -38.100189208984375,
       "step": 1365
     },
     {
       "epoch": 0.7332329821040308,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 8.869733324152551e-07,
       "learning_rate": 9.37834837394275e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 7.683876037597656,
+      "logits/rejected": 7.434299468994141,
+      "logps/chosen": -37.038326263427734,
+      "logps/rejected": -39.22920608520508,
+      "loss": 0.0,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -37.038326263427734,
+      "rewards/margins": 2.190875768661499,
+      "rewards/rejected": -39.22920608520508,
       "step": 1370
     },
     {
       "epoch": 0.7359090148854324,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.388452303590589e-05,
       "learning_rate": 9.370806284612203e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 7.973485469818115,
+      "logits/rejected": 7.760767936706543,
+      "logps/chosen": -36.79266357421875,
+      "logps/rejected": -39.08042526245117,
+      "loss": 0.0,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -36.79266357421875,
+      "rewards/margins": 2.2877659797668457,
+      "rewards/rejected": -39.08042526245117,
       "step": 1375
     },
     {
       "epoch": 0.738585047666834,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.8074447480362218e-06,
       "learning_rate": 9.363221789699912e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.076640129089355,
+      "logits/rejected": 7.6811933517456055,
+      "logps/chosen": -36.773460388183594,
+      "logps/rejected": -38.93933868408203,
+      "loss": 0.0,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -36.773460388183594,
+      "rewards/margins": 2.1658780574798584,
+      "rewards/rejected": -38.93933868408203,
       "step": 1380
     },
     {
       "epoch": 0.7412610804482355,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.1053272986013867e-05,
       "learning_rate": 9.355594962790682e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 7.998993873596191,
+      "logits/rejected": 7.6072235107421875,
+      "logps/chosen": -37.936492919921875,
+      "logps/rejected": -38.762691497802734,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5062500238418579,
+      "rewards/chosen": -37.936492919921875,
+      "rewards/margins": 0.8261971473693848,
+      "rewards/rejected": -38.762691497802734,
       "step": 1385
     },
     {
       "epoch": 0.7439371132296371,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.095058226728124e-06,
       "learning_rate": 9.34792587788002e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 7.89907693862915,
+      "logits/rejected": 7.527780055999756,
+      "logps/chosen": -38.19247817993164,
+      "logps/rejected": -39.80179214477539,
+      "loss": 0.0,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -38.19247817993164,
+      "rewards/margins": 1.6093158721923828,
+      "rewards/rejected": -39.80179214477539,
       "step": 1390
     },
     {
       "epoch": 0.7466131460110387,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 0.00011260936289540233,
       "learning_rate": 9.34021460937342e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 7.540017127990723,
+      "logits/rejected": 7.481609344482422,
+      "logps/chosen": -38.1087760925293,
+      "logps/rejected": -38.50666046142578,
+      "loss": 0.0,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": -38.1087760925293,
+      "rewards/margins": 0.39788302779197693,
+      "rewards/rejected": -38.50666046142578,
       "step": 1395
     },
     {
       "epoch": 0.7492891787924402,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.501187632566141e-08,
       "learning_rate": 9.332461232085646e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 7.9191179275512695,
+      "logits/rejected": 7.414793491363525,
+      "logps/chosen": -37.29340744018555,
+      "logps/rejected": -38.46513748168945,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -37.29340744018555,
+      "rewards/margins": 1.171724557876587,
+      "rewards/rejected": -38.46513748168945,
       "step": 1400
     },
     {
       "epoch": 0.7519652115738418,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.046631814084412e-06,
       "learning_rate": 9.324665821239998e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 7.848758697509766,
+      "logits/rejected": 7.459347724914551,
+      "logps/chosen": -37.425498962402344,
+      "logps/rejected": -38.769317626953125,
+      "loss": 0.0,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -37.425498962402344,
+      "rewards/margins": 1.3438196182250977,
+      "rewards/rejected": -38.769317626953125,
       "step": 1405
     },
     {
       "epoch": 0.7546412443552434,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.710895590921679e-08,
       "learning_rate": 9.316828452467583e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 7.838174343109131,
+      "logits/rejected": 7.629962921142578,
+      "logps/chosen": -38.402591705322266,
+      "logps/rejected": -40.70613479614258,
+      "loss": 0.0,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -38.402591705322266,
+      "rewards/margins": 2.303539752960205,
+      "rewards/rejected": -40.70613479614258,
       "step": 1410
     },
     {
       "epoch": 0.7573172771366449,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.961317145990124e-06,
       "learning_rate": 9.30894920180659e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 7.770921230316162,
+      "logits/rejected": 7.653811454772949,
+      "logps/chosen": -37.28989791870117,
+      "logps/rejected": -39.459571838378906,
+      "loss": 0.0,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -37.28989791870117,
+      "rewards/margins": 2.1696786880493164,
+      "rewards/rejected": -39.459571838378906,
       "step": 1415
     },
     {
       "epoch": 0.7599933099180465,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.0897937311184044e-05,
       "learning_rate": 9.301028145701543e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 7.6099958419799805,
+      "logits/rejected": 7.298924922943115,
+      "logps/chosen": -38.49353790283203,
+      "logps/rejected": -40.264835357666016,
+      "loss": 0.0,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -38.49353790283203,
+      "rewards/margins": 1.7713000774383545,
+      "rewards/rejected": -40.264835357666016,
       "step": 1420
     },
     {
       "epoch": 0.7626693426994481,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.3305567730394303e-05,
       "learning_rate": 9.293065361002563e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 7.934554100036621,
+      "logits/rejected": 7.427609443664551,
+      "logps/chosen": -38.10251235961914,
+      "logps/rejected": -38.60419845581055,
+      "loss": 0.0021,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -38.10251235961914,
+      "rewards/margins": 0.5016874074935913,
+      "rewards/rejected": -38.60419845581055,
       "step": 1425
     },
     {
       "epoch": 0.7653453754808497,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.4062977409438572e-08,
       "learning_rate": 9.285060924964622e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 7.973842620849609,
+      "logits/rejected": 7.8127593994140625,
+      "logps/chosen": -38.819007873535156,
+      "logps/rejected": -40.2440071105957,
+      "loss": 0.0,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -38.819007873535156,
+      "rewards/margins": 1.4250024557113647,
+      "rewards/rejected": -40.2440071105957,
       "step": 1430
     },
     {
       "epoch": 0.7680214082622512,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 9.975349442330303e-06,
       "learning_rate": 9.277014915246792e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 7.839162349700928,
+      "logits/rejected": 7.7627105712890625,
+      "logps/chosen": -38.88307189941406,
+      "logps/rejected": -39.49738693237305,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -38.88307189941406,
+      "rewards/margins": 0.6143161654472351,
+      "rewards/rejected": -39.49738693237305,
       "step": 1435
     },
     {
       "epoch": 0.7706974410436528,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 8.637710104131475e-06,
       "learning_rate": 9.268927409911498e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 7.771187782287598,
+      "logits/rejected": 7.478902339935303,
+      "logps/chosen": -37.142513275146484,
+      "logps/rejected": -38.183467864990234,
+      "loss": 0.0,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -37.142513275146484,
+      "rewards/margins": 1.0409519672393799,
+      "rewards/rejected": -38.183467864990234,
       "step": 1440
     },
     {
       "epoch": 0.7733734738250544,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.0452485738742639e-06,
       "learning_rate": 9.260798487423749e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 7.9516777992248535,
+      "logits/rejected": 7.4535932540893555,
+      "logps/chosen": -36.872398376464844,
+      "logps/rejected": -39.068668365478516,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -36.872398376464844,
+      "rewards/margins": 2.1962697505950928,
+      "rewards/rejected": -39.068668365478516,
       "step": 1445
     },
     {
       "epoch": 0.7760495066064559,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 5.006829784080116e-06,
       "learning_rate": 9.252628226650389e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 7.831016540527344,
+      "logits/rejected": 7.875929355621338,
+      "logps/chosen": -38.368690490722656,
+      "logps/rejected": -39.260658264160156,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -38.368690490722656,
+      "rewards/margins": 0.8919647336006165,
+      "rewards/rejected": -39.260658264160156,
       "step": 1450
     },
     {
       "epoch": 0.7787255393878575,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 5.184657435857802e-08,
       "learning_rate": 9.244416706859321e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.145885467529297,
+      "logits/rejected": 7.772180080413818,
+      "logps/chosen": -38.11443328857422,
+      "logps/rejected": -40.17070770263672,
+      "loss": 0.0004,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -38.11443328857422,
+      "rewards/margins": 2.056274890899658,
+      "rewards/rejected": -40.17070770263672,
       "step": 1455
     },
     {
       "epoch": 0.7814015721692591,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.0298486667660806e-07,
       "learning_rate": 9.23616400771875e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.182382583618164,
+      "logits/rejected": 7.63820743560791,
+      "logps/chosen": -38.30253601074219,
+      "logps/rejected": -40.93608474731445,
+      "loss": 0.0,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -38.30253601074219,
+      "rewards/margins": 2.633549213409424,
+      "rewards/rejected": -40.93608474731445,
       "step": 1460
     },
     {
       "epoch": 0.7840776049506607,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.060329968153281e-09,
       "learning_rate": 9.227870209296395e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 7.824871063232422,
+      "logits/rejected": 7.533638000488281,
+      "logps/chosen": -37.28596496582031,
+      "logps/rejected": -38.919158935546875,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -37.28596496582031,
+      "rewards/margins": 1.6331911087036133,
+      "rewards/rejected": -38.919158935546875,
       "step": 1465
     },
     {
       "epoch": 0.7867536377320622,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 6.307530129491754e-08,
       "learning_rate": 9.219535392058728e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 7.659951210021973,
+      "logits/rejected": 7.760500907897949,
+      "logps/chosen": -38.72264099121094,
+      "logps/rejected": -39.74140167236328,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -38.72264099121094,
+      "rewards/margins": 1.0187584161758423,
+      "rewards/rejected": -39.74140167236328,
       "step": 1470
     },
     {
       "epoch": 0.7894296705134638,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 5.7826480907625454e-05,
       "learning_rate": 9.211159636870181e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 7.978939056396484,
+      "logits/rejected": 7.410494327545166,
+      "logps/chosen": -37.2846565246582,
+      "logps/rejected": -39.65030288696289,
+      "loss": 0.0,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -37.2846565246582,
+      "rewards/margins": 2.3656468391418457,
+      "rewards/rejected": -39.65030288696289,
       "step": 1475
     },
     {
       "epoch": 0.7921057032948654,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.6875708712490975e-05,
       "learning_rate": 9.202743024992367e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 7.731909275054932,
+      "logits/rejected": 7.61098575592041,
+      "logps/chosen": -38.458248138427734,
+      "logps/rejected": -40.160743713378906,
+      "loss": 0.0,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -38.458248138427734,
+      "rewards/margins": 1.7024974822998047,
+      "rewards/rejected": -40.160743713378906,
       "step": 1480
     },
     {
       "epoch": 0.7947817360762669,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 6.337849622040809e-06,
       "learning_rate": 9.194285638083293e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 7.647625923156738,
+      "logits/rejected": 7.290806770324707,
+      "logps/chosen": -36.965736389160156,
+      "logps/rejected": -38.991966247558594,
+      "loss": 0.0,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -36.965736389160156,
+      "rewards/margins": 2.0262298583984375,
+      "rewards/rejected": -38.991966247558594,
       "step": 1485
     },
     {
       "epoch": 0.7974577688576685,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 7.976961720181852e-05,
       "learning_rate": 9.185787558196562e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 7.986492156982422,
+      "logits/rejected": 7.838613986968994,
+      "logps/chosen": -38.393680572509766,
+      "logps/rejected": -40.16486358642578,
+      "loss": 0.0,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -38.393680572509766,
+      "rewards/margins": 1.7711822986602783,
+      "rewards/rejected": -40.16486358642578,
       "step": 1490
     },
     {
       "epoch": 0.8001338016390701,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.4953968429660495e-09,
       "learning_rate": 9.177248867780583e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 7.939672946929932,
+      "logits/rejected": 7.871676445007324,
+      "logps/chosen": -38.46574020385742,
+      "logps/rejected": -39.33441925048828,
+      "loss": 0.0,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -38.46574020385742,
+      "rewards/margins": 0.8686827421188354,
+      "rewards/rejected": -39.33441925048828,
       "step": 1495
     },
     {
       "epoch": 0.8028098344204716,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 0.21504986252539593,
       "learning_rate": 9.168669649677769e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 7.964795112609863,
+      "logits/rejected": 7.848135948181152,
+      "logps/chosen": -38.21345520019531,
+      "logps/rejected": -39.71274185180664,
+      "loss": 0.0009,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -38.21345520019531,
+      "rewards/margins": 1.4992845058441162,
+      "rewards/rejected": -39.71274185180664,
       "step": 1500
     },
     {
       "epoch": 0.8054858672018732,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 7.833226036030094e-06,
       "learning_rate": 9.16004998712373e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 7.877539157867432,
+      "logits/rejected": 7.718256950378418,
+      "logps/chosen": -37.53843307495117,
+      "logps/rejected": -39.277259826660156,
+      "loss": 0.0,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -37.53843307495117,
+      "rewards/margins": 1.7388229370117188,
+      "rewards/rejected": -39.277259826660156,
       "step": 1505
     },
     {
       "epoch": 0.8081618999832748,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.5945502243345816e-06,
       "learning_rate": 9.151389963746472e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 7.8832502365112305,
+      "logits/rejected": 7.068793296813965,
+      "logps/chosen": -37.3635368347168,
+      "logps/rejected": -39.96464157104492,
+      "loss": 0.0,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -37.3635368347168,
+      "rewards/margins": 2.6011085510253906,
+      "rewards/rejected": -39.96464157104492,
       "step": 1510
     },
     {
       "epoch": 0.8108379327646764,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.4660878808362571e-06,
       "learning_rate": 9.142689663565577e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 7.9765191078186035,
+      "logits/rejected": 7.9726762771606445,
+      "logps/chosen": -38.6474609375,
+      "logps/rejected": -39.47627639770508,
+      "loss": 0.0,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -38.6474609375,
+      "rewards/margins": 0.8288129568099976,
+      "rewards/rejected": -39.47627639770508,
       "step": 1515
     },
     {
       "epoch": 0.8135139655460779,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 5.661305889648504e-07,
       "learning_rate": 9.133949170991397e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 7.798492431640625,
+      "logits/rejected": 7.728209018707275,
+      "logps/chosen": -39.126319885253906,
+      "logps/rejected": -40.037742614746094,
+      "loss": 0.0,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -39.126319885253906,
+      "rewards/margins": 0.9114225506782532,
+      "rewards/rejected": -40.037742614746094,
       "step": 1520
     },
     {
       "epoch": 0.8161899983274795,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.2416085141255877e-06,
       "learning_rate": 9.125168570824231e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.422554016113281,
+      "logits/rejected": 7.839612007141113,
+      "logps/chosen": -38.852500915527344,
+      "logps/rejected": -39.63780212402344,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5062500238418579,
+      "rewards/chosen": -38.852500915527344,
+      "rewards/margins": 0.7852976322174072,
+      "rewards/rejected": -39.63780212402344,
       "step": 1525
     },
     {
       "epoch": 0.8188660311088811,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.8940095013098934e-07,
       "learning_rate": 9.116347948253496e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 7.8123602867126465,
+      "logits/rejected": 7.428183078765869,
+      "logps/chosen": -36.50711441040039,
+      "logps/rejected": -38.40597152709961,
+      "loss": 0.0,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -36.50711441040039,
+      "rewards/margins": 1.8988609313964844,
+      "rewards/rejected": -38.40597152709961,
       "step": 1530
     },
     {
       "epoch": 0.8215420638902826,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 7.654121182204357e-07,
       "learning_rate": 9.107487388856916e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 7.914336204528809,
+      "logits/rejected": 7.348167419433594,
+      "logps/chosen": -38.14690399169922,
+      "logps/rejected": -39.80203628540039,
+      "loss": 0.0,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -38.14690399169922,
+      "rewards/margins": 1.655135154724121,
+      "rewards/rejected": -39.80203628540039,
       "step": 1535
     },
     {
       "epoch": 0.8242180966716842,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.167484314988425e-07,
       "learning_rate": 9.098586978599673e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.00414752960205,
+      "logits/rejected": 7.612962245941162,
+      "logps/chosen": -39.2923583984375,
+      "logps/rejected": -40.04558181762695,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -39.2923583984375,
+      "rewards/margins": 0.7532222867012024,
+      "rewards/rejected": -40.04558181762695,
       "step": 1540
     },
     {
       "epoch": 0.8268941294530858,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.4633813092798436e-11,
       "learning_rate": 9.089646803833588e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 7.893210411071777,
+      "logits/rejected": 7.460615634918213,
+      "logps/chosen": -37.18971252441406,
+      "logps/rejected": -39.01754379272461,
+      "loss": 0.0,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -37.18971252441406,
+      "rewards/margins": 1.8278295993804932,
+      "rewards/rejected": -39.01754379272461,
       "step": 1545
     },
     {
       "epoch": 0.8295701622344873,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.6015045712650233e-09,
       "learning_rate": 9.080666951296276e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.132731437683105,
+      "logits/rejected": 7.5076704025268555,
+      "logps/chosen": -37.65477752685547,
+      "logps/rejected": -39.759315490722656,
+      "loss": 0.0,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -37.65477752685547,
+      "rewards/margins": 2.1045339107513428,
+      "rewards/rejected": -39.759315490722656,
       "step": 1550
     },
     {
       "epoch": 0.8322461950158889,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.7997245147723211e-07,
       "learning_rate": 9.071647508110305e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.195427894592285,
+      "logits/rejected": 7.442620754241943,
+      "logps/chosen": -36.724693298339844,
+      "logps/rejected": -39.815242767333984,
+      "loss": 0.0,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -36.724693298339844,
+      "rewards/margins": 3.090546131134033,
+      "rewards/rejected": -39.815242767333984,
       "step": 1555
     },
     {
       "epoch": 0.8349222277972905,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 6.536255965349954e-09,
       "learning_rate": 9.062588561782354e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 7.856629848480225,
+      "logits/rejected": 7.8618903160095215,
+      "logps/chosen": -40.22589111328125,
+      "logps/rejected": -41.21640396118164,
+      "loss": 0.0,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -40.22589111328125,
+      "rewards/margins": 0.9905115962028503,
+      "rewards/rejected": -41.21640396118164,
       "step": 1560
     },
     {
       "epoch": 0.8375982605786921,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 8.558773747013863e-06,
       "learning_rate": 9.053490200202358e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.052000045776367,
+      "logits/rejected": 8.14100170135498,
+      "logps/chosen": -38.748878479003906,
+      "logps/rejected": -39.23004913330078,
+      "loss": 0.0,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -38.748878479003906,
+      "rewards/margins": 0.481168657541275,
+      "rewards/rejected": -39.23004913330078,
       "step": 1565
     },
     {
       "epoch": 0.8402742933600936,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.384392143423722e-07,
       "learning_rate": 9.044352511642661e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.139077186584473,
+      "logits/rejected": 7.994954586029053,
+      "logps/chosen": -39.8129768371582,
+      "logps/rejected": -40.227779388427734,
+      "loss": 0.0,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": -39.8129768371582,
+      "rewards/margins": 0.4148061275482178,
+      "rewards/rejected": -40.227779388427734,
       "step": 1570
     },
     {
       "epoch": 0.8429503261414952,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 0.44767500323759146,
       "learning_rate": 9.03517558475716e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 7.806936740875244,
+      "logits/rejected": 7.571896553039551,
+      "logps/chosen": -37.864234924316406,
+      "logps/rejected": -38.46239471435547,
+      "loss": 0.0017,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": -37.864234924316406,
+      "rewards/margins": 0.5981568098068237,
+      "rewards/rejected": -38.46239471435547,
       "step": 1575
     },
     {
       "epoch": 0.8456263589228968,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 9.082438208754553e-08,
       "learning_rate": 9.025959508580436e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 7.9717912673950195,
+      "logits/rejected": 7.251823425292969,
+      "logps/chosen": -37.87413024902344,
+      "logps/rejected": -39.73711013793945,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -37.87413024902344,
+      "rewards/margins": 1.8629745244979858,
+      "rewards/rejected": -39.73711013793945,
       "step": 1580
     },
     {
       "epoch": 0.8483023917042983,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 0.00022946978330174063,
       "learning_rate": 9.016704372526905e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.054485321044922,
+      "logits/rejected": 7.613073825836182,
+      "logps/chosen": -37.721954345703125,
+      "logps/rejected": -39.49103546142578,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -37.721954345703125,
+      "rewards/margins": 1.7690833806991577,
+      "rewards/rejected": -39.49103546142578,
       "step": 1585
     },
     {
       "epoch": 0.8509784244856999,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 9.462837610014242e-08,
       "learning_rate": 9.007410266389934e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.215543746948242,
+      "logits/rejected": 8.285394668579102,
+      "logps/chosen": -39.02608108520508,
+      "logps/rejected": -40.97217559814453,
+      "loss": 0.0,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -39.02608108520508,
+      "rewards/margins": 1.9460923671722412,
+      "rewards/rejected": -40.97217559814453,
       "step": 1590
     },
     {
       "epoch": 0.8536544572671015,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.8100877377158613e-05,
       "learning_rate": 8.998077280340981e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 7.841731071472168,
+      "logits/rejected": 7.768154144287109,
+      "logps/chosen": -38.7288818359375,
+      "logps/rejected": -39.626564025878906,
+      "loss": 0.0,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -38.7288818359375,
+      "rewards/margins": 0.8976883888244629,
+      "rewards/rejected": -39.626564025878906,
       "step": 1595
     },
     {
       "epoch": 0.8563304900485031,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 9.256304834389255e-06,
       "learning_rate": 8.988705504928722e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.257303237915039,
+      "logits/rejected": 7.808728218078613,
+      "logps/chosen": -38.027042388916016,
+      "logps/rejected": -39.88721466064453,
+      "loss": 0.0,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -38.027042388916016,
+      "rewards/margins": 1.8601722717285156,
+      "rewards/rejected": -39.88721466064453,
       "step": 1600
     },
     {
       "epoch": 0.8563304900485031,
-      "eval_logits/chosen": NaN,
-      "eval_logits/rejected": NaN,
-      "eval_logps/chosen": NaN,
-      "eval_logps/rejected": NaN,
-      "eval_loss": NaN,
-      "eval_rewards/accuracies": 0.0,
-      "eval_rewards/chosen": NaN,
-      "eval_rewards/margins": NaN,
-      "eval_rewards/rejected": NaN,
-      "eval_runtime": 34.8813,
-      "eval_samples_per_second": 38.559,
-      "eval_semantic_entropy": NaN,
-      "eval_steps_per_second": 9.661,
+      "eval_logits/chosen": 6.747125148773193,
+      "eval_logits/rejected": 6.570799350738525,
+      "eval_logps/chosen": -38.59125900268555,
+      "eval_logps/rejected": -40.01066207885742,
+      "eval_loss": 8.234294455178315e-07,
+      "eval_rewards/accuracies": 0.5667656064033508,
+      "eval_rewards/chosen": -38.59125900268555,
+      "eval_rewards/margins": 1.419403314590454,
+      "eval_rewards/rejected": -40.01066207885742,
+      "eval_runtime": 40.2701,
+      "eval_samples_per_second": 33.399,
+      "eval_steps_per_second": 8.368,
       "step": 1600
     },
     {
       "epoch": 0.8590065228299046,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 0.0008057370548501174,
       "learning_rate": 8.979295031078157e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.098664283752441,
+      "logits/rejected": 7.638497829437256,
+      "logps/chosen": -36.81837844848633,
+      "logps/rejected": -39.67369842529297,
+      "loss": 0.0,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -36.81837844848633,
+      "rewards/margins": 2.8553221225738525,
+      "rewards/rejected": -39.67369842529297,
       "step": 1605
     },
     {
       "epoch": 0.8616825556113062,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.4653734778936245e-08,
       "learning_rate": 8.969845950089751e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.21097469329834,
+      "logits/rejected": 7.8763885498046875,
+      "logps/chosen": -38.55016326904297,
+      "logps/rejected": -40.79650115966797,
+      "loss": 0.0,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -38.55016326904297,
+      "rewards/margins": 2.2463347911834717,
+      "rewards/rejected": -40.79650115966797,
       "step": 1610
     },
     {
       "epoch": 0.8643585883927078,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.510124901555926e-05,
       "learning_rate": 8.960358353638526e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.293069839477539,
+      "logits/rejected": 7.962773323059082,
+      "logps/chosen": -39.41699981689453,
+      "logps/rejected": -40.80271530151367,
+      "loss": 0.0,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -39.41699981689453,
+      "rewards/margins": 1.385716438293457,
+      "rewards/rejected": -40.80271530151367,
       "step": 1615
     },
     {
       "epoch": 0.8670346211741093,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.3109905775399387e-06,
       "learning_rate": 8.950832333773184e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.098249435424805,
+      "logits/rejected": 7.695958614349365,
+      "logps/chosen": -39.975311279296875,
+      "logps/rejected": -41.687217712402344,
+      "loss": 0.0,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -39.975311279296875,
+      "rewards/margins": 1.7119166851043701,
+      "rewards/rejected": -41.687217712402344,
       "step": 1620
     },
     {
       "epoch": 0.869710653955511,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 0.0008437320519301152,
       "learning_rate": 8.941267982915213e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.234050750732422,
+      "logits/rejected": 7.85067892074585,
+      "logps/chosen": -39.65726089477539,
+      "logps/rejected": -38.728355407714844,
+      "loss": 0.0,
+      "rewards/accuracies": 0.4937500059604645,
+      "rewards/chosen": -39.65726089477539,
+      "rewards/margins": -0.9288986325263977,
+      "rewards/rejected": -38.728355407714844,
       "step": 1625
     },
     {
       "epoch": 0.8723866867369126,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 7.849080963198015e-06,
       "learning_rate": 8.931665393857983e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.340258598327637,
+      "logits/rejected": 7.9926862716674805,
+      "logps/chosen": -39.31509780883789,
+      "logps/rejected": -41.00221633911133,
+      "loss": 0.0,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -39.31509780883789,
+      "rewards/margins": 1.6871140003204346,
+      "rewards/rejected": -41.00221633911133,
       "step": 1630
     },
     {
       "epoch": 0.875062719518314,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.6115655093926326e-11,
       "learning_rate": 8.922024659765861e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.636944770812988,
+      "logits/rejected": 8.193704605102539,
+      "logps/chosen": -39.755184173583984,
+      "logps/rejected": -41.33856964111328,
+      "loss": 0.0,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": -39.755184173583984,
+      "rewards/margins": 1.5833840370178223,
+      "rewards/rejected": -41.33856964111328,
       "step": 1635
     },
     {
       "epoch": 0.8777387522997157,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.1180178069155614e-09,
       "learning_rate": 8.912345874173288e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.38740062713623,
+      "logits/rejected": 8.054380416870117,
+      "logps/chosen": -39.5725212097168,
+      "logps/rejected": -41.63652801513672,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -39.5725212097168,
+      "rewards/margins": 2.06400990486145,
+      "rewards/rejected": -41.63652801513672,
       "step": 1640
     },
     {
       "epoch": 0.8804147850811173,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.7386987524782393e-06,
       "learning_rate": 8.902629130983885e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.204886436462402,
+      "logits/rejected": 8.130419731140137,
+      "logps/chosen": -39.0472297668457,
+      "logps/rejected": -39.94300079345703,
+      "loss": 0.0006,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -39.0472297668457,
+      "rewards/margins": 0.8957681655883789,
+      "rewards/rejected": -39.94300079345703,
       "step": 1645
     },
     {
       "epoch": 0.8830908178625189,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.0202997218922968e-05,
       "learning_rate": 8.892874524469537e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.438112258911133,
+      "logits/rejected": 8.220708847045898,
+      "logps/chosen": -38.15906524658203,
+      "logps/rejected": -38.746986389160156,
+      "loss": 0.0,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -38.15906524658203,
+      "rewards/margins": 0.5879203081130981,
+      "rewards/rejected": -38.746986389160156,
       "step": 1650
     },
     {
       "epoch": 0.8857668506439204,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.8190647719724933e-07,
       "learning_rate": 8.883082149269478e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.3302001953125,
+      "logits/rejected": 8.150362968444824,
+      "logps/chosen": -39.60652542114258,
+      "logps/rejected": -40.55384063720703,
+      "loss": 0.0,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -39.60652542114258,
+      "rewards/margins": 0.9473205804824829,
+      "rewards/rejected": -40.55384063720703,
       "step": 1655
     },
     {
       "epoch": 0.888442883425322,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 6.371624494529953e-07,
       "learning_rate": 8.873252100389377e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.409173011779785,
+      "logits/rejected": 8.225865364074707,
+      "logps/chosen": -38.64631652832031,
+      "logps/rejected": -39.588199615478516,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -38.64631652832031,
+      "rewards/margins": 0.9418843984603882,
+      "rewards/rejected": -39.588199615478516,
       "step": 1660
     },
     {
       "epoch": 0.8911189162067236,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.3970831175758207e-05,
       "learning_rate": 8.863384473200411e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 7.792959690093994,
+      "logits/rejected": 7.656949043273926,
+      "logps/chosen": -39.32639694213867,
+      "logps/rejected": -40.312599182128906,
+      "loss": 0.0,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -39.32639694213867,
+      "rewards/margins": 0.9861971735954285,
+      "rewards/rejected": -40.312599182128906,
       "step": 1665
     },
     {
       "epoch": 0.8937949489881251,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 9.262325333340677e-07,
       "learning_rate": 8.853479363438342e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.590107917785645,
+      "logits/rejected": 8.037742614746094,
+      "logps/chosen": -40.97118377685547,
+      "logps/rejected": -42.141761779785156,
+      "loss": 0.0,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -40.97118377685547,
+      "rewards/margins": 1.170578956604004,
+      "rewards/rejected": -42.141761779785156,
       "step": 1670
     },
     {
       "epoch": 0.8964709817695267,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.3293312236129075e-06,
       "learning_rate": 8.843536867202588e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.477182388305664,
+      "logits/rejected": 8.070070266723633,
+      "logps/chosen": -40.372779846191406,
+      "logps/rejected": -41.76821517944336,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -40.372779846191406,
+      "rewards/margins": 1.3954370021820068,
+      "rewards/rejected": -41.76821517944336,
       "step": 1675
     },
     {
       "epoch": 0.8991470145509283,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.4413171514850148e-06,
       "learning_rate": 8.833557080955292e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.12739372253418,
+      "logits/rejected": 7.710951805114746,
+      "logps/chosen": -38.87654113769531,
+      "logps/rejected": -39.08584976196289,
+      "loss": 0.0,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -38.87654113769531,
+      "rewards/margins": 0.20930734276771545,
+      "rewards/rejected": -39.08584976196289,
       "step": 1680
     },
     {
       "epoch": 0.9018230473323299,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.9459653876122e-06,
       "learning_rate": 8.823540101520381e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.33852767944336,
+      "logits/rejected": 7.771746635437012,
+      "logps/chosen": -39.13180160522461,
+      "logps/rejected": -40.846595764160156,
+      "loss": 0.0,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -39.13180160522461,
+      "rewards/margins": 1.7147928476333618,
+      "rewards/rejected": -40.846595764160156,
       "step": 1685
     },
     {
       "epoch": 0.9044990801137314,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 0.0006194867369532477,
       "learning_rate": 8.813486026082637e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.267389297485352,
+      "logits/rejected": 7.970789432525635,
+      "logps/chosen": -39.289695739746094,
+      "logps/rejected": -41.38640213012695,
+      "loss": 0.0,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -39.289695739746094,
+      "rewards/margins": 2.096703290939331,
+      "rewards/rejected": -41.38640213012695,
       "step": 1690
     },
     {
       "epoch": 0.907175112895133,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 6.549365291969407e-09,
       "learning_rate": 8.803394952186742e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.466710090637207,
+      "logits/rejected": 8.094820022583008,
+      "logps/chosen": -39.80694580078125,
+      "logps/rejected": -42.62937927246094,
+      "loss": 0.0,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -39.80694580078125,
+      "rewards/margins": 2.8224294185638428,
+      "rewards/rejected": -42.62937927246094,
       "step": 1695
     },
     {
       "epoch": 0.9098511456765346,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.835247780281827e-10,
       "learning_rate": 8.793266977736342e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 7.980792999267578,
+      "logits/rejected": 8.23906135559082,
+      "logps/chosen": -40.428585052490234,
+      "logps/rejected": -40.2591552734375,
+      "loss": 0.0,
+      "rewards/accuracies": 0.45625001192092896,
+      "rewards/chosen": -40.428585052490234,
+      "rewards/margins": -0.1694318801164627,
+      "rewards/rejected": -40.2591552734375,
       "step": 1700
     },
     {
       "epoch": 0.9125271784579361,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.7940095700771216e-11,
       "learning_rate": 8.783102200993085e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.180706024169922,
+      "logits/rejected": 8.084714889526367,
+      "logps/chosen": -38.676658630371094,
+      "logps/rejected": -40.654476165771484,
+      "loss": 0.0,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -38.676658630371094,
+      "rewards/margins": 1.9778194427490234,
+      "rewards/rejected": -40.654476165771484,
       "step": 1705
     },
     {
       "epoch": 0.9152032112393377,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 0.002906266536821842,
       "learning_rate": 8.772900720575683e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.309261322021484,
+      "logits/rejected": 8.181894302368164,
+      "logps/chosen": -38.73280334472656,
+      "logps/rejected": -40.1246452331543,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -38.73280334472656,
+      "rewards/margins": 1.3918428421020508,
+      "rewards/rejected": -40.1246452331543,
       "step": 1710
     },
     {
       "epoch": 0.9178792440207393,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.1588609476828104e-07,
       "learning_rate": 8.762662635458944e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.500656127929688,
+      "logits/rejected": 8.06970500946045,
+      "logps/chosen": -40.282676696777344,
+      "logps/rejected": -41.220420837402344,
+      "loss": 0.0,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -40.282676696777344,
+      "rewards/margins": 0.937739372253418,
+      "rewards/rejected": -41.220420837402344,
       "step": 1715
     },
     {
       "epoch": 0.9205552768021408,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 8.447053040242686e-07,
       "learning_rate": 8.752388044972811e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 7.965966701507568,
+      "logits/rejected": 7.805613040924072,
+      "logps/chosen": -38.75082778930664,
+      "logps/rejected": -40.24550247192383,
+      "loss": 0.0,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -38.75082778930664,
+      "rewards/margins": 1.4946743249893188,
+      "rewards/rejected": -40.24550247192383,
       "step": 1720
     },
     {
       "epoch": 0.9232313095835424,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 8.514210680346087e-08,
       "learning_rate": 8.74207704880141e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 7.839042663574219,
+      "logits/rejected": 7.6489458084106445,
+      "logps/chosen": -38.32817840576172,
+      "logps/rejected": -39.988704681396484,
+      "loss": 0.0,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -38.32817840576172,
+      "rewards/margins": 1.6605284214019775,
+      "rewards/rejected": -39.988704681396484,
       "step": 1725
     },
     {
       "epoch": 0.925907342364944,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.2854338711523137e-08,
       "learning_rate": 8.731729746982068e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 7.906767845153809,
+      "logits/rejected": 7.990408897399902,
+      "logps/chosen": -38.86058807373047,
+      "logps/rejected": -40.13646697998047,
+      "loss": 0.0,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -38.86058807373047,
+      "rewards/margins": 1.2758748531341553,
+      "rewards/rejected": -40.13646697998047,
       "step": 1730
     },
     {
       "epoch": 0.9285833751463456,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.026963968553163e-09,
       "learning_rate": 8.721346239904355e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.595671653747559,
+      "logits/rejected": 7.907167911529541,
+      "logps/chosen": -38.16680908203125,
+      "logps/rejected": -41.04495620727539,
+      "loss": 0.0,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -38.16680908203125,
+      "rewards/margins": 2.8781521320343018,
+      "rewards/rejected": -41.04495620727539,
       "step": 1735
     },
     {
       "epoch": 0.9312594079277471,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.0622232123449494e-07,
       "learning_rate": 8.710926628309101e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.425874710083008,
+      "logits/rejected": 7.8293776512146,
+      "logps/chosen": -38.79697799682617,
+      "logps/rejected": -40.86872100830078,
+      "loss": 0.0,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -38.79697799682617,
+      "rewards/margins": 2.0717406272888184,
+      "rewards/rejected": -40.86872100830078,
       "step": 1740
     },
     {
       "epoch": 0.9339354407091487,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 7.205259694180774e-08,
       "learning_rate": 8.700471013287424e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 7.958510398864746,
+      "logits/rejected": 7.977458953857422,
+      "logps/chosen": -40.835960388183594,
+      "logps/rejected": -40.94975280761719,
+      "loss": 0.0,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": -40.835960388183594,
+      "rewards/margins": 0.11378999799489975,
+      "rewards/rejected": -40.94975280761719,
       "step": 1745
     },
     {
       "epoch": 0.9366114734905503,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.2549297995473493e-07,
       "learning_rate": 8.689979496279746e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.27040958404541,
+      "logits/rejected": 8.281350135803223,
+      "logps/chosen": -40.61737823486328,
+      "logps/rejected": -41.963844299316406,
+      "loss": 0.0,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": -40.61737823486328,
+      "rewards/margins": 1.3464667797088623,
+      "rewards/rejected": -41.963844299316406,
       "step": 1750
     },
     {
       "epoch": 0.9392875062719518,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 5.909337954162276e-10,
       "learning_rate": 8.679452179074811e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.432409286499023,
+      "logits/rejected": 8.010265350341797,
+      "logps/chosen": -39.39599609375,
+      "logps/rejected": -40.23656463623047,
+      "loss": 0.0,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -39.39599609375,
+      "rewards/margins": 0.8405696749687195,
+      "rewards/rejected": -40.23656463623047,
       "step": 1755
     },
     {
       "epoch": 0.9419635390533534,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 6.976137089042503e-05,
       "learning_rate": 8.668889163808698e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.588744163513184,
+      "logits/rejected": 8.143301010131836,
+      "logps/chosen": -39.54351806640625,
+      "logps/rejected": -40.878662109375,
+      "loss": 0.0,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -39.54351806640625,
+      "rewards/margins": 1.3351447582244873,
+      "rewards/rejected": -40.878662109375,
       "step": 1760
     },
     {
       "epoch": 0.944639571834755,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.538311254574968e-06,
       "learning_rate": 8.658290552963827e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.29702377319336,
+      "logits/rejected": 8.092007637023926,
+      "logps/chosen": -40.025142669677734,
+      "logps/rejected": -40.387054443359375,
+      "loss": 0.0,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -40.025142669677734,
+      "rewards/margins": 0.36191269755363464,
+      "rewards/rejected": -40.387054443359375,
       "step": 1765
     },
     {
       "epoch": 0.9473156046161565,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 0.00038598147411679994,
       "learning_rate": 8.647656449367966e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.497171401977539,
+      "logits/rejected": 8.19450855255127,
+      "logps/chosen": -40.634422302246094,
+      "logps/rejected": -41.51022720336914,
+      "loss": 0.0,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -40.634422302246094,
+      "rewards/margins": 0.8758062124252319,
+      "rewards/rejected": -41.51022720336914,
       "step": 1770
     },
     {
       "epoch": 0.9499916373975581,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 0.0012665874093443585,
       "learning_rate": 8.636986956193235e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.121536254882812,
+      "logits/rejected": 7.710322380065918,
+      "logps/chosen": -39.73516082763672,
+      "logps/rejected": -40.877525329589844,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -39.73516082763672,
+      "rewards/margins": 1.142362356185913,
+      "rewards/rejected": -40.877525329589844,
       "step": 1775
     },
     {
       "epoch": 0.9526676701789597,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.821477919277297e-08,
       "learning_rate": 8.626282176955104e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.649648666381836,
+      "logits/rejected": 8.281259536743164,
+      "logps/chosen": -40.564125061035156,
+      "logps/rejected": -42.40605545043945,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -40.564125061035156,
+      "rewards/margins": 1.8419277667999268,
+      "rewards/rejected": -42.40605545043945,
       "step": 1780
     },
     {
       "epoch": 0.9553437029603613,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.78053703715403e-11,
       "learning_rate": 8.615542215511389e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.362005233764648,
+      "logits/rejected": 8.212501525878906,
+      "logps/chosen": -40.52806091308594,
+      "logps/rejected": -41.784908294677734,
+      "loss": 0.0,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -40.52806091308594,
+      "rewards/margins": 1.2568451166152954,
+      "rewards/rejected": -41.784908294677734,
       "step": 1785
     },
     {
       "epoch": 0.9580197357417628,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.030643790104539e-05,
       "learning_rate": 8.604767176061241e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.17466926574707,
+      "logits/rejected": 7.714603424072266,
+      "logps/chosen": -40.766319274902344,
+      "logps/rejected": -41.360496520996094,
+      "loss": 0.0,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": -40.766319274902344,
+      "rewards/margins": 0.5941751599311829,
+      "rewards/rejected": -41.360496520996094,
       "step": 1790
     },
     {
       "epoch": 0.9606957685231644,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.669020807058701e-10,
       "learning_rate": 8.593957163144141e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.524946212768555,
+      "logits/rejected": 8.109848022460938,
+      "logps/chosen": -40.80012893676758,
+      "logps/rejected": -42.442359924316406,
+      "loss": 0.0,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -40.80012893676758,
+      "rewards/margins": 1.6422319412231445,
+      "rewards/rejected": -42.442359924316406,
       "step": 1795
     },
     {
       "epoch": 0.963371801304566,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 7.847506722154255e-06,
       "learning_rate": 8.58311228163888e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.733843803405762,
+      "logits/rejected": 8.57422161102295,
+      "logps/chosen": -40.84361267089844,
+      "logps/rejected": -41.96295166015625,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -40.84361267089844,
+      "rewards/margins": 1.119336724281311,
+      "rewards/rejected": -41.96295166015625,
       "step": 1800
     },
     {
       "epoch": 0.9660478340859675,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.671573866661769e-05,
       "learning_rate": 8.57223263676255e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.078458786010742,
+      "logits/rejected": 8.650040626525879,
+      "logps/chosen": -40.602577209472656,
+      "logps/rejected": -41.828426361083984,
+      "loss": 0.0,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -40.602577209472656,
+      "rewards/margins": 1.225846290588379,
+      "rewards/rejected": -41.828426361083984,
       "step": 1805
     },
     {
       "epoch": 0.9687238668673691,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 7.422031886836489e-07,
       "learning_rate": 8.561318334069511e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.545589447021484,
+      "logits/rejected": 8.307339668273926,
+      "logps/chosen": -41.22007751464844,
+      "logps/rejected": -42.945213317871094,
+      "loss": 0.0,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -41.22007751464844,
+      "rewards/margins": 1.7251379489898682,
+      "rewards/rejected": -42.945213317871094,
       "step": 1810
     },
     {
       "epoch": 0.9713998996487707,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.1178607957254892e-10,
       "learning_rate": 8.550369479450375e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.53073501586914,
+      "logits/rejected": 7.96996545791626,
+      "logps/chosen": -39.37406539916992,
+      "logps/rejected": -41.004798889160156,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -39.37406539916992,
+      "rewards/margins": 1.6307334899902344,
+      "rewards/rejected": -41.004798889160156,
       "step": 1815
     },
     {
       "epoch": 0.9740759324301723,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 5.870717612432264e-07,
       "learning_rate": 8.539386179130977e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.29672622680664,
+      "logits/rejected": 8.00092887878418,
+      "logps/chosen": -40.279754638671875,
+      "logps/rejected": -40.98862075805664,
+      "loss": 0.0001,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -40.279754638671875,
+      "rewards/margins": 0.7088658213615417,
+      "rewards/rejected": -40.98862075805664,
       "step": 1820
     },
     {
       "epoch": 0.9767519652115738,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 0.00023350833597350648,
       "learning_rate": 8.528368539671347e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.658112525939941,
+      "logits/rejected": 7.9644269943237305,
+      "logps/chosen": -42.06859588623047,
+      "logps/rejected": -43.560813903808594,
+      "loss": 0.0,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -42.06859588623047,
+      "rewards/margins": 1.4922200441360474,
+      "rewards/rejected": -43.560813903808594,
       "step": 1825
     },
     {
       "epoch": 0.9794279979929754,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 6.5193348088399285e-06,
       "learning_rate": 8.51731666796467e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 7.822071075439453,
+      "logits/rejected": 7.752615451812744,
+      "logps/chosen": -39.534393310546875,
+      "logps/rejected": -40.040592193603516,
+      "loss": 0.0,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -39.534393310546875,
+      "rewards/margins": 0.5061955451965332,
+      "rewards/rejected": -40.040592193603516,
       "step": 1830
     },
     {
       "epoch": 0.982104030774377,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 8.449476243670447e-08,
       "learning_rate": 8.506230671236254e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.228866577148438,
+      "logits/rejected": 7.923471927642822,
+      "logps/chosen": -40.069217681884766,
+      "logps/rejected": -41.77763366699219,
+      "loss": 0.0,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -40.069217681884766,
+      "rewards/margins": 1.7084156274795532,
+      "rewards/rejected": -41.77763366699219,
       "step": 1835
     },
     {
       "epoch": 0.9847800635557785,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 9.304173878303832e-08,
       "learning_rate": 8.495110657042488e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.36742115020752,
+      "logits/rejected": 7.849230766296387,
+      "logps/chosen": -38.560062408447266,
+      "logps/rejected": -40.37285614013672,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -38.560062408447266,
+      "rewards/margins": 1.8127937316894531,
+      "rewards/rejected": -40.37285614013672,
       "step": 1840
     },
     {
       "epoch": 0.9874560963371801,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.910410273649498e-10,
       "learning_rate": 8.483956733269799e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.148832321166992,
+      "logits/rejected": 8.080046653747559,
+      "logps/chosen": -40.808345794677734,
+      "logps/rejected": -41.62120819091797,
+      "loss": 0.0,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -40.808345794677734,
+      "rewards/margins": 0.8128618001937866,
+      "rewards/rejected": -41.62120819091797,
       "step": 1845
     },
     {
       "epoch": 0.9901321291185817,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.5991306567031894e-09,
       "learning_rate": 8.472769008133602e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.324226379394531,
+      "logits/rejected": 8.336076736450195,
+      "logps/chosen": -38.10661315917969,
+      "logps/rejected": -40.609596252441406,
+      "loss": 0.0,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -38.10661315917969,
+      "rewards/margins": 2.502983570098877,
+      "rewards/rejected": -40.609596252441406,
       "step": 1850
     },
     {
       "epoch": 0.9928081618999832,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.2668849239772056e-10,
       "learning_rate": 8.461547590177259e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.491576194763184,
+      "logits/rejected": 8.102415084838867,
+      "logps/chosen": -40.452545166015625,
+      "logps/rejected": -42.19157409667969,
+      "loss": 0.0,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -40.452545166015625,
+      "rewards/margins": 1.7390296459197998,
+      "rewards/rejected": -42.19157409667969,
       "step": 1855
     },
     {
       "epoch": 0.9954841946813848,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.5169345964498458e-10,
       "learning_rate": 8.450292588271014e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.413893699645996,
+      "logits/rejected": 8.100334167480469,
+      "logps/chosen": -40.15303421020508,
+      "logps/rejected": -41.212860107421875,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -40.15303421020508,
+      "rewards/margins": 1.0598270893096924,
+      "rewards/rejected": -41.212860107421875,
       "step": 1860
     },
     {
       "epoch": 0.9981602274627864,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 0.12548497991369065,
       "learning_rate": 8.439004111610945e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.626049041748047,
+      "logits/rejected": 8.352197647094727,
+      "logps/chosen": -40.972110748291016,
+      "logps/rejected": -41.02737045288086,
+      "loss": 0.0004,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -40.972110748291016,
+      "rewards/margins": 0.0552624836564064,
+      "rewards/rejected": -41.02737045288086,
       "step": 1865
     },
     {
       "epoch": 1.000836260244188,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.9962778138718993e-08,
       "learning_rate": 8.427682269717901e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.327581405639648,
+      "logits/rejected": 7.998424530029297,
+      "logps/chosen": -39.47612380981445,
+      "logps/rejected": -40.70171356201172,
+      "loss": 0.0,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -39.47612380981445,
+      "rewards/margins": 1.2255887985229492,
+      "rewards/rejected": -40.70171356201172,
       "step": 1870
     },
     {
       "epoch": 1.0035122930255895,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.5439795048349868e-05,
       "learning_rate": 8.416327172436446e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.931873321533203,
+      "logits/rejected": 8.444761276245117,
+      "logps/chosen": -40.64853286743164,
+      "logps/rejected": -42.009578704833984,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -40.64853286743164,
+      "rewards/margins": 1.3610453605651855,
+      "rewards/rejected": -42.009578704833984,
       "step": 1875
     },
     {
       "epoch": 1.0061883258069912,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.869508475276221e-10,
       "learning_rate": 8.404938929933778e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.344380378723145,
+      "logits/rejected": 8.09056282043457,
+      "logps/chosen": -41.148170471191406,
+      "logps/rejected": -42.11793899536133,
+      "loss": 0.0,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -41.148170471191406,
+      "rewards/margins": 0.9697696566581726,
+      "rewards/rejected": -42.11793899536133,
       "step": 1880
     },
     {
       "epoch": 1.0088643585883927,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.4687148793435951e-07,
       "learning_rate": 8.39351765269868e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.577982902526855,
+      "logits/rejected": 8.572615623474121,
+      "logps/chosen": -41.54861831665039,
+      "logps/rejected": -42.00185012817383,
+      "loss": 0.0,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -41.54861831665039,
+      "rewards/margins": 0.4532257914543152,
+      "rewards/rejected": -42.00185012817383,
       "step": 1885
     },
     {
       "epoch": 1.0115403913697942,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 5.804577554401265e-09,
       "learning_rate": 8.382063451540431e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.335451126098633,
+      "logits/rejected": 7.899486541748047,
+      "logps/chosen": -39.401573181152344,
+      "logps/rejected": -40.882843017578125,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -39.401573181152344,
+      "rewards/margins": 1.4812655448913574,
+      "rewards/rejected": -40.882843017578125,
       "step": 1890
     },
     {
       "epoch": 1.014216424151196,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 0.00020324075539934118,
       "learning_rate": 8.370576437587742e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
-      "step": 1895
+      "logits/chosen": 8.167135238647461,
+      "logits/rejected": 8.198922157287598,
+      "logps/chosen": -39.173675537109375,
+      "logps/rejected": -40.43407440185547,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -39.173675537109375,
+      "rewards/margins": 1.2603999376296997,
+      "rewards/rejected": -40.43407440185547,
+      "step": 1895
     },
     {
       "epoch": 1.0168924569325974,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.701296079733634e-05,
       "learning_rate": 8.359056722287674e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.065496444702148,
+      "logits/rejected": 8.296707153320312,
+      "logps/chosen": -39.73955535888672,
+      "logps/rejected": -41.647064208984375,
+      "loss": 0.0,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -39.73955535888672,
+      "rewards/margins": 1.9075090885162354,
+      "rewards/rejected": -41.647064208984375,
       "step": 1900
     },
     {
       "epoch": 1.019568489713999,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.5736020436401978e-06,
       "learning_rate": 8.347504417404553e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.390199661254883,
+      "logits/rejected": 8.037742614746094,
+      "logps/chosen": -39.66211700439453,
+      "logps/rejected": -41.27189636230469,
+      "loss": 0.0,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -39.66211700439453,
+      "rewards/margins": 1.6097815036773682,
+      "rewards/rejected": -41.27189636230469,
       "step": 1905
     },
     {
       "epoch": 1.0222445224954007,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 6.755093236349187e-07,
       "learning_rate": 8.335919635018893e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.552318572998047,
+      "logits/rejected": 8.313759803771973,
+      "logps/chosen": -40.83061981201172,
+      "logps/rejected": -41.274044036865234,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -40.83061981201172,
+      "rewards/margins": 0.4434239864349365,
+      "rewards/rejected": -41.274044036865234,
       "step": 1910
     },
     {
       "epoch": 1.0249205552768021,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.731198978173995e-07,
       "learning_rate": 8.324302487526303e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.598410606384277,
+      "logits/rejected": 8.269886016845703,
+      "logps/chosen": -40.91318893432617,
+      "logps/rejected": -41.74647521972656,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -40.91318893432617,
+      "rewards/margins": 0.8332853317260742,
+      "rewards/rejected": -41.74647521972656,
       "step": 1915
     },
     {
       "epoch": 1.0275965880582036,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 8.397767482558755e-08,
       "learning_rate": 8.312653087636398e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.798726081848145,
+      "logits/rejected": 8.53149700164795,
+      "logps/chosen": -41.24126434326172,
+      "logps/rejected": -42.506263732910156,
+      "loss": 0.0,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -41.24126434326172,
+      "rewards/margins": 1.264999508857727,
+      "rewards/rejected": -42.506263732910156,
       "step": 1920
     },
     {
       "epoch": 1.0302726208396054,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.573052666134527e-08,
       "learning_rate": 8.300971548371711e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.764241218566895,
+      "logits/rejected": 8.133979797363281,
+      "logps/chosen": -39.53509521484375,
+      "logps/rejected": -41.868133544921875,
+      "loss": 0.0,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -39.53509521484375,
+      "rewards/margins": 2.333033561706543,
+      "rewards/rejected": -41.868133544921875,
       "step": 1925
     },
     {
       "epoch": 1.0329486536210069,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.044770848959594e-09,
       "learning_rate": 8.289257983066582e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.660727500915527,
+      "logits/rejected": 8.264470100402832,
+      "logps/chosen": -40.130043029785156,
+      "logps/rejected": -42.04378128051758,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -40.130043029785156,
+      "rewards/margins": 1.9137340784072876,
+      "rewards/rejected": -42.04378128051758,
       "step": 1930
     },
     {
       "epoch": 1.0356246864024083,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 6.319664740266095e-07,
       "learning_rate": 8.277512505366077e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.515426635742188,
+      "logits/rejected": 8.033183097839355,
+      "logps/chosen": -39.58656311035156,
+      "logps/rejected": -42.13504409790039,
+      "loss": 0.0,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -39.58656311035156,
+      "rewards/margins": 2.548483371734619,
+      "rewards/rejected": -42.13504409790039,
       "step": 1935
     },
     {
       "epoch": 1.03830071918381,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.165473375068464e-09,
       "learning_rate": 8.265735229224868e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.569108963012695,
+      "logits/rejected": 8.4539794921875,
+      "logps/chosen": -40.05710220336914,
+      "logps/rejected": -42.40205383300781,
+      "loss": 0.0,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -40.05710220336914,
+      "rewards/margins": 2.3449490070343018,
+      "rewards/rejected": -42.40205383300781,
       "step": 1940
     },
     {
       "epoch": 1.0409767519652116,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.0050864873043155e-09,
       "learning_rate": 8.253926268906144e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.463804244995117,
+      "logits/rejected": 8.174898147583008,
+      "logps/chosen": -39.908111572265625,
+      "logps/rejected": -42.075035095214844,
+      "loss": 0.0,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -39.908111572265625,
+      "rewards/margins": 2.1669223308563232,
+      "rewards/rejected": -42.075035095214844,
       "step": 1945
     },
     {
       "epoch": 1.043652784746613,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 0.2002540350699298,
       "learning_rate": 8.242085738980487e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.658209800720215,
+      "logits/rejected": 7.952520847320557,
+      "logps/chosen": -40.68439865112305,
+      "logps/rejected": -42.43914794921875,
+      "loss": 0.0008,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -40.68439865112305,
+      "rewards/margins": 1.7547495365142822,
+      "rewards/rejected": -42.43914794921875,
       "step": 1950
     },
     {
       "epoch": 1.0463288175280148,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.309803159323349e-08,
       "learning_rate": 8.230213754324772e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.807183265686035,
+      "logits/rejected": 8.531861305236816,
+      "logps/chosen": -40.243247985839844,
+      "logps/rejected": -41.38704299926758,
+      "loss": 0.0,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -40.243247985839844,
+      "rewards/margins": 1.143794298171997,
+      "rewards/rejected": -41.38704299926758,
       "step": 1955
     },
     {
       "epoch": 1.0490048503094163,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.0880972689900786e-08,
       "learning_rate": 8.218310430121045e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.500699996948242,
+      "logits/rejected": 8.387083053588867,
+      "logps/chosen": -40.80499267578125,
+      "logps/rejected": -41.40040969848633,
+      "loss": 0.0,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -40.80499267578125,
+      "rewards/margins": 0.5954194664955139,
+      "rewards/rejected": -41.40040969848633,
       "step": 1960
     },
     {
       "epoch": 1.051680883090818,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.1538840832711535e-09,
       "learning_rate": 8.20637588185541e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.188100814819336,
+      "logits/rejected": 8.009719848632812,
+      "logps/chosen": -39.265132904052734,
+      "logps/rejected": -40.734710693359375,
+      "loss": 0.0,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -39.265132904052734,
+      "rewards/margins": 1.4695723056793213,
+      "rewards/rejected": -40.734710693359375,
       "step": 1965
     },
     {
       "epoch": 1.0543569158722195,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.799850130923686e-06,
       "learning_rate": 8.194410225316906e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.669782638549805,
+      "logits/rejected": 8.101269721984863,
+      "logps/chosen": -40.27177810668945,
+      "logps/rejected": -41.751136779785156,
+      "loss": 0.0,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -40.27177810668945,
+      "rewards/margins": 1.4793590307235718,
+      "rewards/rejected": -41.751136779785156,
       "step": 1970
     },
     {
       "epoch": 1.057032948653621,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 9.161146143830141e-07,
       "learning_rate": 8.182413576596385e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.45948314666748,
+      "logits/rejected": 8.418173789978027,
+      "logps/chosen": -40.649803161621094,
+      "logps/rejected": -42.114646911621094,
+      "loss": 0.0,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -40.649803161621094,
+      "rewards/margins": 1.4648405313491821,
+      "rewards/rejected": -42.114646911621094,
       "step": 1975
     },
     {
       "epoch": 1.0597089814350227,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.8078673772845872e-06,
       "learning_rate": 8.170386052085389e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.44881820678711,
+      "logits/rejected": 8.045083045959473,
+      "logps/chosen": -40.18886184692383,
+      "logps/rejected": -40.51119613647461,
+      "loss": 0.0,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": -40.18886184692383,
+      "rewards/margins": 0.32233747839927673,
+      "rewards/rejected": -40.51119613647461,
       "step": 1980
     },
     {
       "epoch": 1.0623850142164242,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.3394034110612003e-06,
       "learning_rate": 8.158327768475008e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.474889755249023,
+      "logits/rejected": 8.058609962463379,
+      "logps/chosen": -39.75669860839844,
+      "logps/rejected": -41.23186111450195,
+      "loss": 0.0,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -39.75669860839844,
+      "rewards/margins": 1.4751619100570679,
+      "rewards/rejected": -41.23186111450195,
       "step": 1985
     },
     {
       "epoch": 1.0650610469978257,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.6865062913200944e-09,
       "learning_rate": 8.146238842754767e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.739977836608887,
+      "logits/rejected": 8.270288467407227,
+      "logps/chosen": -40.680946350097656,
+      "logps/rejected": -41.545631408691406,
+      "loss": 0.0,
+      "rewards/accuracies": 0.48750001192092896,
+      "rewards/chosen": -40.680946350097656,
+      "rewards/margins": 0.8646896481513977,
+      "rewards/rejected": -41.545631408691406,
       "step": 1990
     },
     {
       "epoch": 1.0677370797792274,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 7.802688627737534e-05,
       "learning_rate": 8.134119392211476e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.695432662963867,
+      "logits/rejected": 8.185397148132324,
+      "logps/chosen": -41.02822494506836,
+      "logps/rejected": -41.93169021606445,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -41.02822494506836,
+      "rewards/margins": 0.9034714698791504,
+      "rewards/rejected": -41.93169021606445,
       "step": 1995
     },
     {
       "epoch": 1.0704131125606289,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.1757503810294671e-06,
       "learning_rate": 8.121969534428094e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.822378158569336,
+      "logits/rejected": 8.411437034606934,
+      "logps/chosen": -42.27022171020508,
+      "logps/rejected": -43.45711135864258,
+      "loss": 0.0,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -42.27022171020508,
+      "rewards/margins": 1.1868919134140015,
+      "rewards/rejected": -43.45711135864258,
       "step": 2000
     },
     {
       "epoch": 1.0704131125606289,
-      "eval_logits/chosen": NaN,
-      "eval_logits/rejected": NaN,
-      "eval_logps/chosen": NaN,
-      "eval_logps/rejected": NaN,
-      "eval_loss": NaN,
-      "eval_rewards/accuracies": 0.0,
-      "eval_rewards/chosen": NaN,
-      "eval_rewards/margins": NaN,
-      "eval_rewards/rejected": NaN,
-      "eval_runtime": 34.7555,
-      "eval_samples_per_second": 38.699,
-      "eval_semantic_entropy": NaN,
-      "eval_steps_per_second": 9.696,
+      "eval_logits/chosen": 7.2450642585754395,
+      "eval_logits/rejected": 7.007467746734619,
+      "eval_logps/chosen": -40.77992248535156,
+      "eval_logps/rejected": -42.01736831665039,
+      "eval_loss": 3.1178174708657025e-07,
+      "eval_rewards/accuracies": 0.5675074458122253,
+      "eval_rewards/chosen": -40.77992248535156,
+      "eval_rewards/margins": 1.2374484539031982,
+      "eval_rewards/rejected": -42.01736831665039,
+      "eval_runtime": 40.5107,
+      "eval_samples_per_second": 33.201,
+      "eval_steps_per_second": 8.319,
       "step": 2000
     },
     {
       "epoch": 1.0730891453420304,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.7941930189713664e-07,
       "learning_rate": 8.109789387282599e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.406800270080566,
+      "logits/rejected": 8.302990913391113,
+      "logps/chosen": -40.636985778808594,
+      "logps/rejected": -41.70927047729492,
+      "loss": 0.0,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -40.636985778808594,
+      "rewards/margins": 1.072289228439331,
+      "rewards/rejected": -41.70927047729492,
       "step": 2005
     },
     {
       "epoch": 1.075765178123432,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.101473190733529e-11,
       "learning_rate": 8.097579068946827e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.42392349243164,
+      "logits/rejected": 8.023496627807617,
+      "logps/chosen": -40.87938690185547,
+      "logps/rejected": -42.24049758911133,
+      "loss": 0.0,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -40.87938690185547,
+      "rewards/margins": 1.3611103296279907,
+      "rewards/rejected": -42.24049758911133,
       "step": 2010
     },
     {
       "epoch": 1.0784412109048336,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.5103260176475987e-09,
       "learning_rate": 8.085338697885344e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.387591361999512,
+      "logits/rejected": 8.063955307006836,
+      "logps/chosen": -41.28887176513672,
+      "logps/rejected": -42.1496696472168,
+      "loss": 0.0,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -41.28887176513672,
+      "rewards/margins": 0.8608001470565796,
+      "rewards/rejected": -42.1496696472168,
       "step": 2015
     },
     {
       "epoch": 1.081117243686235,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 9.366365376165455e-10,
       "learning_rate": 8.073068392854282e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.011270523071289,
+      "logits/rejected": 8.2319917678833,
+      "logps/chosen": -40.543949127197266,
+      "logps/rejected": -42.05414581298828,
+      "loss": 0.0,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -40.543949127197266,
+      "rewards/margins": 1.5102001428604126,
+      "rewards/rejected": -42.05414581298828,
       "step": 2020
     },
     {
       "epoch": 1.0837932764676368,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.7005169368649812e-05,
       "learning_rate": 8.060768272900193e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.145383834838867,
+      "logits/rejected": 8.520723342895508,
+      "logps/chosen": -42.4699821472168,
+      "logps/rejected": -43.37511444091797,
+      "loss": 0.0,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -42.4699821472168,
+      "rewards/margins": 0.9051315188407898,
+      "rewards/rejected": -43.37511444091797,
       "step": 2025
     },
     {
       "epoch": 1.0864693092490383,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.195097076140448e-05,
       "learning_rate": 8.0484384573589e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.266286849975586,
+      "logits/rejected": 8.424505233764648,
+      "logps/chosen": -40.704959869384766,
+      "logps/rejected": -41.65985870361328,
+      "loss": 0.0,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -40.704959869384766,
+      "rewards/margins": 0.9548954963684082,
+      "rewards/rejected": -41.65985870361328,
       "step": 2030
     },
     {
       "epoch": 1.0891453420304398,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 5.566721706753348e-05,
       "learning_rate": 8.03607906585432e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.620889663696289,
+      "logits/rejected": 8.355611801147461,
+      "logps/chosen": -41.4559211730957,
+      "logps/rejected": -43.61249923706055,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -41.4559211730957,
+      "rewards/margins": 2.1565775871276855,
+      "rewards/rejected": -43.61249923706055,
       "step": 2035
     },
     {
       "epoch": 1.0918213748118415,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.321497541052285e-06,
       "learning_rate": 8.023690218297329e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.547624588012695,
+      "logits/rejected": 8.592642784118652,
+      "logps/chosen": -41.11884689331055,
+      "logps/rejected": -42.3089714050293,
+      "loss": 0.0,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -41.11884689331055,
+      "rewards/margins": 1.1901240348815918,
+      "rewards/rejected": -42.3089714050293,
       "step": 2040
     },
     {
       "epoch": 1.094497407593243,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.6675678914805667e-07,
       "learning_rate": 8.01127203488458e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.132123947143555,
+      "logits/rejected": 8.042007446289062,
+      "logps/chosen": -39.87526321411133,
+      "logps/rejected": -40.990150451660156,
+      "loss": 0.0,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": -39.87526321411133,
+      "rewards/margins": 1.1148872375488281,
+      "rewards/rejected": -40.990150451660156,
       "step": 2045
     },
     {
       "epoch": 1.0971734403746445,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.163976797291173e-05,
       "learning_rate": 7.998824636097339e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.986612319946289,
+      "logits/rejected": 8.556442260742188,
+      "logps/chosen": -40.445838928222656,
+      "logps/rejected": -42.195899963378906,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -40.445838928222656,
+      "rewards/margins": 1.7500604391098022,
+      "rewards/rejected": -42.195899963378906,
       "step": 2050
     },
     {
       "epoch": 1.0998494731560462,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.624883824363688e-09,
       "learning_rate": 7.986348142700328e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.554445266723633,
+      "logits/rejected": 8.277101516723633,
+      "logps/chosen": -40.31126403808594,
+      "logps/rejected": -42.23910140991211,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -40.31126403808594,
+      "rewards/margins": 1.9278380870819092,
+      "rewards/rejected": -42.23910140991211,
       "step": 2055
     },
     {
       "epoch": 1.1025255059374477,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.546883033987733e-07,
       "learning_rate": 7.973842675740539e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.306285858154297,
+      "logits/rejected": 8.320329666137695,
+      "logps/chosen": -41.140647888183594,
+      "logps/rejected": -41.80480194091797,
+      "loss": 0.0,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -41.140647888183594,
+      "rewards/margins": 0.664159893989563,
+      "rewards/rejected": -41.80480194091797,
       "step": 2060
     },
     {
       "epoch": 1.1052015387188494,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.034807895831652e-05,
       "learning_rate": 7.961308356546066e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.44710922241211,
+      "logits/rejected": 8.226879119873047,
+      "logps/chosen": -40.02400588989258,
+      "logps/rejected": -41.54513168334961,
+      "loss": 0.0,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -40.02400588989258,
+      "rewards/margins": 1.5211279392242432,
+      "rewards/rejected": -41.54513168334961,
       "step": 2065
     },
     {
       "epoch": 1.107877571500251,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.578075163497712e-07,
       "learning_rate": 7.948745306724931e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.627179145812988,
+      "logits/rejected": 8.316874504089355,
+      "logps/chosen": -41.3073844909668,
+      "logps/rejected": -42.705055236816406,
+      "loss": 0.0,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -41.3073844909668,
+      "rewards/margins": 1.3976645469665527,
+      "rewards/rejected": -42.705055236816406,
       "step": 2070
     },
     {
       "epoch": 1.1105536042816524,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.0250644810068454e-09,
       "learning_rate": 7.936153648163897e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.710594177246094,
+      "logits/rejected": 8.54148006439209,
+      "logps/chosen": -40.856868743896484,
+      "logps/rejected": -42.92464828491211,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -40.856868743896484,
+      "rewards/margins": 2.067783832550049,
+      "rewards/rejected": -42.92464828491211,
       "step": 2075
     },
     {
       "epoch": 1.1132296370630541,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 6.890262997631695e-06,
       "learning_rate": 7.92353350302729e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.384446144104004,
+      "logits/rejected": 8.116195678710938,
+      "logps/chosen": -39.415672302246094,
+      "logps/rejected": -40.97673416137695,
+      "loss": 0.0,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -39.415672302246094,
+      "rewards/margins": 1.5610568523406982,
+      "rewards/rejected": -40.97673416137695,
       "step": 2080
     },
     {
       "epoch": 1.1159056698444556,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.159244045559575e-07,
       "learning_rate": 7.910884993755816e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.64844036102295,
+      "logits/rejected": 8.4146089553833,
+      "logps/chosen": -40.42133712768555,
+      "logps/rejected": -42.17142105102539,
+      "loss": 0.0,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -40.42133712768555,
+      "rewards/margins": 1.7500842809677124,
+      "rewards/rejected": -42.17142105102539,
       "step": 2085
     },
     {
       "epoch": 1.118581702625857,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.8231731613935785e-08,
       "learning_rate": 7.898208243065367e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.658584594726562,
+      "logits/rejected": 8.730806350708008,
+      "logps/chosen": -41.568092346191406,
+      "logps/rejected": -41.944313049316406,
+      "loss": 0.0001,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -41.568092346191406,
+      "rewards/margins": 0.37622079253196716,
+      "rewards/rejected": -41.944313049316406,
       "step": 2090
     },
     {
       "epoch": 1.1212577354072588,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.288588814136926e-08,
       "learning_rate": 7.88550337394583e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.935137748718262,
+      "logits/rejected": 8.554451942443848,
+      "logps/chosen": -40.218360900878906,
+      "logps/rejected": -41.84646224975586,
+      "loss": 0.0,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -40.218360900878906,
+      "rewards/margins": 1.6281015872955322,
+      "rewards/rejected": -41.84646224975586,
       "step": 2095
     },
     {
       "epoch": 1.1239337681886603,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.438020051242254e-07,
       "learning_rate": 7.872770509659905e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.250994682312012,
+      "logits/rejected": 8.14515495300293,
+      "logps/chosen": -39.939308166503906,
+      "logps/rejected": -41.26539611816406,
+      "loss": 0.0,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -39.939308166503906,
+      "rewards/margins": 1.3260862827301025,
+      "rewards/rejected": -41.26539611816406,
       "step": 2100
     },
     {
       "epoch": 1.1266098009700618,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.848211855951944e-05,
       "learning_rate": 7.860009773741896e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.618029594421387,
+      "logits/rejected": 8.181163787841797,
+      "logps/chosen": -40.06212615966797,
+      "logps/rejected": -42.03260040283203,
+      "loss": 0.0,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -40.06212615966797,
+      "rewards/margins": 1.9704763889312744,
+      "rewards/rejected": -42.03260040283203,
       "step": 2105
     },
     {
       "epoch": 1.1292858337514635,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.4497593483416872e-06,
       "learning_rate": 7.84722128999652e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.6336088180542,
+      "logits/rejected": 8.369260787963867,
+      "logps/chosen": -40.98212432861328,
+      "logps/rejected": -42.2159309387207,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -40.98212432861328,
+      "rewards/margins": 1.233810544013977,
+      "rewards/rejected": -42.2159309387207,
       "step": 2110
     },
     {
       "epoch": 1.131961866532865,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 0.0012040571520709415,
       "learning_rate": 7.834405182497699e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.384027481079102,
+      "logits/rejected": 8.253840446472168,
+      "logps/chosen": -40.520225524902344,
+      "logps/rejected": -41.95452117919922,
+      "loss": 0.0,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": -40.520225524902344,
+      "rewards/margins": 1.4342975616455078,
+      "rewards/rejected": -41.95452117919922,
       "step": 2115
     },
     {
       "epoch": 1.1346378993142665,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.745515723674222e-07,
       "learning_rate": 7.821561575587368e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.594862937927246,
+      "logits/rejected": 8.447153091430664,
+      "logps/chosen": -40.040931701660156,
+      "logps/rejected": -41.00267791748047,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -40.040931701660156,
+      "rewards/margins": 0.9617475271224976,
+      "rewards/rejected": -41.00267791748047,
       "step": 2120
     },
     {
       "epoch": 1.1373139320956682,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.0375257881718297e-07,
       "learning_rate": 7.808690593874254e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.701242446899414,
+      "logits/rejected": 8.27269172668457,
+      "logps/chosen": -40.88854217529297,
+      "logps/rejected": -41.848472595214844,
+      "loss": 0.0018,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": -40.88854217529297,
+      "rewards/margins": 0.9599350690841675,
+      "rewards/rejected": -41.848472595214844,
       "step": 2125
     },
     {
       "epoch": 1.1399899648770697,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 0.0002448884334940336,
       "learning_rate": 7.79579236223268e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.545909881591797,
+      "logits/rejected": 7.890882968902588,
+      "logps/chosen": -39.64358901977539,
+      "logps/rejected": -42.72437286376953,
+      "loss": 0.0,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -39.64358901977539,
+      "rewards/margins": 3.0807852745056152,
+      "rewards/rejected": -42.72437286376953,
       "step": 2130
     },
     {
       "epoch": 1.1426659976584714,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 6.367315786181002e-07,
       "learning_rate": 7.782867005801346e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.717951774597168,
+      "logits/rejected": 8.202997207641602,
+      "logps/chosen": -39.86112976074219,
+      "logps/rejected": -42.386024475097656,
+      "loss": 0.0,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -39.86112976074219,
+      "rewards/margins": 2.5248966217041016,
+      "rewards/rejected": -42.386024475097656,
       "step": 2135
     },
     {
       "epoch": 1.145342030439873,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 7.71324902428603e-07,
       "learning_rate": 7.769914649982117e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.58655071258545,
+      "logits/rejected": 8.170052528381348,
+      "logps/chosen": -40.371742248535156,
+      "logps/rejected": -42.495365142822266,
+      "loss": 0.0,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -40.371742248535156,
+      "rewards/margins": 2.1236205101013184,
+      "rewards/rejected": -42.495365142822266,
       "step": 2140
     },
     {
       "epoch": 1.1480180632212744,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 8.93364803440133e-09,
       "learning_rate": 7.756935420438803e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.457643508911133,
+      "logits/rejected": 8.24944019317627,
+      "logps/chosen": -40.57135772705078,
+      "logps/rejected": -42.175621032714844,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -40.57135772705078,
+      "rewards/margins": 1.604262113571167,
+      "rewards/rejected": -42.175621032714844,
       "step": 2145
     },
     {
       "epoch": 1.1506940960026761,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 8.526416672830342e-05,
       "learning_rate": 7.743929443095951e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.606661796569824,
+      "logits/rejected": 8.478116989135742,
+      "logps/chosen": -41.13886260986328,
+      "logps/rejected": -42.28718948364258,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -41.13886260986328,
+      "rewards/margins": 1.1483221054077148,
+      "rewards/rejected": -42.28718948364258,
       "step": 2150
     },
     {
       "epoch": 1.1533701287840776,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.4868601065516354e-05,
       "learning_rate": 7.730896844137609e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.28092098236084,
+      "logits/rejected": 8.127330780029297,
+      "logps/chosen": -40.70257568359375,
+      "logps/rejected": -41.38616180419922,
+      "loss": 0.0,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -40.70257568359375,
+      "rewards/margins": 0.6835864186286926,
+      "rewards/rejected": -41.38616180419922,
       "step": 2155
     },
     {
       "epoch": 1.1560461615654791,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.517581487503091e-08,
       "learning_rate": 7.717837750006106e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.886369705200195,
+      "logits/rejected": 8.697736740112305,
+      "logps/chosen": -41.994773864746094,
+      "logps/rejected": -42.7291374206543,
+      "loss": 0.0,
+      "rewards/accuracies": 0.4937500059604645,
+      "rewards/chosen": -41.994773864746094,
+      "rewards/margins": 0.7343606948852539,
+      "rewards/rejected": -42.7291374206543,
       "step": 2160
     },
     {
       "epoch": 1.1587221943468808,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.683699769021294e-08,
       "learning_rate": 7.704752287400832e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.586957931518555,
+      "logits/rejected": 8.184276580810547,
+      "logps/chosen": -41.64616775512695,
+      "logps/rejected": -42.28727722167969,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -41.64616775512695,
+      "rewards/margins": 0.6411095857620239,
+      "rewards/rejected": -42.28727722167969,
       "step": 2165
     },
     {
       "epoch": 1.1613982271282823,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 7.520309082628661e-07,
       "learning_rate": 7.691640583277004e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.825390815734863,
+      "logits/rejected": 8.263834953308105,
+      "logps/chosen": -41.467002868652344,
+      "logps/rejected": -43.19124984741211,
+      "loss": 0.0,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -41.467002868652344,
+      "rewards/margins": 1.7242462635040283,
+      "rewards/rejected": -43.19124984741211,
       "step": 2170
     },
     {
       "epoch": 1.1640742599096838,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.507646332158615e-10,
       "learning_rate": 7.678502764844433e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.964905738830566,
+      "logits/rejected": 8.28453254699707,
+      "logps/chosen": -41.94441604614258,
+      "logps/rejected": -42.04668045043945,
+      "loss": 0.0,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": -41.94441604614258,
+      "rewards/margins": 0.10226273536682129,
+      "rewards/rejected": -42.04668045043945,
       "step": 2175
     },
     {
       "epoch": 1.1667502926910855,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 5.019293275196284e-06,
       "learning_rate": 7.665338959566288e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.768538475036621,
+      "logits/rejected": 8.602264404296875,
+      "logps/chosen": -41.781005859375,
+      "logps/rejected": -42.875083923339844,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -41.781005859375,
+      "rewards/margins": 1.0940748453140259,
+      "rewards/rejected": -42.875083923339844,
       "step": 2180
     },
     {
       "epoch": 1.169426325472487,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.1070368520028168e-06,
       "learning_rate": 7.652149295157868e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.601483345031738,
+      "logits/rejected": 8.105417251586914,
+      "logps/chosen": -41.35640335083008,
+      "logps/rejected": -41.93348693847656,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -41.35640335083008,
+      "rewards/margins": 0.57708340883255,
+      "rewards/rejected": -41.93348693847656,
       "step": 2185
     },
     {
       "epoch": 1.1721023582538885,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 0.00048313440939491784,
       "learning_rate": 7.638933899585354e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.284440994262695,
+      "logits/rejected": 8.253382682800293,
+      "logps/chosen": -40.96122360229492,
+      "logps/rejected": -41.88115310668945,
+      "loss": 0.0,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -40.96122360229492,
+      "rewards/margins": 0.9199298620223999,
+      "rewards/rejected": -41.88115310668945,
       "step": 2190
     },
     {
       "epoch": 1.1747783910352902,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 0.041338124195654795,
       "learning_rate": 7.625692901064573e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.199808120727539,
+      "logits/rejected": 8.141292572021484,
+      "logps/chosen": -40.34364700317383,
+      "logps/rejected": -41.13231658935547,
+      "loss": 0.0001,
+      "rewards/accuracies": 0.5062500238418579,
+      "rewards/chosen": -40.34364700317383,
+      "rewards/margins": 0.7886745929718018,
+      "rewards/rejected": -41.13231658935547,
       "step": 2195
     },
     {
       "epoch": 1.1774544238166917,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.2096897115767505e-07,
       "learning_rate": 7.61242642805975e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.618353843688965,
+      "logits/rejected": 8.716090202331543,
+      "logps/chosen": -41.43954086303711,
+      "logps/rejected": -42.253536224365234,
+      "loss": 0.0,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": -41.43954086303711,
+      "rewards/margins": 0.8139970898628235,
+      "rewards/rejected": -42.253536224365234,
       "step": 2200
     },
     {
       "epoch": 1.1801304565980932,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 7.038101772689373e-08,
       "learning_rate": 7.599134609282266e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.902692794799805,
+      "logits/rejected": 8.209165573120117,
+      "logps/chosen": -41.19422149658203,
+      "logps/rejected": -42.89899444580078,
+      "loss": 0.0,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -41.19422149658203,
+      "rewards/margins": 1.7047698497772217,
+      "rewards/rejected": -42.89899444580078,
       "step": 2205
     },
     {
       "epoch": 1.182806489379495,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 5.368342850833801e-09,
       "learning_rate": 7.585817573689402e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.607969284057617,
+      "logits/rejected": 8.46239185333252,
+      "logps/chosen": -40.59905242919922,
+      "logps/rejected": -42.151268005371094,
+      "loss": 0.0,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -40.59905242919922,
+      "rewards/margins": 1.5522193908691406,
+      "rewards/rejected": -42.151268005371094,
       "step": 2210
     },
     {
       "epoch": 1.1854825221608964,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 5.233976859275776e-08,
       "learning_rate": 7.572475450483098e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.511218070983887,
+      "logits/rejected": 8.185332298278809,
+      "logps/chosen": -39.96785354614258,
+      "logps/rejected": -41.31941604614258,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -39.96785354614258,
+      "rewards/margins": 1.3515615463256836,
+      "rewards/rejected": -41.31941604614258,
       "step": 2215
     },
     {
       "epoch": 1.188158554942298,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.6535416025913438e-06,
       "learning_rate": 7.559108369108689e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.501649856567383,
+      "logits/rejected": 8.205302238464355,
+      "logps/chosen": -39.40612030029297,
+      "logps/rejected": -40.170108795166016,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -39.40612030029297,
+      "rewards/margins": 0.7639927864074707,
+      "rewards/rejected": -40.170108795166016,
       "step": 2220
     },
     {
       "epoch": 1.1908345877236997,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.076636675531797e-07,
       "learning_rate": 7.54571645925366e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.915776252746582,
+      "logits/rejected": 8.282099723815918,
+      "logps/chosen": -39.499900817871094,
+      "logps/rejected": -42.26732635498047,
+      "loss": 0.0,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -39.499900817871094,
+      "rewards/margins": 2.7674288749694824,
+      "rewards/rejected": -42.26732635498047,
       "step": 2225
     },
     {
       "epoch": 1.1935106205051011,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.770966811824327e-08,
       "learning_rate": 7.532299850846378e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.668721199035645,
+      "logits/rejected": 8.15053653717041,
+      "logps/chosen": -40.732810974121094,
+      "logps/rejected": -42.174766540527344,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -40.732810974121094,
+      "rewards/margins": 1.441957712173462,
+      "rewards/rejected": -42.174766540527344,
       "step": 2230
     },
     {
       "epoch": 1.1961866532865026,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.1293539338067661e-07,
       "learning_rate": 7.518858674054838e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.647680282592773,
+      "logits/rejected": 8.115179061889648,
+      "logps/chosen": -40.324615478515625,
+      "logps/rejected": -41.052040100097656,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -40.324615478515625,
+      "rewards/margins": 0.7274282574653625,
+      "rewards/rejected": -41.052040100097656,
       "step": 2235
     },
     {
       "epoch": 1.1988626860679044,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.774380533255003e-10,
       "learning_rate": 7.505393059285394e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.428884506225586,
+      "logits/rejected": 8.079434394836426,
+      "logps/chosen": -40.62153625488281,
+      "logps/rejected": -41.99386978149414,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -40.62153625488281,
+      "rewards/margins": 1.3723335266113281,
+      "rewards/rejected": -41.99386978149414,
       "step": 2240
     },
     {
       "epoch": 1.2015387188493059,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 7.881354785945644e-06,
       "learning_rate": 7.491903137181501e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.486298561096191,
+      "logits/rejected": 8.506063461303711,
+      "logps/chosen": -40.976165771484375,
+      "logps/rejected": -42.27082443237305,
+      "loss": 0.0,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -40.976165771484375,
+      "rewards/margins": 1.2946627140045166,
+      "rewards/rejected": -42.27082443237305,
       "step": 2245
     },
     {
       "epoch": 1.2042147516307076,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.513369944743648e-07,
       "learning_rate": 7.478389038622441e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.356260299682617,
+      "logits/rejected": 8.329123497009277,
+      "logps/chosen": -40.45479202270508,
+      "logps/rejected": -41.39447784423828,
+      "loss": 0.0,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -40.45479202270508,
+      "rewards/margins": 0.9396904110908508,
+      "rewards/rejected": -41.39447784423828,
       "step": 2250
     },
     {
       "epoch": 1.206890784412109,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 0.0005368014391819825,
       "learning_rate": 7.46485089472206e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.505821228027344,
+      "logits/rejected": 8.476261138916016,
+      "logps/chosen": -41.28764724731445,
+      "logps/rejected": -42.163978576660156,
+      "loss": 0.0,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -41.28764724731445,
+      "rewards/margins": 0.8763296008110046,
+      "rewards/rejected": -42.163978576660156,
       "step": 2255
     },
     {
       "epoch": 1.2095668171935106,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 0.00011978943869628825,
       "learning_rate": 7.451288836827487e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.45439624786377,
+      "logits/rejected": 8.619742393493652,
+      "logps/chosen": -41.024810791015625,
+      "logps/rejected": -41.434513092041016,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -41.024810791015625,
+      "rewards/margins": 0.40970200300216675,
+      "rewards/rejected": -41.434513092041016,
       "step": 2260
     },
     {
       "epoch": 1.2122428499749123,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.758227806494229e-10,
       "learning_rate": 7.437702996517869e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.706487655639648,
+      "logits/rejected": 8.562585830688477,
+      "logps/chosen": -39.77735137939453,
+      "logps/rejected": -41.421119689941406,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -39.77735137939453,
+      "rewards/margins": 1.6437727212905884,
+      "rewards/rejected": -41.421119689941406,
       "step": 2265
     },
     {
       "epoch": 1.2149188827563138,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.1688229226088774e-07,
       "learning_rate": 7.424093505603087e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.765771865844727,
+      "logits/rejected": 8.405631065368652,
+      "logps/chosen": -39.97887420654297,
+      "logps/rejected": -41.925376892089844,
+      "loss": 0.0,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -39.97887420654297,
+      "rewards/margins": 1.9465036392211914,
+      "rewards/rejected": -41.925376892089844,
       "step": 2270
     },
     {
       "epoch": 1.2175949155377153,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.902147326128656e-06,
       "learning_rate": 7.410460496122482e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.554079055786133,
+      "logits/rejected": 8.12889575958252,
+      "logps/chosen": -41.50530242919922,
+      "logps/rejected": -42.36071014404297,
+      "loss": 0.0,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -41.50530242919922,
+      "rewards/margins": 0.8554102182388306,
+      "rewards/rejected": -42.36071014404297,
       "step": 2275
     },
     {
       "epoch": 1.220270948319117,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 0.00017505075350152178,
       "learning_rate": 7.396804100343572e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.910680770874023,
+      "logits/rejected": 8.220748901367188,
+      "logps/chosen": -40.28874588012695,
+      "logps/rejected": -41.78064727783203,
+      "loss": 0.0,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -40.28874588012695,
+      "rewards/margins": 1.4918994903564453,
+      "rewards/rejected": -41.78064727783203,
       "step": 2280
     },
     {
       "epoch": 1.2229469811005185,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 0.00021053339995770096,
       "learning_rate": 7.383124450760768e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.962723731994629,
+      "logits/rejected": 8.456936836242676,
+      "logps/chosen": -40.37126922607422,
+      "logps/rejected": -42.131126403808594,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -40.37126922607422,
+      "rewards/margins": 1.7598581314086914,
+      "rewards/rejected": -42.131126403808594,
       "step": 2285
     },
     {
       "epoch": 1.22562301388192,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.1889449556908217e-10,
       "learning_rate": 7.369421680094091e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.063736915588379,
+      "logits/rejected": 8.632521629333496,
+      "logps/chosen": -40.83104705810547,
+      "logps/rejected": -42.700775146484375,
+      "loss": 0.0,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -40.83104705810547,
+      "rewards/margins": 1.8697303533554077,
+      "rewards/rejected": -42.700775146484375,
       "step": 2290
     },
     {
       "epoch": 1.2282990466633217,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.806326530729962e-05,
       "learning_rate": 7.355695921287881e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.770237922668457,
+      "logits/rejected": 8.363417625427246,
+      "logps/chosen": -40.73759078979492,
+      "logps/rejected": -41.025428771972656,
+      "loss": 0.0,
+      "rewards/accuracies": 0.4937500059604645,
+      "rewards/chosen": -40.73759078979492,
+      "rewards/margins": 0.28783732652664185,
+      "rewards/rejected": -41.025428771972656,
       "step": 2295
     },
     {
       "epoch": 1.2309750794447232,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 8.513334799377988e-08,
       "learning_rate": 7.341947307509513e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.68481731414795,
+      "logits/rejected": 8.428956985473633,
+      "logps/chosen": -40.888038635253906,
+      "logps/rejected": -42.52493667602539,
+      "loss": 0.0,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -40.888038635253906,
+      "rewards/margins": 1.6368995904922485,
+      "rewards/rejected": -42.52493667602539,
       "step": 2300
     },
     {
       "epoch": 1.233651112226125,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.824512679842653e-10,
       "learning_rate": 7.328175972148094e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.70347785949707,
+      "logits/rejected": 8.293722152709961,
+      "logps/chosen": -40.107975006103516,
+      "logps/rejected": -42.11811447143555,
+      "loss": 0.0,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -40.107975006103516,
+      "rewards/margins": 2.0101377964019775,
+      "rewards/rejected": -42.11811447143555,
       "step": 2305
     },
     {
       "epoch": 1.2363271450075264,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.017785882524667e-10,
       "learning_rate": 7.314382048813185e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.495558738708496,
+      "logits/rejected": 7.869775295257568,
+      "logps/chosen": -40.17514419555664,
+      "logps/rejected": -43.199771881103516,
+      "loss": 0.0003,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -40.17514419555664,
+      "rewards/margins": 3.024625062942505,
+      "rewards/rejected": -43.199771881103516,
       "step": 2310
     },
     {
       "epoch": 1.2390031777889279,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.198501928119521e-08,
       "learning_rate": 7.300565671333486e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.782133102416992,
+      "logits/rejected": 8.277894973754883,
+      "logps/chosen": -40.93571853637695,
+      "logps/rejected": -42.298927307128906,
+      "loss": 0.0,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -40.93571853637695,
+      "rewards/margins": 1.3632045984268188,
+      "rewards/rejected": -42.298927307128906,
       "step": 2315
     },
     {
       "epoch": 1.2416792105703296,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.6340808748596537e-05,
       "learning_rate": 7.286726973755554e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.759393692016602,
+      "logits/rejected": 8.505353927612305,
+      "logps/chosen": -41.74432373046875,
+      "logps/rejected": -41.89595413208008,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -41.74432373046875,
+      "rewards/margins": 0.1516321450471878,
+      "rewards/rejected": -41.89595413208008,
       "step": 2320
     },
     {
       "epoch": 1.244355243351731,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.483248990420863e-05,
       "learning_rate": 7.272866090342493e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.273099899291992,
+      "logits/rejected": 8.017354965209961,
+      "logps/chosen": -39.90392303466797,
+      "logps/rejected": -41.7261848449707,
+      "loss": 0.0,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -39.90392303466797,
+      "rewards/margins": 1.8222602605819702,
+      "rewards/rejected": -41.7261848449707,
       "step": 2325
     },
     {
       "epoch": 1.2470312761331326,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 5.264857678984263e-07,
       "learning_rate": 7.258983155572656e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.998651504516602,
+      "logits/rejected": 8.72075366973877,
+      "logps/chosen": -42.4549560546875,
+      "logps/rejected": -42.265193939208984,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -42.4549560546875,
+      "rewards/margins": -0.18975695967674255,
+      "rewards/rejected": -42.265193939208984,
       "step": 2330
     },
     {
       "epoch": 1.2497073089145343,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 7.337966513512116e-05,
       "learning_rate": 7.245078304138335e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.475242614746094,
+      "logits/rejected": 8.307760238647461,
+      "logps/chosen": -40.506065368652344,
+      "logps/rejected": -41.600563049316406,
+      "loss": 0.0,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -40.506065368652344,
+      "rewards/margins": 1.0944931507110596,
+      "rewards/rejected": -41.600563049316406,
       "step": 2335
     },
     {
       "epoch": 1.2523833416959358,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.5507335338129486e-09,
       "learning_rate": 7.231151670944462e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.902246475219727,
+      "logits/rejected": 8.067272186279297,
+      "logps/chosen": -41.660545349121094,
+      "logps/rejected": -42.258522033691406,
+      "loss": 0.0,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -41.660545349121094,
+      "rewards/margins": 0.597974419593811,
+      "rewards/rejected": -42.258522033691406,
       "step": 2340
     },
     {
       "epoch": 1.2550593744773373,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.189400627470942e-05,
       "learning_rate": 7.217203391107291e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.499944686889648,
+      "logits/rejected": 8.12531852722168,
+      "logps/chosen": -41.47569274902344,
+      "logps/rejected": -42.175228118896484,
+      "loss": 0.0,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -41.47569274902344,
+      "rewards/margins": 0.6995365619659424,
+      "rewards/rejected": -42.175228118896484,
       "step": 2345
     },
     {
       "epoch": 1.257735407258739,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 0.29839958899710833,
       "learning_rate": 7.203233599953096e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.346668243408203,
+      "logits/rejected": 8.014691352844238,
+      "logps/chosen": -39.29924774169922,
+      "logps/rejected": -40.19755935668945,
+      "loss": 0.0006,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -39.29924774169922,
+      "rewards/margins": 0.8983109593391418,
+      "rewards/rejected": -40.19755935668945,
       "step": 2350
     },
     {
       "epoch": 1.2604114400401405,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 5.266689759473824e-05,
       "learning_rate": 7.189242433016852e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.41518497467041,
+      "logits/rejected": 8.157347679138184,
+      "logps/chosen": -41.577552795410156,
+      "logps/rejected": -43.21418762207031,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -41.577552795410156,
+      "rewards/margins": 1.6366382837295532,
+      "rewards/rejected": -43.21418762207031,
       "step": 2355
     },
     {
       "epoch": 1.263087472821542,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.7561907247547575e-11,
       "learning_rate": 7.17523002604092e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.851011276245117,
+      "logits/rejected": 8.375604629516602,
+      "logps/chosen": -41.504669189453125,
+      "logps/rejected": -42.88933181762695,
+      "loss": 0.0,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -41.504669189453125,
+      "rewards/margins": 1.3846628665924072,
+      "rewards/rejected": -42.88933181762695,
       "step": 2360
     },
     {
       "epoch": 1.2657635056029437,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.218024672871632e-07,
       "learning_rate": 7.161196514973734e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.617021560668945,
+      "logits/rejected": 8.004721641540527,
+      "logps/chosen": -40.308563232421875,
+      "logps/rejected": -42.25050354003906,
+      "loss": 0.0017,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -40.308563232421875,
+      "rewards/margins": 1.941937804222107,
+      "rewards/rejected": -42.25050354003906,
       "step": 2365
     },
     {
       "epoch": 1.2684395383843452,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.037749140875123e-07,
       "learning_rate": 7.147142035968483e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.663287162780762,
+      "logits/rejected": 8.161401748657227,
+      "logps/chosen": -40.75321960449219,
+      "logps/rejected": -42.39347457885742,
+      "loss": 0.0,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -40.75321960449219,
+      "rewards/margins": 1.6402534246444702,
+      "rewards/rejected": -42.39347457885742,
       "step": 2370
     },
     {
       "epoch": 1.2711155711657467,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 6.167313201711084e-06,
       "learning_rate": 7.133066725381781e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.902474403381348,
+      "logits/rejected": 8.259099006652832,
+      "logps/chosen": -41.818199157714844,
+      "logps/rejected": -42.90471649169922,
+      "loss": 0.0,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -41.818199157714844,
+      "rewards/margins": 1.0865098237991333,
+      "rewards/rejected": -42.90471649169922,
       "step": 2375
     },
     {
       "epoch": 1.2737916039471484,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 9.884818779452493e-08,
       "learning_rate": 7.118970719772354e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.613125801086426,
+      "logits/rejected": 8.22067928314209,
+      "logps/chosen": -40.86578369140625,
+      "logps/rejected": -42.80434799194336,
+      "loss": 0.0,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -40.86578369140625,
+      "rewards/margins": 1.9385671615600586,
+      "rewards/rejected": -42.80434799194336,
       "step": 2380
     },
     {
       "epoch": 1.27646763672855,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 0.0002653684648249868,
       "learning_rate": 7.104854155899711e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.731677055358887,
+      "logits/rejected": 8.44113540649414,
+      "logps/chosen": -42.01833724975586,
+      "logps/rejected": -42.770538330078125,
+      "loss": 0.0,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": -42.01833724975586,
+      "rewards/margins": 0.7522023916244507,
+      "rewards/rejected": -42.770538330078125,
       "step": 2385
     },
     {
       "epoch": 1.2791436695099514,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.4042740382718883e-11,
       "learning_rate": 7.090717170722817e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.50440502166748,
+      "logits/rejected": 8.578502655029297,
+      "logps/chosen": -42.388492584228516,
+      "logps/rejected": -43.734596252441406,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -42.388492584228516,
+      "rewards/margins": 1.346106767654419,
+      "rewards/rejected": -43.734596252441406,
       "step": 2390
     },
     {
       "epoch": 1.2818197022913531,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 9.263269870484311e-06,
       "learning_rate": 7.076559901398762e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.335229873657227,
+      "logits/rejected": 8.712800979614258,
+      "logps/chosen": -40.85382080078125,
+      "logps/rejected": -43.03199005126953,
+      "loss": 0.0,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -40.85382080078125,
+      "rewards/margins": 2.1781651973724365,
+      "rewards/rejected": -43.03199005126953,
       "step": 2395
     },
     {
       "epoch": 1.2844957350727546,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.5843446801085798e-05,
       "learning_rate": 7.062382485281436e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.836427688598633,
+      "logits/rejected": 8.298908233642578,
+      "logps/chosen": -40.521949768066406,
+      "logps/rejected": -41.85470199584961,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -40.521949768066406,
+      "rewards/margins": 1.332749605178833,
+      "rewards/rejected": -41.85470199584961,
       "step": 2400
     },
     {
       "epoch": 1.2844957350727546,
-      "eval_logits/chosen": NaN,
-      "eval_logits/rejected": NaN,
-      "eval_logps/chosen": NaN,
-      "eval_logps/rejected": NaN,
-      "eval_loss": NaN,
-      "eval_rewards/accuracies": 0.0,
-      "eval_rewards/chosen": NaN,
-      "eval_rewards/margins": NaN,
-      "eval_rewards/rejected": NaN,
-      "eval_runtime": 35.1218,
-      "eval_samples_per_second": 38.295,
-      "eval_semantic_entropy": NaN,
-      "eval_steps_per_second": 9.595,
+      "eval_logits/chosen": 7.18833589553833,
+      "eval_logits/rejected": 6.942528247833252,
+      "eval_logps/chosen": -40.9809455871582,
+      "eval_logps/rejected": -42.20896530151367,
+      "eval_loss": 3.199867535386147e-07,
+      "eval_rewards/accuracies": 0.5645400881767273,
+      "eval_rewards/chosen": -40.9809455871582,
+      "eval_rewards/margins": 1.2280218601226807,
+      "eval_rewards/rejected": -42.20896530151367,
+      "eval_runtime": 40.518,
+      "eval_samples_per_second": 33.195,
+      "eval_steps_per_second": 8.317,
       "step": 2400
     },
     {
       "epoch": 1.287171767854156,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.571179791147905e-10,
       "learning_rate": 7.048185059920193e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.714685440063477,
+      "logits/rejected": 8.403592109680176,
+      "logps/chosen": -41.647098541259766,
+      "logps/rejected": -43.28430938720703,
+      "loss": 0.0,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -41.647098541259766,
+      "rewards/margins": 1.6372133493423462,
+      "rewards/rejected": -43.28430938720703,
       "step": 2405
     },
     {
       "epoch": 1.2898478006355578,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.0588356424467986e-07,
       "learning_rate": 7.033967763058516e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.547478675842285,
+      "logits/rejected": 8.087888717651367,
+      "logps/chosen": -40.34328079223633,
+      "logps/rejected": -41.562259674072266,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -40.34328079223633,
+      "rewards/margins": 1.2189751863479614,
+      "rewards/rejected": -41.562259674072266,
       "step": 2410
     },
     {
       "epoch": 1.2925238334169593,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.7050150197833705e-11,
       "learning_rate": 7.019730732632681e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.655824661254883,
+      "logits/rejected": 8.174910545349121,
+      "logps/chosen": -41.71727752685547,
+      "logps/rejected": -42.924339294433594,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -41.71727752685547,
+      "rewards/margins": 1.2070648670196533,
+      "rewards/rejected": -42.924339294433594,
       "step": 2415
     },
     {
       "epoch": 1.2951998661983608,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 8.798761144920488e-09,
       "learning_rate": 7.005474106770418e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.676004409790039,
+      "logits/rejected": 8.410015106201172,
+      "logps/chosen": -41.435325622558594,
+      "logps/rejected": -42.68471145629883,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -41.435325622558594,
+      "rewards/margins": 1.2493826150894165,
+      "rewards/rejected": -42.68471145629883,
       "step": 2420
     },
     {
       "epoch": 1.2978758989797625,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 6.038303646700376e-08,
       "learning_rate": 6.991198023789577e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.771553039550781,
+      "logits/rejected": 8.683374404907227,
+      "logps/chosen": -42.4216423034668,
+      "logps/rejected": -42.75839614868164,
+      "loss": 0.0,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": -42.4216423034668,
+      "rewards/margins": 0.3367560803890228,
+      "rewards/rejected": -42.75839614868164,
       "step": 2425
     },
     {
       "epoch": 1.300551931761164,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 7.260274701556586e-09,
       "learning_rate": 6.976902622196776e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.623698234558105,
+      "logits/rejected": 8.419386863708496,
+      "logps/chosen": -41.76908493041992,
+      "logps/rejected": -42.57697296142578,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -41.76908493041992,
+      "rewards/margins": 0.8078799247741699,
+      "rewards/rejected": -42.57697296142578,
       "step": 2430
     },
     {
       "epoch": 1.3032279645425655,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.7354807323703488e-11,
       "learning_rate": 6.962588040686064e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.827204704284668,
+      "logits/rejected": 8.322413444519043,
+      "logps/chosen": -41.5187873840332,
+      "logps/rejected": -42.699493408203125,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -41.5187873840332,
+      "rewards/margins": 1.1807054281234741,
+      "rewards/rejected": -42.699493408203125,
       "step": 2435
     },
     {
       "epoch": 1.3059039973239672,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.6773791010464444e-10,
       "learning_rate": 6.948254418137573e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.62433910369873,
+      "logits/rejected": 8.21703052520752,
+      "logps/chosen": -41.080535888671875,
+      "logps/rejected": -42.85414123535156,
+      "loss": 0.0,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -41.080535888671875,
+      "rewards/margins": 1.7736066579818726,
+      "rewards/rejected": -42.85414123535156,
       "step": 2440
     },
     {
       "epoch": 1.3085800301053687,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 0.006580855672145723,
       "learning_rate": 6.933901893616174e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.711406707763672,
+      "logits/rejected": 8.392984390258789,
+      "logps/chosen": -40.449989318847656,
+      "logps/rejected": -41.890872955322266,
+      "loss": 0.0,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -40.449989318847656,
+      "rewards/margins": 1.4408788681030273,
+      "rewards/rejected": -41.890872955322266,
       "step": 2445
     },
     {
       "epoch": 1.3112560628867704,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.3504241809224855e-08,
       "learning_rate": 6.919530606370121e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.752779006958008,
+      "logits/rejected": 8.183549880981445,
+      "logps/chosen": -42.184120178222656,
+      "logps/rejected": -42.82275390625,
+      "loss": 0.0,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -42.184120178222656,
+      "rewards/margins": 0.6386359333992004,
+      "rewards/rejected": -42.82275390625,
       "step": 2450
     },
     {
       "epoch": 1.313932095668172,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 8.266462065946908e-09,
       "learning_rate": 6.905140695829706e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.778003692626953,
+      "logits/rejected": 8.015851020812988,
+      "logps/chosen": -41.44707107543945,
+      "logps/rejected": -43.082847595214844,
+      "loss": 0.0,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -41.44707107543945,
+      "rewards/margins": 1.6357781887054443,
+      "rewards/rejected": -43.082847595214844,
       "step": 2455
     },
     {
       "epoch": 1.3166081284495736,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.754618605966783e-13,
       "learning_rate": 6.890732301605904e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.421499252319336,
+      "logits/rejected": 8.113317489624023,
+      "logps/chosen": -42.11381149291992,
+      "logps/rejected": -42.73085403442383,
+      "loss": 0.0,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -42.11381149291992,
+      "rewards/margins": 0.6170390844345093,
+      "rewards/rejected": -42.73085403442383,
       "step": 2460
     },
     {
       "epoch": 1.3192841612309751,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.4550976081624315e-08,
       "learning_rate": 6.876305563489021e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.965174674987793,
+      "logits/rejected": 8.695558547973633,
+      "logps/chosen": -40.9244384765625,
+      "logps/rejected": -42.84111785888672,
+      "loss": 0.0,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -40.9244384765625,
+      "rewards/margins": 1.9166736602783203,
+      "rewards/rejected": -42.84111785888672,
       "step": 2465
     },
     {
       "epoch": 1.3219601940123766,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.211747862621378e-06,
       "learning_rate": 6.861860621447331e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.004940032958984,
+      "logits/rejected": 8.56294059753418,
+      "logps/chosen": -41.2769660949707,
+      "logps/rejected": -41.99330520629883,
+      "loss": 0.0,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -41.2769660949707,
+      "rewards/margins": 0.7163404226303101,
+      "rewards/rejected": -41.99330520629883,
       "step": 2470
     },
     {
       "epoch": 1.3246362267937783,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.3576926471003525e-07,
       "learning_rate": 6.847397615625725e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.54549503326416,
+      "logits/rejected": 8.454249382019043,
+      "logps/chosen": -42.046653747558594,
+      "logps/rejected": -42.49961471557617,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -42.046653747558594,
+      "rewards/margins": 0.45295557379722595,
+      "rewards/rejected": -42.49961471557617,
       "step": 2475
     },
     {
       "epoch": 1.3273122595751798,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.749330501273377e-07,
       "learning_rate": 6.83291668634435e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.07643985748291,
+      "logits/rejected": 8.617594718933105,
+      "logps/chosen": -40.69664764404297,
+      "logps/rejected": -43.09975051879883,
+      "loss": 0.0,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -40.69664764404297,
+      "rewards/margins": 2.4031033515930176,
+      "rewards/rejected": -43.09975051879883,
       "step": 2480
     },
     {
       "epoch": 1.3299882923565813,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 0.0005062999094676158,
       "learning_rate": 6.818417974097246e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.428836822509766,
+      "logits/rejected": 8.086463928222656,
+      "logps/chosen": -40.843238830566406,
+      "logps/rejected": -41.650115966796875,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -40.843238830566406,
+      "rewards/margins": 0.8068798184394836,
+      "rewards/rejected": -41.650115966796875,
       "step": 2485
     },
     {
       "epoch": 1.332664325137983,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.2960091028267025e-07,
       "learning_rate": 6.803901619550981e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.54697322845459,
+      "logits/rejected": 8.415047645568848,
+      "logps/chosen": -40.57067108154297,
+      "logps/rejected": -41.406795501708984,
+      "loss": 0.0,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -40.57067108154297,
+      "rewards/margins": 0.8361251950263977,
+      "rewards/rejected": -41.406795501708984,
       "step": 2490
     },
     {
       "epoch": 1.3353403579193845,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.9652714309888683e-06,
       "learning_rate": 6.789367763543292e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.747296333312988,
+      "logits/rejected": 8.594714164733887,
+      "logps/chosen": -42.303531646728516,
+      "logps/rejected": -42.46321487426758,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5062500238418579,
+      "rewards/chosen": -42.303531646728516,
+      "rewards/margins": 0.15968793630599976,
+      "rewards/rejected": -42.46321487426758,
       "step": 2495
     },
     {
       "epoch": 1.338016390700786,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.1589347068855514e-07,
       "learning_rate": 6.774816547081714e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.406895637512207,
+      "logits/rejected": 7.916224479675293,
+      "logps/chosen": -40.95653533935547,
+      "logps/rejected": -42.47511672973633,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -40.95653533935547,
+      "rewards/margins": 1.518578290939331,
+      "rewards/rejected": -42.47511672973633,
       "step": 2500
     },
     {
       "epoch": 1.3406924234821878,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 8.431653731678739e-08,
       "learning_rate": 6.760248111342211e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.756312370300293,
+      "logits/rejected": 8.310503959655762,
+      "logps/chosen": -42.473899841308594,
+      "logps/rejected": -44.119346618652344,
+      "loss": 0.0,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -42.473899841308594,
+      "rewards/margins": 1.6454436779022217,
+      "rewards/rejected": -44.119346618652344,
       "step": 2505
     },
     {
       "epoch": 1.3433684562635893,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 0.0020758887081444408,
       "learning_rate": 6.745662597667813e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.71196174621582,
+      "logits/rejected": 8.32054615020752,
+      "logps/chosen": -41.26520919799805,
+      "logps/rejected": -42.23027038574219,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -41.26520919799805,
+      "rewards/margins": 0.9650588035583496,
+      "rewards/rejected": -42.23027038574219,
       "step": 2510
     },
     {
       "epoch": 1.3460444890449907,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 5.018287026327811e-10,
       "learning_rate": 6.731060147567236e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.812787055969238,
+      "logits/rejected": 8.561102867126465,
+      "logps/chosen": -41.989990234375,
+      "logps/rejected": -43.05455780029297,
+      "loss": 0.0,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -41.989990234375,
+      "rewards/margins": 1.0645701885223389,
+      "rewards/rejected": -43.05455780029297,
       "step": 2515
     },
     {
       "epoch": 1.3487205218263925,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.0039794717480037e-09,
       "learning_rate": 6.716440902713515e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.890619277954102,
+      "logits/rejected": 8.662046432495117,
+      "logps/chosen": -41.35208511352539,
+      "logps/rejected": -42.011566162109375,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -41.35208511352539,
+      "rewards/margins": 0.6594769954681396,
+      "rewards/rejected": -42.011566162109375,
       "step": 2520
     },
     {
       "epoch": 1.351396554607794,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.0519214631288001e-06,
       "learning_rate": 6.701805004942627e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.953778266906738,
+      "logits/rejected": 8.594943046569824,
+      "logps/chosen": -41.30754852294922,
+      "logps/rejected": -42.41514205932617,
+      "loss": 0.0,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -41.30754852294922,
+      "rewards/margins": 1.1075888872146606,
+      "rewards/rejected": -42.41514205932617,
       "step": 2525
     },
     {
       "epoch": 1.3540725873891954,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 7.248119894564718e-07,
       "learning_rate": 6.687152596252119e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.74620532989502,
+      "logits/rejected": 8.61555004119873,
+      "logps/chosen": -41.74907684326172,
+      "logps/rejected": -42.83419418334961,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -41.74907684326172,
+      "rewards/margins": 1.085118055343628,
+      "rewards/rejected": -42.83419418334961,
       "step": 2530
     },
     {
       "epoch": 1.3567486201705972,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 6.083875027474749e-10,
       "learning_rate": 6.672483818799722e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.921418190002441,
+      "logits/rejected": 8.451409339904785,
+      "logps/chosen": -41.394901275634766,
+      "logps/rejected": -42.724754333496094,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5062500238418579,
+      "rewards/chosen": -41.394901275634766,
+      "rewards/margins": 1.3298547267913818,
+      "rewards/rejected": -42.724754333496094,
       "step": 2535
     },
     {
       "epoch": 1.3594246529519987,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 5.15472179116309e-10,
       "learning_rate": 6.657798814901978e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.760702133178711,
+      "logits/rejected": 8.053281784057617,
+      "logps/chosen": -42.08037185668945,
+      "logps/rejected": -42.4610710144043,
+      "loss": 0.0,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": -42.08037185668945,
+      "rewards/margins": 0.38069939613342285,
+      "rewards/rejected": -42.4610710144043,
       "step": 2540
     },
     {
       "epoch": 1.3621006857334002,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.235811601808339e-06,
       "learning_rate": 6.643097727032863e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.981941223144531,
+      "logits/rejected": 8.411722183227539,
+      "logps/chosen": -42.244911193847656,
+      "logps/rejected": -43.563201904296875,
+      "loss": 0.0003,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -42.244911193847656,
+      "rewards/margins": 1.3182884454727173,
+      "rewards/rejected": -43.563201904296875,
       "step": 2545
     },
     {
       "epoch": 1.3647767185148019,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.0000092335168433e-06,
       "learning_rate": 6.628380697822392e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.122479438781738,
+      "logits/rejected": 8.642129898071289,
+      "logps/chosen": -41.744998931884766,
+      "logps/rejected": -42.602012634277344,
+      "loss": 0.0,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": -41.744998931884766,
+      "rewards/margins": 0.8570182919502258,
+      "rewards/rejected": -42.602012634277344,
       "step": 2550
     },
     {
       "epoch": 1.3674527512962034,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.3407544871507295e-05,
       "learning_rate": 6.61364787005525e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.453834533691406,
+      "logits/rejected": 8.173998832702637,
+      "logps/chosen": -40.599891662597656,
+      "logps/rejected": -42.157222747802734,
+      "loss": 0.0,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -40.599891662597656,
+      "rewards/margins": 1.5573351383209229,
+      "rewards/rejected": -42.157222747802734,
       "step": 2555
     },
     {
       "epoch": 1.3701287840776049,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.0003703037058036e-10,
       "learning_rate": 6.598899386669395e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.911181449890137,
+      "logits/rejected": 8.530158042907715,
+      "logps/chosen": -41.685951232910156,
+      "logps/rejected": -42.94929504394531,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -41.685951232910156,
+      "rewards/margins": 1.2633416652679443,
+      "rewards/rejected": -42.94929504394531,
       "step": 2560
     },
     {
       "epoch": 1.3728048168590066,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.957785349843777e-12,
       "learning_rate": 6.584135390754679e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.727920532226562,
+      "logits/rejected": 8.331493377685547,
+      "logps/chosen": -41.623451232910156,
+      "logps/rejected": -43.342689514160156,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -41.623451232910156,
+      "rewards/margins": 1.719238042831421,
+      "rewards/rejected": -43.342689514160156,
       "step": 2565
     },
     {
       "epoch": 1.375480849640408,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.4989211397416e-08,
       "learning_rate": 6.569356025551454e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.65626335144043,
+      "logits/rejected": 8.505366325378418,
+      "logps/chosen": -42.0156364440918,
+      "logps/rejected": -42.59636306762695,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -42.0156364440918,
+      "rewards/margins": 0.5807242393493652,
+      "rewards/rejected": -42.59636306762695,
       "step": 2570
     },
     {
       "epoch": 1.3781568824218096,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.629328258757614e-08,
       "learning_rate": 6.554561434449186e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.792577743530273,
+      "logits/rejected": 8.441924095153809,
+      "logps/chosen": -41.50666809082031,
+      "logps/rejected": -42.96664047241211,
+      "loss": 0.0,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -41.50666809082031,
+      "rewards/margins": 1.4599723815917969,
+      "rewards/rejected": -42.96664047241211,
       "step": 2575
     },
     {
       "epoch": 1.3808329152032113,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.8217557630437925e-07,
       "learning_rate": 6.539751760985063e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.742188453674316,
+      "logits/rejected": 8.44264030456543,
+      "logps/chosen": -40.404754638671875,
+      "logps/rejected": -41.509552001953125,
+      "loss": 0.0,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -40.404754638671875,
+      "rewards/margins": 1.1047991514205933,
+      "rewards/rejected": -41.509552001953125,
       "step": 2580
     },
     {
       "epoch": 1.3835089479846128,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 8.694552809611475e-08,
       "learning_rate": 6.524927148842602e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.650606155395508,
+      "logits/rejected": 8.200862884521484,
+      "logps/chosen": -42.68472671508789,
+      "logps/rejected": -44.271270751953125,
+      "loss": 0.0,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -42.68472671508789,
+      "rewards/margins": 1.5865452289581299,
+      "rewards/rejected": -44.271270751953125,
       "step": 2585
     },
     {
       "epoch": 1.3861849807660143,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 6.3547481321882486e-15,
       "learning_rate": 6.510087741850254e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.042112350463867,
+      "logits/rejected": 8.794302940368652,
+      "logps/chosen": -42.84346389770508,
+      "logps/rejected": -44.69168472290039,
+      "loss": 0.0,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -42.84346389770508,
+      "rewards/margins": 1.8482208251953125,
+      "rewards/rejected": -44.69168472290039,
       "step": 2590
     },
     {
       "epoch": 1.388861013547416,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.923662942049298e-08,
       "learning_rate": 6.495233683980012e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.974845886230469,
+      "logits/rejected": 8.706317901611328,
+      "logps/chosen": -41.35472106933594,
+      "logps/rejected": -42.301124572753906,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5062500238418579,
+      "rewards/chosen": -41.35472106933594,
+      "rewards/margins": 0.9464014172554016,
+      "rewards/rejected": -42.301124572753906,
       "step": 2595
     },
     {
       "epoch": 1.3915370463288175,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.7267148692331677e-05,
       "learning_rate": 6.480365119346011e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.6168851852417,
+      "logits/rejected": 8.161127090454102,
+      "logps/chosen": -40.83750915527344,
+      "logps/rejected": -41.776756286621094,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -40.83750915527344,
+      "rewards/margins": 0.9392474889755249,
+      "rewards/rejected": -41.776756286621094,
       "step": 2600
     },
     {
       "epoch": 1.394213079110219,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 6.897164505210194e-07,
       "learning_rate": 6.465482192203129e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.35374641418457,
+      "logits/rejected": 8.269282341003418,
+      "logps/chosen": -41.55049514770508,
+      "logps/rejected": -42.83757400512695,
+      "loss": 0.0,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -41.55049514770508,
+      "rewards/margins": 1.2870848178863525,
+      "rewards/rejected": -42.83757400512695,
       "step": 2605
     },
     {
       "epoch": 1.3968891118916207,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.750062726316517e-06,
       "learning_rate": 6.45058504694559e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.309649467468262,
+      "logits/rejected": 8.38338565826416,
+      "logps/chosen": -41.148685455322266,
+      "logps/rejected": -43.27328872680664,
+      "loss": 0.0,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -41.148685455322266,
+      "rewards/margins": 2.124605178833008,
+      "rewards/rejected": -43.27328872680664,
       "step": 2610
     },
     {
       "epoch": 1.3995651446730222,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.8949276543353387e-05,
       "learning_rate": 6.435673828105564e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.728071212768555,
+      "logits/rejected": 8.415145874023438,
+      "logps/chosen": -39.964942932128906,
+      "logps/rejected": -41.9430046081543,
+      "loss": 0.0,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -39.964942932128906,
+      "rewards/margins": 1.9780590534210205,
+      "rewards/rejected": -41.9430046081543,
       "step": 2615
     },
     {
       "epoch": 1.402241177454424,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.354581939964938e-12,
       "learning_rate": 6.420748680351763e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.721088409423828,
+      "logits/rejected": 8.743288040161133,
+      "logps/chosen": -41.229286193847656,
+      "logps/rejected": -42.19605255126953,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -41.229286193847656,
+      "rewards/margins": 0.9667743444442749,
+      "rewards/rejected": -42.19605255126953,
       "step": 2620
     },
     {
       "epoch": 1.4049172102358254,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 5.081077099835597e-09,
       "learning_rate": 6.405809748488032e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.89842414855957,
+      "logits/rejected": 8.644752502441406,
+      "logps/chosen": -42.63088607788086,
+      "logps/rejected": -43.74695587158203,
+      "loss": 0.0,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -42.63088607788086,
+      "rewards/margins": 1.116062045097351,
+      "rewards/rejected": -43.74695587158203,
       "step": 2625
     },
     {
       "epoch": 1.4075932430172269,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 7.119297127373652e-11,
       "learning_rate": 6.390857177451956e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.8165922164917,
+      "logits/rejected": 8.361710548400879,
+      "logps/chosen": -41.20853805541992,
+      "logps/rejected": -42.973976135253906,
+      "loss": 0.0008,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -41.20853805541992,
+      "rewards/margins": 1.7654361724853516,
+      "rewards/rejected": -42.973976135253906,
       "step": 2630
     },
     {
       "epoch": 1.4102692757986286,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 7.145017418345644e-07,
       "learning_rate": 6.375891112313445e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.590654373168945,
+      "logits/rejected": 8.280073165893555,
+      "logps/chosen": -41.35600662231445,
+      "logps/rejected": -43.14480972290039,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -41.35600662231445,
+      "rewards/margins": 1.7888002395629883,
+      "rewards/rejected": -43.14480972290039,
       "step": 2635
     },
     {
       "epoch": 1.41294530858003,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.339794923848692e-08,
       "learning_rate": 6.360911698273326e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.734606742858887,
+      "logits/rejected": 8.351479530334473,
+      "logps/chosen": -41.319435119628906,
+      "logps/rejected": -42.49150085449219,
+      "loss": 0.0,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -41.319435119628906,
+      "rewards/margins": 1.172070860862732,
+      "rewards/rejected": -42.49150085449219,
       "step": 2640
     },
     {
       "epoch": 1.4156213413614318,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.9412070211162934e-11,
       "learning_rate": 6.345919080661944e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.945253372192383,
+      "logits/rejected": 8.674893379211426,
+      "logps/chosen": -42.319496154785156,
+      "logps/rejected": -43.33086395263672,
+      "loss": 0.0,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -42.319496154785156,
+      "rewards/margins": 1.0113706588745117,
+      "rewards/rejected": -43.33086395263672,
       "step": 2645
     },
     {
       "epoch": 1.4182973741428333,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.162284700657952e-10,
       "learning_rate": 6.330913404937737e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.95710563659668,
+      "logits/rejected": 8.489824295043945,
+      "logps/chosen": -42.271392822265625,
+      "logps/rejected": -43.92520523071289,
+      "loss": 0.0,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -42.271392822265625,
+      "rewards/margins": 1.6538140773773193,
+      "rewards/rejected": -43.92520523071289,
       "step": 2650
     },
     {
       "epoch": 1.4209734069242348,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.5087827651696878e-05,
       "learning_rate": 6.315894816685838e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.598902702331543,
+      "logits/rejected": 8.227259635925293,
+      "logps/chosen": -42.930458068847656,
+      "logps/rejected": -43.603599548339844,
+      "loss": 0.0,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -42.930458068847656,
+      "rewards/margins": 0.6731430888175964,
+      "rewards/rejected": -43.603599548339844,
       "step": 2655
     },
     {
       "epoch": 1.4236494397056365,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 5.030919522031955e-13,
       "learning_rate": 6.300863461616657e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.209463119506836,
+      "logits/rejected": 7.915236473083496,
+      "logps/chosen": -42.28984832763672,
+      "logps/rejected": -43.53640365600586,
+      "loss": 0.0,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -42.28984832763672,
+      "rewards/margins": 1.2465569972991943,
+      "rewards/rejected": -43.53640365600586,
       "step": 2660
     },
     {
       "epoch": 1.426325472487038,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.8482206258726657e-09,
       "learning_rate": 6.285819485564465e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.046075820922852,
+      "logits/rejected": 8.36937141418457,
+      "logps/chosen": -41.806270599365234,
+      "logps/rejected": -42.67213439941406,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -41.806270599365234,
+      "rewards/margins": 0.8658647537231445,
+      "rewards/rejected": -42.67213439941406,
       "step": 2665
     },
     {
       "epoch": 1.4290015052684395,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.7938968593213507e-07,
       "learning_rate": 6.270763034485986e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.585349082946777,
+      "logits/rejected": 8.288152694702148,
+      "logps/chosen": -40.58457946777344,
+      "logps/rejected": -41.821144104003906,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -40.58457946777344,
+      "rewards/margins": 1.236563801765442,
+      "rewards/rejected": -41.821144104003906,
       "step": 2670
     },
     {
       "epoch": 1.4316775380498412,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 8.576400073891725e-06,
       "learning_rate": 6.255694254458972e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.569849967956543,
+      "logits/rejected": 8.177288055419922,
+      "logps/chosen": -40.41807174682617,
+      "logps/rejected": -42.742557525634766,
+      "loss": 0.0033,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -40.41807174682617,
+      "rewards/margins": 2.3244881629943848,
+      "rewards/rejected": -42.742557525634766,
       "step": 2675
     },
     {
       "epoch": 1.4343535708312427,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.5803549186674131e-06,
       "learning_rate": 6.240613291680795e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.672123908996582,
+      "logits/rejected": 8.24134349822998,
+      "logps/chosen": -42.231529235839844,
+      "logps/rejected": -44.24704360961914,
+      "loss": 0.0,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -42.231529235839844,
+      "rewards/margins": 2.0155093669891357,
+      "rewards/rejected": -44.24704360961914,
       "step": 2680
     },
     {
       "epoch": 1.4370296036126442,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 0.0003616355373746601,
       "learning_rate": 6.225520292467021e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.974054336547852,
+      "logits/rejected": 8.090018272399902,
+      "logps/chosen": -41.403297424316406,
+      "logps/rejected": -43.79715347290039,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -41.403297424316406,
+      "rewards/margins": 2.3938541412353516,
+      "rewards/rejected": -43.79715347290039,
       "step": 2685
     },
     {
       "epoch": 1.439705636394046,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.955351860021308e-11,
       "learning_rate": 6.210415403249993e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.829411506652832,
+      "logits/rejected": 8.101358413696289,
+      "logps/chosen": -40.76591491699219,
+      "logps/rejected": -43.67510223388672,
+      "loss": 0.0,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -40.76591491699219,
+      "rewards/margins": 2.909186363220215,
+      "rewards/rejected": -43.67510223388672,
       "step": 2690
     },
     {
       "epoch": 1.4423816691754474,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.2346477441096996e-06,
       "learning_rate": 6.195298770577415e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.410456657409668,
+      "logits/rejected": 8.339765548706055,
+      "logps/chosen": -42.111656188964844,
+      "logps/rejected": -42.86857986450195,
+      "loss": 0.0002,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -42.111656188964844,
+      "rewards/margins": 0.7569217681884766,
+      "rewards/rejected": -42.86857986450195,
       "step": 2695
     },
     {
       "epoch": 1.445057701956849,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.9201134335457046e-07,
       "learning_rate": 6.180170541110923e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.710095405578613,
+      "logits/rejected": 8.143321990966797,
+      "logps/chosen": -41.566593170166016,
+      "logps/rejected": -42.4012336730957,
+      "loss": 0.0,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -41.566593170166016,
+      "rewards/margins": 0.8346397280693054,
+      "rewards/rejected": -42.4012336730957,
       "step": 2700
     },
     {
       "epoch": 1.4477337347382506,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 6.147084902213062e-09,
       "learning_rate": 6.165030861624663e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.157144546508789,
+      "logits/rejected": 8.170949935913086,
+      "logps/chosen": -41.26401901245117,
+      "logps/rejected": -42.615875244140625,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -41.26401901245117,
+      "rewards/margins": 1.3518584966659546,
+      "rewards/rejected": -42.615875244140625,
       "step": 2705
     },
     {
       "epoch": 1.4504097675196521,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.4680937589798419e-08,
       "learning_rate": 6.149879879003876e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.618410110473633,
+      "logits/rejected": 8.747294425964355,
+      "logps/chosen": -41.62251663208008,
+      "logps/rejected": -42.37772750854492,
+      "loss": 0.0,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -41.62251663208008,
+      "rewards/margins": 0.7552111744880676,
+      "rewards/rejected": -42.37772750854492,
       "step": 2710
     },
     {
       "epoch": 1.4530858003010536,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 5.780344334401673e-07,
       "learning_rate": 6.13471774024346e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.884693145751953,
+      "logits/rejected": 8.513826370239258,
+      "logps/chosen": -40.83106231689453,
+      "logps/rejected": -41.84722137451172,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -40.83106231689453,
+      "rewards/margins": 1.0161610841751099,
+      "rewards/rejected": -41.84722137451172,
       "step": 2715
     },
     {
       "epoch": 1.4557618330824553,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.316326782492241e-09,
       "learning_rate": 6.119544592446551e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.547552108764648,
+      "logits/rejected": 8.374757766723633,
+      "logps/chosen": -41.661956787109375,
+      "logps/rejected": -43.59321975708008,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -41.661956787109375,
+      "rewards/margins": 1.931265115737915,
+      "rewards/rejected": -43.59321975708008,
       "step": 2720
     },
     {
       "epoch": 1.4584378658638568,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.304987298256512e-07,
       "learning_rate": 6.104360582823096e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.907578468322754,
+      "logits/rejected": 8.626019477844238,
+      "logps/chosen": -40.874298095703125,
+      "logps/rejected": -43.00669479370117,
+      "loss": 0.0,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -40.874298095703125,
+      "rewards/margins": 2.132399320602417,
+      "rewards/rejected": -43.00669479370117,
       "step": 2725
     },
     {
       "epoch": 1.4611138986452583,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.5052392047264423e-09,
       "learning_rate": 6.089165858688423e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.702375411987305,
+      "logits/rejected": 8.062834739685059,
+      "logps/chosen": -41.73476028442383,
+      "logps/rejected": -42.20301818847656,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -41.73476028442383,
+      "rewards/margins": 0.4682607054710388,
+      "rewards/rejected": -42.20301818847656,
       "step": 2730
     },
     {
       "epoch": 1.46378993142666,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.4971702048636752e-07,
       "learning_rate": 6.073960567461811e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.04012680053711,
+      "logits/rejected": 8.473031997680664,
+      "logps/chosen": -41.67736053466797,
+      "logps/rejected": -43.5404052734375,
+      "loss": 0.0,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -41.67736053466797,
+      "rewards/margins": 1.8630421161651611,
+      "rewards/rejected": -43.5404052734375,
       "step": 2735
     },
     {
       "epoch": 1.4664659642080615,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 8.926331926282415e-09,
       "learning_rate": 6.058744856665065e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.426590919494629,
+      "logits/rejected": 8.039485931396484,
+      "logps/chosen": -41.57054138183594,
+      "logps/rejected": -43.211910247802734,
+      "loss": 0.0,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -41.57054138183594,
+      "rewards/margins": 1.6413624286651611,
+      "rewards/rejected": -43.211910247802734,
       "step": 2740
     },
     {
       "epoch": 1.469141996989463,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 8.14014674716484e-05,
       "learning_rate": 6.043518873921074e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.057416915893555,
+      "logits/rejected": 8.462617874145508,
+      "logps/chosen": -40.877845764160156,
+      "logps/rejected": -42.28983688354492,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -40.877845764160156,
+      "rewards/margins": 1.4119914770126343,
+      "rewards/rejected": -42.28983688354492,
       "step": 2745
     },
     {
       "epoch": 1.4718180297708647,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 5.254469351115687e-09,
       "learning_rate": 6.028282766952393e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.44697380065918,
+      "logits/rejected": 8.054309844970703,
+      "logps/chosen": -41.75263214111328,
+      "logps/rejected": -42.920040130615234,
+      "loss": 0.0,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -41.75263214111328,
+      "rewards/margins": 1.1674166917800903,
+      "rewards/rejected": -42.920040130615234,
       "step": 2750
     },
     {
       "epoch": 1.4744940625522662,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 8.455064514128113e-09,
       "learning_rate": 6.013036683579798e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.765707015991211,
+      "logits/rejected": 8.471872329711914,
+      "logps/chosen": -41.94719314575195,
+      "logps/rejected": -43.32516098022461,
+      "loss": 0.0,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -41.94719314575195,
+      "rewards/margins": 1.3779693841934204,
+      "rewards/rejected": -43.32516098022461,
       "step": 2755
     },
     {
       "epoch": 1.4771700953336677,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 5.591991893940422e-05,
       "learning_rate": 5.997780771720854e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.819055557250977,
+      "logits/rejected": 8.026742935180664,
+      "logps/chosen": -40.92245101928711,
+      "logps/rejected": -42.21430587768555,
+      "loss": 0.0,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -40.92245101928711,
+      "rewards/margins": 1.2918500900268555,
+      "rewards/rejected": -42.21430587768555,
       "step": 2760
     },
     {
       "epoch": 1.4798461281150694,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.967908968636662e-14,
       "learning_rate": 5.982515179388486e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.710647583007812,
+      "logits/rejected": 8.366626739501953,
+      "logps/chosen": -42.1692008972168,
+      "logps/rejected": -43.35497283935547,
+      "loss": 0.0,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -42.1692008972168,
+      "rewards/margins": 1.1857683658599854,
+      "rewards/rejected": -43.35497283935547,
       "step": 2765
     },
     {
       "epoch": 1.482522160896471,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 0.00014213200505656836,
       "learning_rate": 5.967240054689541e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.115320205688477,
+      "logits/rejected": 9.052719116210938,
+      "logps/chosen": -41.38245391845703,
+      "logps/rejected": -42.613563537597656,
+      "loss": 0.0,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -41.38245391845703,
+      "rewards/margins": 1.2311127185821533,
+      "rewards/rejected": -42.613563537597656,
       "step": 2770
     },
     {
       "epoch": 1.4851981936778724,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.2366252018341718e-06,
       "learning_rate": 5.951955545823342e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.947187423706055,
+      "logits/rejected": 8.704947471618652,
+      "logps/chosen": -41.465145111083984,
+      "logps/rejected": -43.29473876953125,
+      "loss": 0.0,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -41.465145111083984,
+      "rewards/margins": 1.829595923423767,
+      "rewards/rejected": -43.29473876953125,
       "step": 2775
     },
     {
       "epoch": 1.4878742264592741,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 6.5358773561707e-09,
       "learning_rate": 5.936661801080263e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.623039245605469,
+      "logits/rejected": 8.29152774810791,
+      "logps/chosen": -41.36957550048828,
+      "logps/rejected": -42.660457611083984,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -41.36957550048828,
+      "rewards/margins": 1.2908824682235718,
+      "rewards/rejected": -42.660457611083984,
       "step": 2780
     },
     {
       "epoch": 1.4905502592406756,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.4478548016769985e-07,
       "learning_rate": 5.92135896884028e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.315587997436523,
+      "logits/rejected": 8.763214111328125,
+      "logps/chosen": -42.89904022216797,
+      "logps/rejected": -43.45850372314453,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -42.89904022216797,
+      "rewards/margins": 0.559464156627655,
+      "rewards/rejected": -43.45850372314453,
       "step": 2785
     },
     {
       "epoch": 1.4932262920220774,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.6752017095706123e-07,
       "learning_rate": 5.906047197571541e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.681356430053711,
+      "logits/rejected": 8.956047058105469,
+      "logps/chosen": -42.605445861816406,
+      "logps/rejected": -42.976097106933594,
+      "loss": 0.0,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -42.605445861816406,
+      "rewards/margins": 0.37064868211746216,
+      "rewards/rejected": -42.976097106933594,
       "step": 2790
     },
     {
       "epoch": 1.4959023248034788,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.3092466133363111e-06,
       "learning_rate": 5.890726635828919e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.810511589050293,
+      "logits/rejected": 8.950778007507324,
+      "logps/chosen": -42.79461669921875,
+      "logps/rejected": -43.614627838134766,
+      "loss": 0.0,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": -42.79461669921875,
+      "rewards/margins": 0.8200146555900574,
+      "rewards/rejected": -43.614627838134766,
       "step": 2795
     },
     {
       "epoch": 1.4985783575848803,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.1611370573916827e-09,
       "learning_rate": 5.875397432252569e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.87797737121582,
+      "logits/rejected": 8.874384880065918,
+      "logps/chosen": -41.66436004638672,
+      "logps/rejected": -41.99811553955078,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5062500238418579,
+      "rewards/chosen": -41.66436004638672,
+      "rewards/margins": 0.3337518572807312,
+      "rewards/rejected": -41.99811553955078,
       "step": 2800
     },
     {
       "epoch": 1.4985783575848803,
-      "eval_logits/chosen": NaN,
-      "eval_logits/rejected": NaN,
-      "eval_logps/chosen": NaN,
-      "eval_logps/rejected": NaN,
-      "eval_loss": NaN,
-      "eval_rewards/accuracies": 0.0,
-      "eval_rewards/chosen": NaN,
-      "eval_rewards/margins": NaN,
-      "eval_rewards/rejected": NaN,
-      "eval_runtime": 35.2612,
-      "eval_samples_per_second": 38.144,
-      "eval_semantic_entropy": NaN,
-      "eval_steps_per_second": 9.557,
+      "eval_logits/chosen": 7.462876796722412,
+      "eval_logits/rejected": 7.207109451293945,
+      "eval_logps/chosen": -41.718467712402344,
+      "eval_logps/rejected": -42.90156555175781,
+      "eval_loss": 2.9896133924012247e-07,
+      "eval_rewards/accuracies": 0.5630564093589783,
+      "eval_rewards/chosen": -41.718467712402344,
+      "eval_rewards/margins": 1.1831080913543701,
+      "eval_rewards/rejected": -42.90156555175781,
+      "eval_runtime": 40.3486,
+      "eval_samples_per_second": 33.334,
+      "eval_steps_per_second": 8.352,
       "step": 2800
     },
     {
       "epoch": 1.5012543903662818,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 5.240221569606272e-09,
       "learning_rate": 5.860059735566491e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.308040618896484,
+      "logits/rejected": 8.846551895141602,
+      "logps/chosen": -42.58149337768555,
+      "logps/rejected": -43.26437759399414,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -42.58149337768555,
+      "rewards/margins": 0.6828859448432922,
+      "rewards/rejected": -43.26437759399414,
       "step": 2805
     },
     {
       "epoch": 1.5039304231476835,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.5241795381715775e-10,
       "learning_rate": 5.844713694577087e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.802517890930176,
+      "logits/rejected": 8.490335464477539,
+      "logps/chosen": -41.10953903198242,
+      "logps/rejected": -42.98762893676758,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -41.10953903198242,
+      "rewards/margins": 1.8780874013900757,
+      "rewards/rejected": -42.98762893676758,
       "step": 2810
     },
     {
       "epoch": 1.5066064559290853,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 0.00014955291429823763,
       "learning_rate": 5.829359458171714e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.675622940063477,
+      "logits/rejected": 8.524667739868164,
+      "logps/chosen": -42.0042724609375,
+      "logps/rejected": -43.8996696472168,
+      "loss": 0.0,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -42.0042724609375,
+      "rewards/margins": 1.895399808883667,
+      "rewards/rejected": -43.8996696472168,
       "step": 2815
     },
     {
       "epoch": 1.5092824887104868,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.446093561651825e-05,
       "learning_rate": 5.81399717531724e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.089492797851562,
+      "logits/rejected": 8.260062217712402,
+      "logps/chosen": -41.917442321777344,
+      "logps/rejected": -43.20248794555664,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -41.917442321777344,
+      "rewards/margins": 1.285048246383667,
+      "rewards/rejected": -43.20248794555664,
       "step": 2820
     },
     {
       "epoch": 1.5119585214918883,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 5.8508346770030255e-05,
       "learning_rate": 5.798626995058602e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.176536560058594,
+      "logits/rejected": 8.630223274230957,
+      "logps/chosen": -41.753700256347656,
+      "logps/rejected": -42.61920166015625,
+      "loss": 0.0,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": -41.753700256347656,
+      "rewards/margins": 0.8655007481575012,
+      "rewards/rejected": -42.61920166015625,
       "step": 2825
     },
     {
       "epoch": 1.51463455427329,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 6.5935377030015784e-12,
       "learning_rate": 5.783249066517354e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.406403541564941,
+      "logits/rejected": 8.233708381652832,
+      "logps/chosen": -42.976043701171875,
+      "logps/rejected": -44.048439025878906,
+      "loss": 0.0,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -42.976043701171875,
+      "rewards/margins": 1.0723955631256104,
+      "rewards/rejected": -44.048439025878906,
       "step": 2830
     },
     {
       "epoch": 1.5173105870546915,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.629062625186188e-07,
       "learning_rate": 5.767863538890228e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.886978149414062,
+      "logits/rejected": 8.5150146484375,
+      "logps/chosen": -41.98692321777344,
+      "logps/rejected": -43.80683517456055,
+      "loss": 0.0006,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -41.98692321777344,
+      "rewards/margins": 1.8199154138565063,
+      "rewards/rejected": -43.80683517456055,
       "step": 2835
     },
     {
       "epoch": 1.519986619836093,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.3701190253475095e-09,
       "learning_rate": 5.75247056144768e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.84416389465332,
+      "logits/rejected": 8.61463737487793,
+      "logps/chosen": -42.14787673950195,
+      "logps/rejected": -43.078182220458984,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -42.14787673950195,
+      "rewards/margins": 0.9303110241889954,
+      "rewards/rejected": -43.078182220458984,
       "step": 2840
     },
     {
       "epoch": 1.5226626526174947,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 5.131912228535976e-08,
       "learning_rate": 5.737070283532444e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.2601900100708,
+      "logits/rejected": 8.791196823120117,
+      "logps/chosen": -42.051273345947266,
+      "logps/rejected": -43.37104797363281,
+      "loss": 0.0,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -42.051273345947266,
+      "rewards/margins": 1.319777250289917,
+      "rewards/rejected": -43.37104797363281,
       "step": 2845
     },
     {
       "epoch": 1.5253386853988962,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.412108424898287e-11,
       "learning_rate": 5.721662854558084e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.788159370422363,
+      "logits/rejected": 8.487028121948242,
+      "logps/chosen": -41.32537841796875,
+      "logps/rejected": -42.331199645996094,
+      "loss": 0.0,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": -41.32537841796875,
+      "rewards/margins": 1.005817174911499,
+      "rewards/rejected": -42.331199645996094,
       "step": 2850
     },
     {
       "epoch": 1.5280147181802977,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.261682624227416e-12,
       "learning_rate": 5.706248424007545e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.630437850952148,
+      "logits/rejected": 8.20901870727539,
+      "logps/chosen": -40.639930725097656,
+      "logps/rejected": -42.065704345703125,
+      "loss": 0.0,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -40.639930725097656,
+      "rewards/margins": 1.4257698059082031,
+      "rewards/rejected": -42.065704345703125,
       "step": 2855
     },
     {
       "epoch": 1.5306907509616994,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 7.364941016173929e-05,
       "learning_rate": 5.690827141431699e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.075970649719238,
+      "logits/rejected": 8.477521896362305,
+      "logps/chosen": -42.701927185058594,
+      "logps/rejected": -43.13222885131836,
+      "loss": 0.0,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -42.701927185058594,
+      "rewards/margins": 0.43030399084091187,
+      "rewards/rejected": -43.13222885131836,
       "step": 2860
     },
     {
       "epoch": 1.5333667837431009,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.2940298544455694e-10,
       "learning_rate": 5.675399156447897e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.055513381958008,
+      "logits/rejected": 8.465234756469727,
+      "logps/chosen": -41.51704025268555,
+      "logps/rejected": -42.361717224121094,
+      "loss": 0.0,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -41.51704025268555,
+      "rewards/margins": 0.8446768522262573,
+      "rewards/rejected": -42.361717224121094,
       "step": 2865
     },
     {
       "epoch": 1.5360428165245024,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 0.0004173381613143238,
       "learning_rate": 5.659964618738515e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.119138717651367,
+      "logits/rejected": 8.610302925109863,
+      "logps/chosen": -41.553932189941406,
+      "logps/rejected": -42.481388092041016,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -41.553932189941406,
+      "rewards/margins": 0.9274513125419617,
+      "rewards/rejected": -42.481388092041016,
       "step": 2870
     },
     {
       "epoch": 1.538718849305904,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 5.233463765742127e-05,
       "learning_rate": 5.644523678049509e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.967458724975586,
+      "logits/rejected": 8.663304328918457,
+      "logps/chosen": -41.5849723815918,
+      "logps/rejected": -42.68383026123047,
+      "loss": 0.0,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -41.5849723815918,
+      "rewards/margins": 1.0988551378250122,
+      "rewards/rejected": -42.68383026123047,
       "step": 2875
     },
     {
       "epoch": 1.5413948820873056,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.9087525451153584e-05,
       "learning_rate": 5.629076484188952e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.666072845458984,
+      "logits/rejected": 8.390230178833008,
+      "logps/chosen": -42.19350814819336,
+      "logps/rejected": -43.1552734375,
+      "loss": 0.0,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -42.19350814819336,
+      "rewards/margins": 0.9617654085159302,
+      "rewards/rejected": -43.1552734375,
       "step": 2880
     },
     {
       "epoch": 1.544070914868707,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.9015629756792404e-09,
       "learning_rate": 5.613623187025587e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.060524940490723,
+      "logits/rejected": 8.695516586303711,
+      "logps/chosen": -43.05659103393555,
+      "logps/rejected": -43.17965316772461,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -43.05659103393555,
+      "rewards/margins": 0.12306338548660278,
+      "rewards/rejected": -43.17965316772461,
       "step": 2885
     },
     {
       "epoch": 1.5467469476501088,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.4957535969279874e-08,
       "learning_rate": 5.598163936487369e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.534125328063965,
+      "logits/rejected": 7.987432956695557,
+      "logps/chosen": -39.602210998535156,
+      "logps/rejected": -41.279685974121094,
+      "loss": 0.0,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -39.602210998535156,
+      "rewards/margins": 1.677476167678833,
+      "rewards/rejected": -41.279685974121094,
       "step": 2890
     },
     {
       "epoch": 1.5494229804315103,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.8436341546900504e-07,
       "learning_rate": 5.582698882560017e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.013283729553223,
+      "logits/rejected": 8.313443183898926,
+      "logps/chosen": -42.68204116821289,
+      "logps/rejected": -43.479148864746094,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -42.68204116821289,
+      "rewards/margins": 0.7971112728118896,
+      "rewards/rejected": -43.479148864746094,
       "step": 2895
     },
     {
       "epoch": 1.5520990132129118,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.079327247052572e-08,
       "learning_rate": 5.567228175285549e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.712634086608887,
+      "logits/rejected": 8.37256908416748,
+      "logps/chosen": -42.44874572753906,
+      "logps/rejected": -42.86425018310547,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -42.44874572753906,
+      "rewards/margins": 0.41550564765930176,
+      "rewards/rejected": -42.86425018310547,
       "step": 2900
     },
     {
       "epoch": 1.5547750459943135,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 5.73634186703197e-10,
       "learning_rate": 5.551751964760838e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.806078910827637,
+      "logits/rejected": 8.834476470947266,
+      "logps/chosen": -42.70290756225586,
+      "logps/rejected": -43.19768524169922,
+      "loss": 0.0,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -42.70290756225586,
+      "rewards/margins": 0.4947790503501892,
+      "rewards/rejected": -43.19768524169922,
       "step": 2905
     },
     {
       "epoch": 1.557451078775715,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.8021702646588545e-09,
       "learning_rate": 5.536270401136145e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.93919849395752,
+      "logits/rejected": 8.48941707611084,
+      "logps/chosen": -42.132591247558594,
+      "logps/rejected": -42.882015228271484,
+      "loss": 0.0,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -42.132591247558594,
+      "rewards/margins": 0.7494180798530579,
+      "rewards/rejected": -42.882015228271484,
       "step": 2910
     },
     {
       "epoch": 1.5601271115571165,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.64860085575252e-08,
       "learning_rate": 5.520783634613667e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.664732933044434,
+      "logits/rejected": 8.2017240524292,
+      "logps/chosen": -42.85877227783203,
+      "logps/rejected": -43.6851921081543,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -42.85877227783203,
+      "rewards/margins": 0.8264285922050476,
+      "rewards/rejected": -43.6851921081543,
       "step": 2915
     },
     {
       "epoch": 1.5628031443385182,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 8.214540923455313e-10,
       "learning_rate": 5.505291815446082e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.691978454589844,
+      "logits/rejected": 8.322660446166992,
+      "logps/chosen": -41.80618667602539,
+      "logps/rejected": -43.3055534362793,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -41.80618667602539,
+      "rewards/margins": 1.499367356300354,
+      "rewards/rejected": -43.3055534362793,
       "step": 2920
     },
     {
       "epoch": 1.5654791771199197,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.1086264503927307e-08,
       "learning_rate": 5.489795093935089e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.877646446228027,
+      "logits/rejected": 8.47821044921875,
+      "logps/chosen": -42.01649856567383,
+      "logps/rejected": -42.49527359008789,
+      "loss": 0.0,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -42.01649856567383,
+      "rewards/margins": 0.4787684977054596,
+      "rewards/rejected": -42.49527359008789,
       "step": 2925
     },
     {
       "epoch": 1.5681552099013212,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.6309603960220027e-09,
       "learning_rate": 5.474293620429946e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.955114364624023,
+      "logits/rejected": 8.372546195983887,
+      "logps/chosen": -41.6548957824707,
+      "logps/rejected": -43.28264617919922,
+      "loss": 0.0,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -41.6548957824707,
+      "rewards/margins": 1.6277482509613037,
+      "rewards/rejected": -43.28264617919922,
       "step": 2930
     },
     {
       "epoch": 1.570831242682723,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 0.00011882941871267415,
       "learning_rate": 5.458787545326018e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.98145580291748,
+      "logits/rejected": 8.75607681274414,
+      "logps/chosen": -41.506309509277344,
+      "logps/rejected": -42.45687484741211,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -41.506309509277344,
+      "rewards/margins": 0.9505647420883179,
+      "rewards/rejected": -42.45687484741211,
       "step": 2935
     },
     {
       "epoch": 1.5735072754641244,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.113898426379346e-06,
       "learning_rate": 5.443277019063311e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.05129337310791,
+      "logits/rejected": 8.804384231567383,
+      "logps/chosen": -42.770999908447266,
+      "logps/rejected": -44.222930908203125,
+      "loss": 0.0,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -42.770999908447266,
+      "rewards/margins": 1.4519296884536743,
+      "rewards/rejected": -44.222930908203125,
       "step": 2940
     },
     {
       "epoch": 1.5761833082455259,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.8471975020122704e-07,
       "learning_rate": 5.427762192125023e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.79440689086914,
+      "logits/rejected": 8.30298900604248,
+      "logps/chosen": -42.04811096191406,
+      "logps/rejected": -42.66920471191406,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -42.04811096191406,
+      "rewards/margins": 0.6210967302322388,
+      "rewards/rejected": -42.66920471191406,
       "step": 2945
     },
     {
       "epoch": 1.5788593410269276,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.551237060611357e-07,
       "learning_rate": 5.41224321503607e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.793144226074219,
+      "logits/rejected": 8.105385780334473,
+      "logps/chosen": -41.787071228027344,
+      "logps/rejected": -44.501731872558594,
+      "loss": 0.0,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -41.787071228027344,
+      "rewards/margins": 2.714656114578247,
+      "rewards/rejected": -44.501731872558594,
       "step": 2950
     },
     {
       "epoch": 1.5815353738083293,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 0.00012709212277160943,
       "learning_rate": 5.396720238361637e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.662508964538574,
+      "logits/rejected": 8.256942749023438,
+      "logps/chosen": -41.13361358642578,
+      "logps/rejected": -42.892234802246094,
+      "loss": 0.0,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -41.13361358642578,
+      "rewards/margins": 1.758623480796814,
+      "rewards/rejected": -42.892234802246094,
       "step": 2955
     },
     {
       "epoch": 1.5842114065897306,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.068168534742173e-06,
       "learning_rate": 5.381193412705711e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.76036262512207,
+      "logits/rejected": 8.307029724121094,
+      "logps/chosen": -40.78772735595703,
+      "logps/rejected": -41.676429748535156,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -40.78772735595703,
+      "rewards/margins": 0.8887056112289429,
+      "rewards/rejected": -41.676429748535156,
       "step": 2960
     },
     {
       "epoch": 1.5868874393711323,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.706116894915754e-09,
       "learning_rate": 5.365662888709622e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.910306930541992,
+      "logits/rejected": 8.499940872192383,
+      "logps/chosen": -41.561187744140625,
+      "logps/rejected": -42.84279251098633,
+      "loss": 0.0,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -41.561187744140625,
+      "rewards/margins": 1.2816061973571777,
+      "rewards/rejected": -42.84279251098633,
       "step": 2965
     },
     {
       "epoch": 1.589563472152534,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 5.775261032822579e-09,
       "learning_rate": 5.350128817050585e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.142675399780273,
+      "logits/rejected": 8.506601333618164,
+      "logps/chosen": -42.51160430908203,
+      "logps/rejected": -43.86318588256836,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -42.51160430908203,
+      "rewards/margins": 1.3515797853469849,
+      "rewards/rejected": -43.86318588256836,
       "step": 2970
     },
     {
       "epoch": 1.5922395049339353,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.6340854899170017e-09,
       "learning_rate": 5.334591348440229e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.863957405090332,
+      "logits/rejected": 8.378945350646973,
+      "logps/chosen": -41.649898529052734,
+      "logps/rejected": -42.979164123535156,
+      "loss": 0.0,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -41.649898529052734,
+      "rewards/margins": 1.3292641639709473,
+      "rewards/rejected": -42.979164123535156,
       "step": 2975
     },
     {
       "epoch": 1.594915537715337,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.7223451805564094e-09,
       "learning_rate": 5.319050633623141e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.876564025878906,
+      "logits/rejected": 8.450920104980469,
+      "logps/chosen": -41.80146026611328,
+      "logps/rejected": -43.48046875,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -41.80146026611328,
+      "rewards/margins": 1.6790072917938232,
+      "rewards/rejected": -43.48046875,
       "step": 2980
     },
     {
       "epoch": 1.5975915704967387,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 6.255182515691606e-10,
       "learning_rate": 5.303506823375409e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.682907104492188,
+      "logits/rejected": 8.112383842468262,
+      "logps/chosen": -41.28458023071289,
+      "logps/rejected": -43.36217498779297,
+      "loss": 0.0,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -41.28458023071289,
+      "rewards/margins": 2.07759690284729,
+      "rewards/rejected": -43.36217498779297,
       "step": 2985
     },
     {
       "epoch": 1.60026760327814,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.2504293400865813e-11,
       "learning_rate": 5.287960068503143e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.328814506530762,
+      "logits/rejected": 8.755146980285645,
+      "logps/chosen": -42.91911697387695,
+      "logps/rejected": -44.47560501098633,
+      "loss": 0.0,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -42.91911697387695,
+      "rewards/margins": 1.5564887523651123,
+      "rewards/rejected": -44.47560501098633,
       "step": 2990
     },
     {
       "epoch": 1.6029436360595417,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.319810814356769e-10,
       "learning_rate": 5.272410519841032e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.68800163269043,
+      "logits/rejected": 8.423990249633789,
+      "logps/chosen": -41.551368713378906,
+      "logps/rejected": -42.98131561279297,
+      "loss": 0.0,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -41.551368713378906,
+      "rewards/margins": 1.4299461841583252,
+      "rewards/rejected": -42.98131561279297,
       "step": 2995
     },
     {
       "epoch": 1.6056196688409434,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 0.00010166629368870996,
       "learning_rate": 5.256858328250861e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.132532119750977,
+      "logits/rejected": 8.534920692443848,
+      "logps/chosen": -42.26913070678711,
+      "logps/rejected": -43.59767150878906,
+      "loss": 0.0,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -42.26913070678711,
+      "rewards/margins": 1.328540563583374,
+      "rewards/rejected": -43.59767150878906,
       "step": 3000
     },
     {
       "epoch": 1.608295701622345,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.233041001418665e-07,
       "learning_rate": 5.241303644620063e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.308691024780273,
+      "logits/rejected": 8.77791976928711,
+      "logps/chosen": -43.174034118652344,
+      "logps/rejected": -43.86066818237305,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -43.174034118652344,
+      "rewards/margins": 0.6866382360458374,
+      "rewards/rejected": -43.86066818237305,
       "step": 3005
     },
     {
       "epoch": 1.6109717344037464,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.3349442928205522e-08,
       "learning_rate": 5.225746619860248e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.183420181274414,
+      "logits/rejected": 8.742216110229492,
+      "logps/chosen": -42.111778259277344,
+      "logps/rejected": -43.35272979736328,
+      "loss": 0.0,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -42.111778259277344,
+      "rewards/margins": 1.2409462928771973,
+      "rewards/rejected": -43.35272979736328,
       "step": 3010
     },
     {
       "epoch": 1.6136477671851481,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.0978312359132904e-05,
       "learning_rate": 5.210187404905735e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.602811813354492,
+      "logits/rejected": 8.375425338745117,
+      "logps/chosen": -43.508853912353516,
+      "logps/rejected": -43.21955108642578,
+      "loss": 0.0,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": -43.508853912353516,
+      "rewards/margins": -0.28930503129959106,
+      "rewards/rejected": -43.21955108642578,
       "step": 3015
     },
     {
       "epoch": 1.6163237999665496,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 8.028707284773641e-07,
       "learning_rate": 5.194626150712098e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.231069564819336,
+      "logits/rejected": 8.754720687866211,
+      "logps/chosen": -42.7666015625,
+      "logps/rejected": -43.946319580078125,
+      "loss": 0.0,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -42.7666015625,
+      "rewards/margins": 1.179720401763916,
+      "rewards/rejected": -43.946319580078125,
       "step": 3020
     },
     {
       "epoch": 1.6189998327479511,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.2500797381750772e-05,
       "learning_rate": 5.179063008254695e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.906530380249023,
+      "logits/rejected": 8.253819465637207,
+      "logps/chosen": -41.62322998046875,
+      "logps/rejected": -42.454429626464844,
+      "loss": 0.0,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -41.62322998046875,
+      "rewards/margins": 0.8311986923217773,
+      "rewards/rejected": -42.454429626464844,
       "step": 3025
     },
     {
       "epoch": 1.6216758655293528,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 0.2365918569251992,
       "learning_rate": 5.163498128527199e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.86352825164795,
+      "logits/rejected": 8.38776969909668,
+      "logps/chosen": -41.10457992553711,
+      "logps/rejected": -43.19374084472656,
+      "loss": 0.0004,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -41.10457992553711,
+      "rewards/margins": 2.089158296585083,
+      "rewards/rejected": -43.19374084472656,
       "step": 3030
     },
     {
       "epoch": 1.6243518983107543,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 9.665003025245416e-08,
       "learning_rate": 5.147931662540144e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.516685485839844,
+      "logits/rejected": 8.1073637008667,
+      "logps/chosen": -40.8177375793457,
+      "logps/rejected": -42.197906494140625,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -40.8177375793457,
+      "rewards/margins": 1.3801673650741577,
+      "rewards/rejected": -42.197906494140625,
       "step": 3035
     },
     {
       "epoch": 1.6270279310921558,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 8.338258397717798e-09,
       "learning_rate": 5.132363761319449e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.531752586364746,
+      "logits/rejected": 8.431139945983887,
+      "logps/chosen": -41.4906120300293,
+      "logps/rejected": -42.645660400390625,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -41.4906120300293,
+      "rewards/margins": 1.1550530195236206,
+      "rewards/rejected": -42.645660400390625,
       "step": 3040
     },
     {
       "epoch": 1.6297039638735575,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.982968074960198e-07,
       "learning_rate": 5.116794575904962e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.606687545776367,
+      "logits/rejected": 8.396788597106934,
+      "logps/chosen": -42.18719482421875,
+      "logps/rejected": -42.94538116455078,
+      "loss": 0.0,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -42.18719482421875,
+      "rewards/margins": 0.7581872344017029,
+      "rewards/rejected": -42.94538116455078,
       "step": 3045
     },
     {
       "epoch": 1.632379996654959,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 6.090119324707206e-08,
       "learning_rate": 5.101224257348987e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.812459945678711,
+      "logits/rejected": 8.619714736938477,
+      "logps/chosen": -41.88135528564453,
+      "logps/rejected": -42.764671325683594,
+      "loss": 0.0,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -41.88135528564453,
+      "rewards/margins": 0.8833135366439819,
+      "rewards/rejected": -42.764671325683594,
       "step": 3050
     },
     {
       "epoch": 1.6350560294363605,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 5.954460611989324e-07,
       "learning_rate": 5.085652956714823e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.324695587158203,
+      "logits/rejected": 8.874685287475586,
+      "logps/chosen": -42.326904296875,
+      "logps/rejected": -43.86783981323242,
+      "loss": 0.0,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -42.326904296875,
+      "rewards/margins": 1.5409380197525024,
+      "rewards/rejected": -43.86783981323242,
       "step": 3055
     },
     {
       "epoch": 1.6377320622177622,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 5.783703876784812e-06,
       "learning_rate": 5.070080825075298e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.880809783935547,
+      "logits/rejected": 8.35167121887207,
+      "logps/chosen": -41.295589447021484,
+      "logps/rejected": -42.723167419433594,
+      "loss": 0.0,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -41.295589447021484,
+      "rewards/margins": 1.4275810718536377,
+      "rewards/rejected": -42.723167419433594,
       "step": 3060
     },
     {
       "epoch": 1.6404080949991637,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.231494740385337e-11,
       "learning_rate": 5.0545080135113e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.833189010620117,
+      "logits/rejected": 8.609594345092773,
+      "logps/chosen": -43.12578582763672,
+      "logps/rejected": -44.05109405517578,
+      "loss": 0.0,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -43.12578582763672,
+      "rewards/margins": 0.9253073930740356,
+      "rewards/rejected": -44.05109405517578,
       "step": 3065
     },
     {
       "epoch": 1.6430841277805652,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.7928006753618e-07,
       "learning_rate": 5.038934673110316e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.685979843139648,
+      "logits/rejected": 8.558248519897461,
+      "logps/chosen": -41.523372650146484,
+      "logps/rejected": -42.68616485595703,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -41.523372650146484,
+      "rewards/margins": 1.1627862453460693,
+      "rewards/rejected": -42.68616485595703,
       "step": 3070
     },
     {
       "epoch": 1.645760160561967,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.930661480298808e-08,
       "learning_rate": 5.023360954964963e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.807648658752441,
+      "logits/rejected": 8.644891738891602,
+      "logps/chosen": -41.85204315185547,
+      "logps/rejected": -42.57401657104492,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -41.85204315185547,
+      "rewards/margins": 0.7219740152359009,
+      "rewards/rejected": -42.57401657104492,
       "step": 3075
     },
     {
       "epoch": 1.6484361933433684,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.9041645577364865e-07,
       "learning_rate": 5.007787010171524e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.918360710144043,
+      "logits/rejected": 8.456040382385254,
+      "logps/chosen": -41.612388610839844,
+      "logps/rejected": -42.90841293334961,
+      "loss": 0.0,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -41.612388610839844,
+      "rewards/margins": 1.2960256338119507,
+      "rewards/rejected": -42.90841293334961,
       "step": 3080
     },
     {
       "epoch": 1.65111222612477,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.4511974835885142e-10,
       "learning_rate": 4.992212989828477e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.849992752075195,
+      "logits/rejected": 8.772486686706543,
+      "logps/chosen": -43.92622756958008,
+      "logps/rejected": -43.822044372558594,
+      "loss": 0.0,
+      "rewards/accuracies": 0.48750001192092896,
+      "rewards/chosen": -43.92622756958008,
+      "rewards/margins": -0.10418035089969635,
+      "rewards/rejected": -43.822044372558594,
       "step": 3085
     },
     {
       "epoch": 1.6537882589061716,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.290444107710993e-07,
       "learning_rate": 4.976639045035036e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.790390968322754,
+      "logits/rejected": 8.568853378295898,
+      "logps/chosen": -43.10242462158203,
+      "logps/rejected": -44.56400680541992,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -43.10242462158203,
+      "rewards/margins": 1.461582064628601,
+      "rewards/rejected": -44.56400680541992,
       "step": 3090
     },
     {
       "epoch": 1.6564642916875731,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 0.255098012707323,
       "learning_rate": 4.961065326889683e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.562554359436035,
+      "logits/rejected": 8.066210746765137,
+      "logps/chosen": -41.56775665283203,
+      "logps/rejected": -43.64479064941406,
+      "loss": 0.0015,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -41.56775665283203,
+      "rewards/margins": 2.0770323276519775,
+      "rewards/rejected": -43.64479064941406,
       "step": 3095
     },
     {
       "epoch": 1.6591403244689746,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.558531275998947e-11,
       "learning_rate": 4.9454919864887e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.840633392333984,
+      "logits/rejected": 8.545955657958984,
+      "logps/chosen": -42.236541748046875,
+      "logps/rejected": -43.12220001220703,
+      "loss": 0.0,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -42.236541748046875,
+      "rewards/margins": 0.8856558799743652,
+      "rewards/rejected": -43.12220001220703,
       "step": 3100
     },
     {
       "epoch": 1.6618163572503764,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.7567761127007805e-11,
       "learning_rate": 4.929919174924701e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.287446022033691,
+      "logits/rejected": 8.558603286743164,
+      "logps/chosen": -41.95670700073242,
+      "logps/rejected": -43.232784271240234,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -41.95670700073242,
+      "rewards/margins": 1.2760719060897827,
+      "rewards/rejected": -43.232784271240234,
       "step": 3105
     },
     {
       "epoch": 1.6644923900317778,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.530764878293562e-10,
       "learning_rate": 4.914347043285177e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.711963653564453,
+      "logits/rejected": 8.42900276184082,
+      "logps/chosen": -41.76972579956055,
+      "logps/rejected": -43.00391387939453,
+      "loss": 0.0,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -41.76972579956055,
+      "rewards/margins": 1.2341920137405396,
+      "rewards/rejected": -43.00391387939453,
       "step": 3110
     },
     {
       "epoch": 1.6671684228131793,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.468041771541444e-06,
       "learning_rate": 4.898775742651013e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.626786231994629,
+      "logits/rejected": 8.341765403747559,
+      "logps/chosen": -42.278099060058594,
+      "logps/rejected": -43.66221237182617,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -42.278099060058594,
+      "rewards/margins": 1.3841116428375244,
+      "rewards/rejected": -43.66221237182617,
       "step": 3115
     },
     {
       "epoch": 1.669844455594581,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.222630235347758e-07,
       "learning_rate": 4.883205424095037e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.043296813964844,
+      "logits/rejected": 8.531747817993164,
+      "logps/chosen": -41.197139739990234,
+      "logps/rejected": -43.73224639892578,
+      "loss": 0.0,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -41.197139739990234,
+      "rewards/margins": 2.5351052284240723,
+      "rewards/rejected": -43.73224639892578,
       "step": 3120
     },
     {
       "epoch": 1.6725204883759828,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 0.007342146661988087,
       "learning_rate": 4.86763623868055e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.002099990844727,
+      "logits/rejected": 8.551979064941406,
+      "logps/chosen": -41.3134651184082,
+      "logps/rejected": -42.92992401123047,
+      "loss": 0.0,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -41.3134651184082,
+      "rewards/margins": 1.6164617538452148,
+      "rewards/rejected": -42.92992401123047,
       "step": 3125
     },
     {
       "epoch": 1.675196521157384,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.2857323561762575e-05,
       "learning_rate": 4.852068337459856e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.533288955688477,
+      "logits/rejected": 7.754534721374512,
+      "logps/chosen": -40.462135314941406,
+      "logps/rejected": -42.256534576416016,
+      "loss": 0.0,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -40.462135314941406,
+      "rewards/margins": 1.794398307800293,
+      "rewards/rejected": -42.256534576416016,
       "step": 3130
     },
     {
       "epoch": 1.6778725539387858,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.3326694082558964e-11,
       "learning_rate": 4.8365018714728e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.699747085571289,
+      "logits/rejected": 8.487561225891113,
+      "logps/chosen": -40.05158233642578,
+      "logps/rejected": -42.60285186767578,
+      "loss": 0.0,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -40.05158233642578,
+      "rewards/margins": 2.5512731075286865,
+      "rewards/rejected": -42.60285186767578,
       "step": 3135
     },
     {
       "epoch": 1.6805485867201875,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 0.005859951953461854,
       "learning_rate": 4.820936991745304e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.187182426452637,
+      "logits/rejected": 8.881650924682617,
+      "logps/chosen": -41.60440444946289,
+      "logps/rejected": -43.28955841064453,
+      "loss": 0.0,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -41.60440444946289,
+      "rewards/margins": 1.6851520538330078,
+      "rewards/rejected": -43.28955841064453,
       "step": 3140
     },
     {
       "epoch": 1.6832246195015887,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.798701663725085e-05,
       "learning_rate": 4.8053738492879e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.451224327087402,
+      "logits/rejected": 8.048616409301758,
+      "logps/chosen": -41.843589782714844,
+      "logps/rejected": -42.90123748779297,
+      "loss": 0.0,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -41.843589782714844,
+      "rewards/margins": 1.0576448440551758,
+      "rewards/rejected": -42.90123748779297,
       "step": 3145
     },
     {
       "epoch": 1.6859006522829905,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 0.28570398984484674,
       "learning_rate": 4.789812595094265e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.953600883483887,
+      "logits/rejected": 8.573225021362305,
+      "logps/chosen": -40.707984924316406,
+      "logps/rejected": -41.96226119995117,
+      "loss": 0.001,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -40.707984924316406,
+      "rewards/margins": 1.2542773485183716,
+      "rewards/rejected": -41.96226119995117,
       "step": 3150
     },
     {
       "epoch": 1.6885766850643922,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 5.159242648943684e-09,
       "learning_rate": 4.774253380139752e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.168846130371094,
+      "logits/rejected": 8.793378829956055,
+      "logps/chosen": -42.01328659057617,
+      "logps/rejected": -43.22798538208008,
+      "loss": 0.0006,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -42.01328659057617,
+      "rewards/margins": 1.2146974802017212,
+      "rewards/rejected": -43.22798538208008,
       "step": 3155
     },
     {
       "epoch": 1.6912527178457935,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.518944929336543e-08,
       "learning_rate": 4.758696355379936e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.72606372833252,
+      "logits/rejected": 8.94891357421875,
+      "logps/chosen": -42.74934768676758,
+      "logps/rejected": -43.51082992553711,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5062500238418579,
+      "rewards/chosen": -42.74934768676758,
+      "rewards/margins": 0.7614868879318237,
+      "rewards/rejected": -43.51082992553711,
       "step": 3160
     },
     {
       "epoch": 1.6939287506271952,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 5.6925332966308006e-08,
       "learning_rate": 4.743141671749138e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.195821762084961,
+      "logits/rejected": 8.750822067260742,
+      "logps/chosen": -40.8465576171875,
+      "logps/rejected": -41.42811965942383,
+      "loss": 0.0,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -40.8465576171875,
+      "rewards/margins": 0.58156818151474,
+      "rewards/rejected": -41.42811965942383,
       "step": 3165
     },
     {
       "epoch": 1.6966047834085969,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.879495282623498e-06,
       "learning_rate": 4.727589480158968e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.07165813446045,
+      "logits/rejected": 8.73912239074707,
+      "logps/chosen": -42.56293869018555,
+      "logps/rejected": -44.23527526855469,
+      "loss": 0.0,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -42.56293869018555,
+      "rewards/margins": 1.6723308563232422,
+      "rewards/rejected": -44.23527526855469,
       "step": 3170
     },
     {
       "epoch": 1.6992808161899984,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.5730025945502565e-06,
       "learning_rate": 4.712039931496855e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.848831176757812,
+      "logits/rejected": 8.2403564453125,
+      "logps/chosen": -42.49853515625,
+      "logps/rejected": -43.76799011230469,
+      "loss": 0.0,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -42.49853515625,
+      "rewards/margins": 1.2694507837295532,
+      "rewards/rejected": -43.76799011230469,
       "step": 3175
     },
     {
       "epoch": 1.7019568489713999,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.1691293002224624e-08,
       "learning_rate": 4.6964931766245905e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.707847595214844,
+      "logits/rejected": 8.587430953979492,
+      "logps/chosen": -42.651466369628906,
+      "logps/rejected": -42.958961486816406,
+      "loss": 0.0,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -42.651466369628906,
+      "rewards/margins": 0.30750173330307007,
+      "rewards/rejected": -42.958961486816406,
       "step": 3180
     },
     {
       "epoch": 1.7046328817528016,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 6.2873077178589455e-06,
       "learning_rate": 4.6809493663768575e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.599573135375977,
+      "logits/rejected": 8.256446838378906,
+      "logps/chosen": -43.25920104980469,
+      "logps/rejected": -43.644981384277344,
+      "loss": 0.0,
+      "rewards/accuracies": 0.4749999940395355,
+      "rewards/chosen": -43.25920104980469,
+      "rewards/margins": 0.3857851028442383,
+      "rewards/rejected": -43.644981384277344,
       "step": 3185
     },
     {
       "epoch": 1.707308914534203,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.4720259755373406e-10,
       "learning_rate": 4.6654086515597716e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.07559585571289,
+      "logits/rejected": 8.394869804382324,
+      "logps/chosen": -41.30718994140625,
+      "logps/rejected": -43.453346252441406,
+      "loss": 0.0,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -41.30718994140625,
+      "rewards/margins": 2.1461596488952637,
+      "rewards/rejected": -43.453346252441406,
       "step": 3190
     },
     {
       "epoch": 1.7099849473156046,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 0.15178583958928252,
       "learning_rate": 4.6498711829494154e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.776254653930664,
+      "logits/rejected": 8.530635833740234,
+      "logps/chosen": -41.66889953613281,
+      "logps/rejected": -42.503753662109375,
+      "loss": 0.0006,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -41.66889953613281,
+      "rewards/margins": 0.8348552584648132,
+      "rewards/rejected": -42.503753662109375,
       "step": 3195
     },
     {
       "epoch": 1.7126609800970063,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.4032553875680568e-10,
       "learning_rate": 4.6343371112903777e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.957771301269531,
+      "logits/rejected": 8.334024429321289,
+      "logps/chosen": -42.99253463745117,
+      "logps/rejected": -43.53425216674805,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -42.99253463745117,
+      "rewards/margins": 0.5417177677154541,
+      "rewards/rejected": -43.53425216674805,
       "step": 3200
     },
     {
       "epoch": 1.7126609800970063,
-      "eval_logits/chosen": NaN,
-      "eval_logits/rejected": NaN,
-      "eval_logps/chosen": NaN,
-      "eval_logps/rejected": NaN,
-      "eval_loss": NaN,
-      "eval_rewards/accuracies": 0.0,
-      "eval_rewards/chosen": NaN,
-      "eval_rewards/margins": NaN,
-      "eval_rewards/rejected": NaN,
-      "eval_runtime": 35.2595,
-      "eval_samples_per_second": 38.146,
-      "eval_semantic_entropy": NaN,
-      "eval_steps_per_second": 9.558,
+      "eval_logits/chosen": 7.346385478973389,
+      "eval_logits/rejected": 7.090656757354736,
+      "eval_logps/chosen": -41.73725509643555,
+      "eval_logps/rejected": -42.9487419128418,
+      "eval_loss": 2.6891856919064594e-07,
+      "eval_rewards/accuracies": 0.5675074458122253,
+      "eval_rewards/chosen": -41.73725509643555,
+      "eval_rewards/margins": 1.211488127708435,
+      "eval_rewards/rejected": -42.9487419128418,
+      "eval_runtime": 40.2607,
+      "eval_samples_per_second": 33.407,
+      "eval_steps_per_second": 8.37,
       "step": 3200
     },
     {
       "epoch": 1.7153370128784078,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 8.079141973703885e-09,
       "learning_rate": 4.618806587294291e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.810081481933594,
+      "logits/rejected": 8.665169715881348,
+      "logps/chosen": -41.76563262939453,
+      "logps/rejected": -43.30455780029297,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -41.76563262939453,
+      "rewards/margins": 1.5389270782470703,
+      "rewards/rejected": -43.30455780029297,
       "step": 3205
     },
     {
       "epoch": 1.7180130456598093,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 0.0005438265096157191,
       "learning_rate": 4.603279761638365e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.74320125579834,
+      "logits/rejected": 8.397790908813477,
+      "logps/chosen": -42.464683532714844,
+      "logps/rejected": -43.62255096435547,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -42.464683532714844,
+      "rewards/margins": 1.1578786373138428,
+      "rewards/rejected": -43.62255096435547,
       "step": 3210
     },
     {
       "epoch": 1.720689078441211,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 9.406511668348795e-09,
       "learning_rate": 4.5877567849639315e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.017813682556152,
+      "logits/rejected": 8.773551940917969,
+      "logps/chosen": -42.185218811035156,
+      "logps/rejected": -43.79137420654297,
+      "loss": 0.0,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -42.185218811035156,
+      "rewards/margins": 1.606148362159729,
+      "rewards/rejected": -43.79137420654297,
       "step": 3215
     },
     {
       "epoch": 1.7233651112226125,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 9.8056721958456e-10,
       "learning_rate": 4.572237807874979e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.223387718200684,
+      "logits/rejected": 8.160594940185547,
+      "logps/chosen": -41.920066833496094,
+      "logps/rejected": -43.96269989013672,
+      "loss": 0.0,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -41.920066833496094,
+      "rewards/margins": 2.0426363945007324,
+      "rewards/rejected": -43.96269989013672,
       "step": 3220
     },
     {
       "epoch": 1.726041144004014,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.674313073824762e-07,
       "learning_rate": 4.5567229809366895e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.829940795898438,
+      "logits/rejected": 8.46506118774414,
+      "logps/chosen": -42.271976470947266,
+      "logps/rejected": -43.912628173828125,
+      "loss": 0.0,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -42.271976470947266,
+      "rewards/margins": 1.6406564712524414,
+      "rewards/rejected": -43.912628173828125,
       "step": 3225
     },
     {
       "epoch": 1.7287171767854157,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.4514295156358748e-07,
       "learning_rate": 4.541212454673984e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.992741584777832,
+      "logits/rejected": 8.347302436828613,
+      "logps/chosen": -42.49037551879883,
+      "logps/rejected": -44.81675720214844,
+      "loss": 0.0,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -42.49037551879883,
+      "rewards/margins": 2.3263790607452393,
+      "rewards/rejected": -44.81675720214844,
       "step": 3230
     },
     {
       "epoch": 1.7313932095668172,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 7.571709446908636e-08,
       "learning_rate": 4.525706379570055e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.3079252243042,
+      "logits/rejected": 8.997198104858398,
+      "logps/chosen": -41.79115676879883,
+      "logps/rejected": -43.184913635253906,
+      "loss": 0.0002,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -41.79115676879883,
+      "rewards/margins": 1.3937630653381348,
+      "rewards/rejected": -43.184913635253906,
       "step": 3235
     },
     {
       "epoch": 1.7340692423482187,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.9759526482329537e-07,
       "learning_rate": 4.510204906064911e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.885148048400879,
+      "logits/rejected": 8.273990631103516,
+      "logps/chosen": -41.427982330322266,
+      "logps/rejected": -43.95401382446289,
+      "loss": 0.0,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -41.427982330322266,
+      "rewards/margins": 2.5260348320007324,
+      "rewards/rejected": -43.95401382446289,
       "step": 3240
     },
     {
       "epoch": 1.7367452751296204,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.6378500244696348e-05,
       "learning_rate": 4.4947081845539177e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.159140586853027,
+      "logits/rejected": 8.722005844116211,
+      "logps/chosen": -42.572532653808594,
+      "logps/rejected": -43.89950180053711,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -42.572532653808594,
+      "rewards/margins": 1.326970100402832,
+      "rewards/rejected": -43.89950180053711,
       "step": 3245
     },
     {
       "epoch": 1.739421307911022,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.3489562044606567e-06,
       "learning_rate": 4.479216365386333e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.948904037475586,
+      "logits/rejected": 8.283456802368164,
+      "logps/chosen": -41.316383361816406,
+      "logps/rejected": -43.39640808105469,
+      "loss": 0.0,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -41.316383361816406,
+      "rewards/margins": 2.0800294876098633,
+      "rewards/rejected": -43.39640808105469,
       "step": 3250
     },
     {
       "epoch": 1.7420973406924234,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.0662338456938564e-08,
       "learning_rate": 4.4637295988638555e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.030210494995117,
+      "logits/rejected": 8.936519622802734,
+      "logps/chosen": -41.05961990356445,
+      "logps/rejected": -43.70038604736328,
+      "loss": 0.0,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -41.05961990356445,
+      "rewards/margins": 2.6407690048217773,
+      "rewards/rejected": -43.70038604736328,
       "step": 3255
     },
     {
       "epoch": 1.744773373473825,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.379130417332738e-05,
       "learning_rate": 4.4482480352391623e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.51530647277832,
+      "logits/rejected": 8.508938789367676,
+      "logps/chosen": -41.11503219604492,
+      "logps/rejected": -42.8921012878418,
+      "loss": 0.0,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -41.11503219604492,
+      "rewards/margins": 1.7770659923553467,
+      "rewards/rejected": -42.8921012878418,
       "step": 3260
     },
     {
       "epoch": 1.7474494062552266,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.0016449783260726e-11,
       "learning_rate": 4.4327718247144507e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.70158576965332,
+      "logits/rejected": 8.335428237915039,
+      "logps/chosen": -42.67002868652344,
+      "logps/rejected": -43.72126007080078,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -42.67002868652344,
+      "rewards/margins": 1.0512341260910034,
+      "rewards/rejected": -43.72126007080078,
       "step": 3265
     },
     {
       "epoch": 1.750125439036628,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.2840676079622715e-05,
       "learning_rate": 4.417301117439984e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.127232551574707,
+      "logits/rejected": 8.941888809204102,
+      "logps/chosen": -41.003997802734375,
+      "logps/rejected": -42.68315505981445,
+      "loss": 0.0,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -41.003997802734375,
+      "rewards/margins": 1.6791555881500244,
+      "rewards/rejected": -42.68315505981445,
       "step": 3270
     },
     {
       "epoch": 1.7528014718180298,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.628372060347152e-11,
       "learning_rate": 4.401836063512631e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.382959365844727,
+      "logits/rejected": 8.315888404846191,
+      "logps/chosen": -43.123863220214844,
+      "logps/rejected": -44.72638702392578,
+      "loss": 0.0003,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -43.123863220214844,
+      "rewards/margins": 1.6025243997573853,
+      "rewards/rejected": -44.72638702392578,
       "step": 3275
     },
     {
       "epoch": 1.7554775045994313,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.5167838552479552e-12,
       "learning_rate": 4.386376812974413e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.1256742477417,
+      "logits/rejected": 9.009529113769531,
+      "logps/chosen": -44.24250030517578,
+      "logps/rejected": -45.69837188720703,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -44.24250030517578,
+      "rewards/margins": 1.455871820449829,
+      "rewards/rejected": -45.69837188720703,
       "step": 3280
     },
     {
       "epoch": 1.7581535373808328,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.2631180011183867e-07,
       "learning_rate": 4.370923515811048e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.232044219970703,
+      "logits/rejected": 8.624814987182617,
+      "logps/chosen": -42.31184768676758,
+      "logps/rejected": -43.475624084472656,
+      "loss": 0.0,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -42.31184768676758,
+      "rewards/margins": 1.1637694835662842,
+      "rewards/rejected": -43.475624084472656,
       "step": 3285
     },
     {
       "epoch": 1.7608295701622345,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.660488176309308e-07,
       "learning_rate": 4.35547632195049e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.911782264709473,
+      "logits/rejected": 8.576136589050293,
+      "logps/chosen": -43.006004333496094,
+      "logps/rejected": -43.88723373413086,
+      "loss": 0.0,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -43.006004333496094,
+      "rewards/margins": 0.8812358975410461,
+      "rewards/rejected": -43.88723373413086,
       "step": 3290
     },
     {
       "epoch": 1.763505602943636,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.725766750356587e-11,
       "learning_rate": 4.340035381261484e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.171948432922363,
+      "logits/rejected": 8.768861770629883,
+      "logps/chosen": -43.28553009033203,
+      "logps/rejected": -44.162353515625,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -43.28553009033203,
+      "rewards/margins": 0.8768193125724792,
+      "rewards/rejected": -44.162353515625,
       "step": 3295
     },
     {
       "epoch": 1.7661816357250375,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.3895684667074638e-08,
       "learning_rate": 4.324600843552104e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.119513511657715,
+      "logits/rejected": 8.699051856994629,
+      "logps/chosen": -42.70151901245117,
+      "logps/rejected": -43.992042541503906,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -42.70151901245117,
+      "rewards/margins": 1.2905250787734985,
+      "rewards/rejected": -43.992042541503906,
       "step": 3300
     },
     {
       "epoch": 1.7688576685064392,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 0.0001982070200975259,
       "learning_rate": 4.309172858568302e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.971323013305664,
+      "logits/rejected": 8.495769500732422,
+      "logps/chosen": -42.44605255126953,
+      "logps/rejected": -43.94598388671875,
+      "loss": 0.0,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -42.44605255126953,
+      "rewards/margins": 1.4999330043792725,
+      "rewards/rejected": -43.94598388671875,
       "step": 3305
     },
     {
       "epoch": 1.771533701287841,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.528499885958161e-10,
       "learning_rate": 4.293751575992455e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.718727111816406,
+      "logits/rejected": 8.726655960083008,
+      "logps/chosen": -42.47847366333008,
+      "logps/rejected": -43.79438400268555,
+      "loss": 0.0,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -42.47847366333008,
+      "rewards/margins": 1.3159105777740479,
+      "rewards/rejected": -43.79438400268555,
       "step": 3310
     },
     {
       "epoch": 1.7742097340692422,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.3055398504239128e-05,
       "learning_rate": 4.278337145441916e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.055317878723145,
+      "logits/rejected": 8.470276832580566,
+      "logps/chosen": -41.76118850708008,
+      "logps/rejected": -42.783653259277344,
+      "loss": 0.0,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -41.76118850708008,
+      "rewards/margins": 1.0224684476852417,
+      "rewards/rejected": -42.783653259277344,
       "step": 3315
     },
     {
       "epoch": 1.776885766850644,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.468722492300036e-08,
       "learning_rate": 4.262929716467556e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.097979545593262,
+      "logits/rejected": 8.224995613098145,
+      "logps/chosen": -42.98595428466797,
+      "logps/rejected": -44.47277069091797,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -42.98595428466797,
+      "rewards/margins": 1.4868137836456299,
+      "rewards/rejected": -44.47277069091797,
       "step": 3320
     },
     {
       "epoch": 1.7795617996320456,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.573175488908807e-11,
       "learning_rate": 4.247529438552321e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.045320510864258,
+      "logits/rejected": 8.452642440795898,
+      "logps/chosen": -41.813297271728516,
+      "logps/rejected": -42.73538589477539,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -41.813297271728516,
+      "rewards/margins": 0.9220888018608093,
+      "rewards/rejected": -42.73538589477539,
       "step": 3325
     },
     {
       "epoch": 1.782237832413447,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 7.361206936683324e-07,
       "learning_rate": 4.232136461109773e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.049830436706543,
+      "logits/rejected": 8.575239181518555,
+      "logps/chosen": -42.71787643432617,
+      "logps/rejected": -43.28335952758789,
+      "loss": 0.0,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -42.71787643432617,
+      "rewards/margins": 0.5654757618904114,
+      "rewards/rejected": -43.28335952758789,
       "step": 3330
     },
     {
       "epoch": 1.7849138651948486,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 0.08316199103227653,
       "learning_rate": 4.216750933482646e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.697958946228027,
+      "logits/rejected": 8.333512306213379,
+      "logps/chosen": -42.26152801513672,
+      "logps/rejected": -43.87995910644531,
+      "loss": 0.0007,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -42.26152801513672,
+      "rewards/margins": 1.6184282302856445,
+      "rewards/rejected": -43.87995910644531,
       "step": 3335
     },
     {
       "epoch": 1.7875898979762503,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.0258879061185094e-07,
       "learning_rate": 4.2013730049413986e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.910024642944336,
+      "logits/rejected": 8.129594802856445,
+      "logps/chosen": -41.321998596191406,
+      "logps/rejected": -43.52183151245117,
+      "loss": 0.0,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -41.321998596191406,
+      "rewards/margins": 2.19983172416687,
+      "rewards/rejected": -43.52183151245117,
       "step": 3340
     },
     {
       "epoch": 1.7902659307576518,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.7136050765706833e-07,
       "learning_rate": 4.1860028246827594e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.940666198730469,
+      "logits/rejected": 8.155450820922852,
+      "logps/chosen": -43.48246765136719,
+      "logps/rejected": -44.367820739746094,
+      "loss": 0.0,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -43.48246765136719,
+      "rewards/margins": 0.8853505849838257,
+      "rewards/rejected": -44.367820739746094,
       "step": 3345
     },
     {
       "epoch": 1.7929419635390533,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.1149191386631303e-09,
       "learning_rate": 4.170640541828285e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.452614784240723,
+      "logits/rejected": 9.082446098327637,
+      "logps/chosen": -42.882118225097656,
+      "logps/rejected": -43.46571350097656,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -42.882118225097656,
+      "rewards/margins": 0.5835954546928406,
+      "rewards/rejected": -43.46571350097656,
       "step": 3350
     },
     {
       "epoch": 1.795617996320455,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.0719172991565895e-10,
       "learning_rate": 4.1552863054229116e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.989095687866211,
+      "logits/rejected": 8.857994079589844,
+      "logps/chosen": -42.419124603271484,
+      "logps/rejected": -44.075111389160156,
+      "loss": 0.0,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -42.419124603271484,
+      "rewards/margins": 1.6559826135635376,
+      "rewards/rejected": -44.075111389160156,
       "step": 3355
     },
     {
       "epoch": 1.7982940291018565,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.9478594166596984e-05,
       "learning_rate": 4.139940264433508e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.005237579345703,
+      "logits/rejected": 8.315083503723145,
+      "logps/chosen": -43.112281799316406,
+      "logps/rejected": -44.65152359008789,
+      "loss": 0.0,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -43.112281799316406,
+      "rewards/margins": 1.5392391681671143,
+      "rewards/rejected": -44.65152359008789,
       "step": 3360
     },
     {
       "epoch": 1.800970061883258,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.87954822593384e-09,
       "learning_rate": 4.1246025677474303e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.404446601867676,
+      "logits/rejected": 8.712800979614258,
+      "logps/chosen": -42.869224548339844,
+      "logps/rejected": -44.20194625854492,
+      "loss": 0.0,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -42.869224548339844,
+      "rewards/margins": 1.3327208757400513,
+      "rewards/rejected": -44.20194625854492,
       "step": 3365
     },
     {
       "epoch": 1.8036460946646597,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 7.400907333812676e-10,
       "learning_rate": 4.10927336417108e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.088297843933105,
+      "logits/rejected": 8.417060852050781,
+      "logps/chosen": -42.23302459716797,
+      "logps/rejected": -43.39586639404297,
+      "loss": 0.0,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -42.23302459716797,
+      "rewards/margins": 1.1628367900848389,
+      "rewards/rejected": -43.39586639404297,
       "step": 3370
     },
     {
       "epoch": 1.8063221274460612,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 5.508248979305815e-07,
       "learning_rate": 4.093952802428457e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.810931205749512,
+      "logits/rejected": 8.732718467712402,
+      "logps/chosen": -42.82012939453125,
+      "logps/rejected": -43.050628662109375,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -42.82012939453125,
+      "rewards/margins": 0.23050299286842346,
+      "rewards/rejected": -43.050628662109375,
       "step": 3375
     },
     {
       "epoch": 1.8089981602274627,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.867034886440815e-11,
       "learning_rate": 4.0786410311597184e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.028701782226562,
+      "logits/rejected": 8.337693214416504,
+      "logps/chosen": -41.94820022583008,
+      "logps/rejected": -42.89849090576172,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -41.94820022583008,
+      "rewards/margins": 0.9502902030944824,
+      "rewards/rejected": -42.89849090576172,
       "step": 3380
     },
     {
       "epoch": 1.8116741930088645,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 7.006047306199805e-08,
       "learning_rate": 4.063338198919737e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.156309127807617,
+      "logits/rejected": 9.12796688079834,
+      "logps/chosen": -42.02606964111328,
+      "logps/rejected": -42.69218826293945,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5062500238418579,
+      "rewards/chosen": -42.02606964111328,
+      "rewards/margins": 0.6661237478256226,
+      "rewards/rejected": -42.69218826293945,
       "step": 3385
     },
     {
       "epoch": 1.814350225790266,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 6.430880006939129e-13,
       "learning_rate": 4.0480444541766575e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.370391845703125,
+      "logits/rejected": 8.873638153076172,
+      "logps/chosen": -43.99597930908203,
+      "logps/rejected": -45.247283935546875,
+      "loss": 0.0,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -43.99597930908203,
+      "rewards/margins": 1.2513020038604736,
+      "rewards/rejected": -45.247283935546875,
       "step": 3390
     },
     {
       "epoch": 1.8170262585716674,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.104439514603226e-10,
       "learning_rate": 4.0327599453104606e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.182734489440918,
+      "logits/rejected": 8.787361145019531,
+      "logps/chosen": -41.466064453125,
+      "logps/rejected": -43.58794403076172,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -41.466064453125,
+      "rewards/margins": 2.1218810081481934,
+      "rewards/rejected": -43.58794403076172,
       "step": 3395
     },
     {
       "epoch": 1.8197022913530692,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 7.199554098129442e-08,
       "learning_rate": 4.017484820611514e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.849800109863281,
+      "logits/rejected": 8.342534065246582,
+      "logps/chosen": -42.05352020263672,
+      "logps/rejected": -43.215274810791016,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -42.05352020263672,
+      "rewards/margins": 1.1617573499679565,
+      "rewards/rejected": -43.215274810791016,
       "step": 3400
     },
     {
       "epoch": 1.8223783241344707,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.0734951778870868e-09,
       "learning_rate": 4.002219228279148e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.902721405029297,
+      "logits/rejected": 8.534134864807129,
+      "logps/chosen": -42.02064895629883,
+      "logps/rejected": -43.605655670166016,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -42.02064895629883,
+      "rewards/margins": 1.5850058794021606,
+      "rewards/rejected": -43.605655670166016,
       "step": 3405
     },
     {
       "epoch": 1.8250543569158721,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.655388047059721e-06,
       "learning_rate": 3.9869633164202045e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.266801834106445,
+      "logits/rejected": 8.2295503616333,
+      "logps/chosen": -41.05113983154297,
+      "logps/rejected": -42.347869873046875,
+      "loss": 0.0001,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -41.05113983154297,
+      "rewards/margins": 1.2967309951782227,
+      "rewards/rejected": -42.347869873046875,
       "step": 3410
     },
     {
       "epoch": 1.8277303896972739,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 5.081088193193098e-07,
       "learning_rate": 3.9717172330476077e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.28183364868164,
+      "logits/rejected": 8.853327751159668,
+      "logps/chosen": -42.637420654296875,
+      "logps/rejected": -43.296817779541016,
+      "loss": 0.0,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -42.637420654296875,
+      "rewards/margins": 0.6593946218490601,
+      "rewards/rejected": -43.296817779541016,
       "step": 3415
     },
     {
       "epoch": 1.8304064224786754,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 7.796237438434782e-07,
       "learning_rate": 3.956481126078927e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.982412338256836,
+      "logits/rejected": 8.659948348999023,
+      "logps/chosen": -43.46683883666992,
+      "logps/rejected": -43.597984313964844,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -43.46683883666992,
+      "rewards/margins": 0.13114602863788605,
+      "rewards/rejected": -43.597984313964844,
       "step": 3420
     },
     {
       "epoch": 1.8330824552600768,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.6506318648743666e-10,
       "learning_rate": 3.941255143334937e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.835341453552246,
+      "logits/rejected": 8.636839866638184,
+      "logps/chosen": -42.88193893432617,
+      "logps/rejected": -43.88673400878906,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -42.88193893432617,
+      "rewards/margins": 1.0047924518585205,
+      "rewards/rejected": -43.88673400878906,
       "step": 3425
     },
     {
       "epoch": 1.8357584880414786,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 0.0002164451830728961,
       "learning_rate": 3.9260394325381895e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.640530586242676,
+      "logits/rejected": 8.23536491394043,
+      "logps/chosen": -41.761924743652344,
+      "logps/rejected": -43.662899017333984,
+      "loss": 0.0,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -41.761924743652344,
+      "rewards/margins": 1.9009778499603271,
+      "rewards/rejected": -43.662899017333984,
       "step": 3430
     },
     {
       "epoch": 1.83843452082288,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.359026558039456e-09,
       "learning_rate": 3.9108341413115784e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.89246654510498,
+      "logits/rejected": 8.473149299621582,
+      "logps/chosen": -41.694679260253906,
+      "logps/rejected": -42.69411087036133,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -41.694679260253906,
+      "rewards/margins": 0.9994314908981323,
+      "rewards/rejected": -42.69411087036133,
       "step": 3435
     },
     {
       "epoch": 1.8411105536042816,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.754802318560265e-11,
       "learning_rate": 3.895639417176905e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.135909080505371,
+      "logits/rejected": 8.80331802368164,
+      "logps/chosen": -43.21285629272461,
+      "logps/rejected": -44.86457824707031,
+      "loss": 0.0,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -43.21285629272461,
+      "rewards/margins": 1.651720404624939,
+      "rewards/rejected": -44.86457824707031,
       "step": 3440
     },
     {
       "epoch": 1.8437865863856833,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 5.367246114941999e-07,
       "learning_rate": 3.8804554075534497e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.278481483459473,
+      "logits/rejected": 8.533929824829102,
+      "logps/chosen": -42.80669403076172,
+      "logps/rejected": -44.85102081298828,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -42.80669403076172,
+      "rewards/margins": 2.0443243980407715,
+      "rewards/rejected": -44.85102081298828,
       "step": 3445
     },
     {
       "epoch": 1.8464626191670848,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.1818562359115935e-10,
       "learning_rate": 3.8652822597565403e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.049799919128418,
+      "logits/rejected": 8.538946151733398,
+      "logps/chosen": -42.586944580078125,
+      "logps/rejected": -44.1765022277832,
+      "loss": 0.0,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -42.586944580078125,
+      "rewards/margins": 1.5895531177520752,
+      "rewards/rejected": -44.1765022277832,
       "step": 3450
     },
     {
       "epoch": 1.8491386519484863,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.064864283126891e-09,
       "learning_rate": 3.850120120996123e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.163946151733398,
+      "logits/rejected": 8.482351303100586,
+      "logps/chosen": -42.87347412109375,
+      "logps/rejected": -44.42333984375,
+      "loss": 0.0,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -42.87347412109375,
+      "rewards/margins": 1.549865961074829,
+      "rewards/rejected": -44.42333984375,
       "step": 3455
     },
     {
       "epoch": 1.851814684729888,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.786478618743661e-10,
       "learning_rate": 3.8349691383753356e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.889766693115234,
+      "logits/rejected": 8.509964942932129,
+      "logps/chosen": -43.50269317626953,
+      "logps/rejected": -43.71992874145508,
+      "loss": 0.0,
+      "rewards/accuracies": 0.4937500059604645,
+      "rewards/chosen": -43.50269317626953,
+      "rewards/margins": 0.2172294557094574,
+      "rewards/rejected": -43.71992874145508,
       "step": 3460
     },
     {
       "epoch": 1.8544907175112895,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 5.7483043963092624e-08,
       "learning_rate": 3.819829458889078e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.000165939331055,
+      "logits/rejected": 8.670591354370117,
+      "logps/chosen": -41.933738708496094,
+      "logps/rejected": -43.36604309082031,
+      "loss": 0.0,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -41.933738708496094,
+      "rewards/margins": 1.432308316230774,
+      "rewards/rejected": -43.36604309082031,
       "step": 3465
     },
     {
       "epoch": 1.857166750292691,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 5.884668307367081e-07,
       "learning_rate": 3.804701229422585e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.101262092590332,
+      "logits/rejected": 8.968666076660156,
+      "logps/chosen": -42.483253479003906,
+      "logps/rejected": -43.319339752197266,
+      "loss": 0.0011,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -42.483253479003906,
+      "rewards/margins": 0.8360843658447266,
+      "rewards/rejected": -43.319339752197266,
       "step": 3470
     },
     {
       "epoch": 1.8598427830740927,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.549068521775006e-07,
       "learning_rate": 3.789584596750007e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.936643600463867,
+      "logits/rejected": 8.723246574401855,
+      "logps/chosen": -42.338253021240234,
+      "logps/rejected": -43.11792755126953,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -42.338253021240234,
+      "rewards/margins": 0.7796715497970581,
+      "rewards/rejected": -43.11792755126953,
       "step": 3475
     },
     {
       "epoch": 1.8625188158554944,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.6126903926992024e-09,
       "learning_rate": 3.77447970753298e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.634635925292969,
+      "logits/rejected": 8.493051528930664,
+      "logps/chosen": -43.30158233642578,
+      "logps/rejected": -43.481624603271484,
+      "loss": 0.0,
+      "rewards/accuracies": 0.46875,
+      "rewards/chosen": -43.30158233642578,
+      "rewards/margins": 0.18004471063613892,
+      "rewards/rejected": -43.481624603271484,
       "step": 3480
     },
     {
       "epoch": 1.8651948486368957,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 8.27703423016579e-06,
       "learning_rate": 3.7593867083192057e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.836604118347168,
+      "logits/rejected": 8.464637756347656,
+      "logps/chosen": -43.114402770996094,
+      "logps/rejected": -44.425872802734375,
+      "loss": 0.0,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -43.114402770996094,
+      "rewards/margins": 1.3114697933197021,
+      "rewards/rejected": -44.425872802734375,
       "step": 3485
     },
     {
       "epoch": 1.8678708814182974,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.175922808413568e-08,
       "learning_rate": 3.7443057455410276e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.917654037475586,
+      "logits/rejected": 8.54902458190918,
+      "logps/chosen": -41.48270797729492,
+      "logps/rejected": -43.69658660888672,
+      "loss": 0.0,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -41.48270797729492,
+      "rewards/margins": 2.213876485824585,
+      "rewards/rejected": -43.69658660888672,
       "step": 3490
     },
     {
       "epoch": 1.870546914199699,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.038271582546916e-12,
       "learning_rate": 3.7292369655140145e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.515264511108398,
+      "logits/rejected": 8.955233573913574,
+      "logps/chosen": -43.51061248779297,
+      "logps/rejected": -44.70892333984375,
+      "loss": 0.0,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -43.51061248779297,
+      "rewards/margins": 1.1983106136322021,
+      "rewards/rejected": -44.70892333984375,
       "step": 3495
     },
     {
       "epoch": 1.8732229469811004,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 5.474106460186873e-06,
       "learning_rate": 3.714180514435534e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.58936882019043,
+      "logits/rejected": 8.22518253326416,
+      "logps/chosen": -42.50271224975586,
+      "logps/rejected": -43.35732650756836,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -42.50271224975586,
+      "rewards/margins": 0.8546162843704224,
+      "rewards/rejected": -43.35732650756836,
       "step": 3500
     },
     {
       "epoch": 1.875898979762502,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.0473348054224673e-11,
       "learning_rate": 3.6991365383833426e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.062959671020508,
+      "logits/rejected": 8.590060234069824,
+      "logps/chosen": -42.446285247802734,
+      "logps/rejected": -44.069419860839844,
+      "loss": 0.0,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -42.446285247802734,
+      "rewards/margins": 1.6231279373168945,
+      "rewards/rejected": -44.069419860839844,
       "step": 3505
     },
     {
       "epoch": 1.8785750125439038,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.3398116243221677e-07,
       "learning_rate": 3.684105183314162e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.43688678741455,
+      "logits/rejected": 8.930704116821289,
+      "logps/chosen": -43.761383056640625,
+      "logps/rejected": -44.966285705566406,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -43.761383056640625,
+      "rewards/margins": 1.204900860786438,
+      "rewards/rejected": -44.966285705566406,
       "step": 3510
     },
     {
       "epoch": 1.881251045325305,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 8.336248701066181e-08,
       "learning_rate": 3.669086595062263e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.222817420959473,
+      "logits/rejected": 8.599294662475586,
+      "logps/chosen": -41.42193603515625,
+      "logps/rejected": -43.319358825683594,
+      "loss": 0.0,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -41.42193603515625,
+      "rewards/margins": 1.8974149227142334,
+      "rewards/rejected": -43.319358825683594,
       "step": 3515
     },
     {
       "epoch": 1.8839270781067068,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.99134572985627e-12,
       "learning_rate": 3.654080919338056e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.26179313659668,
+      "logits/rejected": 8.471601486206055,
+      "logps/chosen": -44.294681549072266,
+      "logps/rejected": -44.31082534790039,
+      "loss": 0.0,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -44.294681549072266,
+      "rewards/margins": 0.016147326678037643,
+      "rewards/rejected": -44.31082534790039,
       "step": 3520
     },
     {
       "epoch": 1.8866031108881085,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.744513978095259e-08,
       "learning_rate": 3.639088301726673e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.830717086791992,
+      "logits/rejected": 8.207534790039062,
+      "logps/chosen": -42.37540054321289,
+      "logps/rejected": -43.42636489868164,
+      "loss": 0.0,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -42.37540054321289,
+      "rewards/margins": 1.0509672164916992,
+      "rewards/rejected": -43.42636489868164,
       "step": 3525
     },
     {
       "epoch": 1.88927914366951,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.5537686977102167e-07,
       "learning_rate": 3.624108887686556e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.95891284942627,
+      "logits/rejected": 8.840760231018066,
+      "logps/chosen": -42.549034118652344,
+      "logps/rejected": -43.4303092956543,
+      "loss": 0.0,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -42.549034118652344,
+      "rewards/margins": 0.8812732696533203,
+      "rewards/rejected": -43.4303092956543,
       "step": 3530
     },
     {
       "epoch": 1.8919551764509115,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.149750348081701e-09,
       "learning_rate": 3.6091428225480433e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.090771675109863,
+      "logits/rejected": 8.770219802856445,
+      "logps/chosen": -42.282737731933594,
+      "logps/rejected": -43.812442779541016,
+      "loss": 0.0,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -42.282737731933594,
+      "rewards/margins": 1.5297024250030518,
+      "rewards/rejected": -43.812442779541016,
       "step": 3535
     },
     {
       "epoch": 1.8946312092323132,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.8547164379454796e-06,
       "learning_rate": 3.5941902515119674e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.243340492248535,
+      "logits/rejected": 8.353669166564941,
+      "logps/chosen": -43.447418212890625,
+      "logps/rejected": -43.65853500366211,
+      "loss": 0.0,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -43.447418212890625,
+      "rewards/margins": 0.2111167162656784,
+      "rewards/rejected": -43.65853500366211,
       "step": 3540
     },
     {
       "epoch": 1.8973072420137147,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 5.766401250556353e-08,
       "learning_rate": 3.5792513196482373e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.161626815795898,
+      "logits/rejected": 8.19868278503418,
+      "logps/chosen": -41.153018951416016,
+      "logps/rejected": -44.10417938232422,
+      "loss": 0.0,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -41.153018951416016,
+      "rewards/margins": 2.9511661529541016,
+      "rewards/rejected": -44.10417938232422,
       "step": 3545
     },
     {
       "epoch": 1.8999832747951162,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.863281745562108e-06,
       "learning_rate": 3.5643261718944346e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.499422073364258,
+      "logits/rejected": 8.118656158447266,
+      "logps/chosen": -41.36172103881836,
+      "logps/rejected": -42.181983947753906,
+      "loss": 0.0,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -41.36172103881836,
+      "rewards/margins": 0.8202651143074036,
+      "rewards/rejected": -42.181983947753906,
       "step": 3550
     },
     {
       "epoch": 1.902659307576518,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.842174477720752e-11,
       "learning_rate": 3.5494149530544087e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.111536979675293,
+      "logits/rejected": 8.595871925354004,
+      "logps/chosen": -42.133121490478516,
+      "logps/rejected": -42.95072937011719,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5062500238418579,
+      "rewards/chosen": -42.133121490478516,
+      "rewards/margins": 0.8176079988479614,
+      "rewards/rejected": -42.95072937011719,
       "step": 3555
     },
     {
       "epoch": 1.9053353403579194,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.6768610019370401e-09,
       "learning_rate": 3.534517807796871e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.257651329040527,
+      "logits/rejected": 8.800093650817871,
+      "logps/chosen": -42.34269332885742,
+      "logps/rejected": -43.44968795776367,
+      "loss": 0.0,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -42.34269332885742,
+      "rewards/margins": 1.1069934368133545,
+      "rewards/rejected": -43.44968795776367,
       "step": 3560
     },
     {
       "epoch": 1.908011373139321,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.061411973573918e-07,
       "learning_rate": 3.519634880653988e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.94935417175293,
+      "logits/rejected": 8.788725852966309,
+      "logps/chosen": -42.50508499145508,
+      "logps/rejected": -44.64189910888672,
+      "loss": 0.0,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -42.50508499145508,
+      "rewards/margins": 2.1368160247802734,
+      "rewards/rejected": -44.64189910888672,
       "step": 3565
     },
     {
       "epoch": 1.9106874059207226,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 7.560747657077877e-09,
       "learning_rate": 3.504766316019987e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.354012489318848,
+      "logits/rejected": 8.713424682617188,
+      "logps/chosen": -43.34133529663086,
+      "logps/rejected": -44.46308898925781,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -43.34133529663086,
+      "rewards/margins": 1.121755838394165,
+      "rewards/rejected": -44.46308898925781,
       "step": 3570
     },
     {
       "epoch": 1.913363438702124,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.0318364823111741e-09,
       "learning_rate": 3.489912258149745e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.05507755279541,
+      "logits/rejected": 8.614204406738281,
+      "logps/chosen": -43.152435302734375,
+      "logps/rejected": -44.185462951660156,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -43.152435302734375,
+      "rewards/margins": 1.0330299139022827,
+      "rewards/rejected": -44.185462951660156,
       "step": 3575
     },
     {
       "epoch": 1.9160394714835256,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.114034268115957e-09,
       "learning_rate": 3.475072851157397e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.823284149169922,
+      "logits/rejected": 8.767995834350586,
+      "logps/chosen": -42.063812255859375,
+      "logps/rejected": -43.452903747558594,
+      "loss": 0.0,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -42.063812255859375,
+      "rewards/margins": 1.389090895652771,
+      "rewards/rejected": -43.452903747558594,
       "step": 3580
     },
     {
       "epoch": 1.9187155042649273,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 9.86677188072468e-09,
       "learning_rate": 3.460248239014936e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.538291931152344,
+      "logits/rejected": 8.421481132507324,
+      "logps/chosen": -42.41411209106445,
+      "logps/rejected": -43.50041580200195,
+      "loss": 0.0,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -42.41411209106445,
+      "rewards/margins": 1.086301565170288,
+      "rewards/rejected": -43.50041580200195,
       "step": 3585
     },
     {
       "epoch": 1.9213915370463288,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.3948647739649184e-09,
       "learning_rate": 3.4454385655508134e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.809771537780762,
+      "logits/rejected": 8.738973617553711,
+      "logps/chosen": -42.37629699707031,
+      "logps/rejected": -43.13481140136719,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -42.37629699707031,
+      "rewards/margins": 0.7585160732269287,
+      "rewards/rejected": -43.13481140136719,
       "step": 3590
     },
     {
       "epoch": 1.9240675698277303,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 5.569565001193057e-10,
       "learning_rate": 3.4306439744485447e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.207661628723145,
+      "logits/rejected": 8.426923751831055,
+      "logps/chosen": -42.792816162109375,
+      "logps/rejected": -43.77056121826172,
+      "loss": 0.0,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -42.792816162109375,
+      "rewards/margins": 0.9777439832687378,
+      "rewards/rejected": -43.77056121826172,
       "step": 3595
     },
     {
       "epoch": 1.926743602609132,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 6.788564572550981e-06,
       "learning_rate": 3.415864609245322e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.945123672485352,
+      "logits/rejected": 8.362665176391602,
+      "logps/chosen": -43.345211029052734,
+      "logps/rejected": -44.7468147277832,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -43.345211029052734,
+      "rewards/margins": 1.401606559753418,
+      "rewards/rejected": -44.7468147277832,
       "step": 3600
     },
     {
       "epoch": 1.926743602609132,
-      "eval_logits/chosen": NaN,
-      "eval_logits/rejected": NaN,
-      "eval_logps/chosen": NaN,
-      "eval_logps/rejected": NaN,
-      "eval_loss": NaN,
-      "eval_rewards/accuracies": 0.0,
-      "eval_rewards/chosen": NaN,
-      "eval_rewards/margins": NaN,
-      "eval_rewards/rejected": NaN,
-      "eval_runtime": 35.2415,
-      "eval_samples_per_second": 38.165,
-      "eval_semantic_entropy": NaN,
-      "eval_steps_per_second": 9.563,
+      "eval_logits/chosen": 7.481541156768799,
+      "eval_logits/rejected": 7.208014965057373,
+      "eval_logps/chosen": -42.3165168762207,
+      "eval_logps/rejected": -43.4863395690918,
+      "eval_loss": 2.4548253918510454e-07,
+      "eval_rewards/accuracies": 0.5667656064033508,
+      "eval_rewards/chosen": -42.3165168762207,
+      "eval_rewards/margins": 1.169822096824646,
+      "eval_rewards/rejected": -43.4863395690918,
+      "eval_runtime": 40.3739,
+      "eval_samples_per_second": 33.314,
+      "eval_steps_per_second": 8.347,
       "step": 3600
     },
     {
       "epoch": 1.9294196353905335,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.0193225510697874e-07,
       "learning_rate": 3.401100613330605e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.300555229187012,
+      "logits/rejected": 9.18540096282959,
+      "logps/chosen": -43.99010467529297,
+      "logps/rejected": -44.124046325683594,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -43.99010467529297,
+      "rewards/margins": 0.13394129276275635,
+      "rewards/rejected": -44.124046325683594,
       "step": 3605
     },
     {
       "epoch": 1.932095668171935,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.6554967928936995e-07,
       "learning_rate": 3.3863521299447514e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.391404151916504,
+      "logits/rejected": 8.67957878112793,
+      "logps/chosen": -42.815818786621094,
+      "logps/rejected": -44.18699264526367,
+      "loss": 0.0,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -42.815818786621094,
+      "rewards/margins": 1.371173620223999,
+      "rewards/rejected": -44.18699264526367,
       "step": 3610
     },
     {
       "epoch": 1.9347717009533367,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.4645055223839736e-06,
       "learning_rate": 3.371619302177609e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.836959838867188,
+      "logits/rejected": 8.367228507995605,
+      "logps/chosen": -41.58897018432617,
+      "logps/rejected": -43.02305603027344,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -41.58897018432617,
+      "rewards/margins": 1.4340813159942627,
+      "rewards/rejected": -43.02305603027344,
       "step": 3615
     },
     {
       "epoch": 1.9374477337347382,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 5.102221872778029e-08,
       "learning_rate": 3.3569022729671393e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.9774751663208,
+      "logits/rejected": 8.841785430908203,
+      "logps/chosen": -41.4398307800293,
+      "logps/rejected": -42.85535430908203,
+      "loss": 0.0,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -41.4398307800293,
+      "rewards/margins": 1.4155213832855225,
+      "rewards/rejected": -42.85535430908203,
       "step": 3620
     },
     {
       "epoch": 1.9401237665161397,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.953960942036806e-08,
       "learning_rate": 3.342201185098024e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.801599502563477,
+      "logits/rejected": 9.148088455200195,
+      "logps/chosen": -42.89041519165039,
+      "logps/rejected": -43.07294464111328,
+      "loss": 0.0,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -42.89041519165039,
+      "rewards/margins": 0.18253205716609955,
+      "rewards/rejected": -43.07294464111328,
       "step": 3625
     },
     {
       "epoch": 1.9427997992975414,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.628925314304078e-08,
       "learning_rate": 3.3275161812002807e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.998138427734375,
+      "logits/rejected": 8.883317947387695,
+      "logps/chosen": -42.71221160888672,
+      "logps/rejected": -43.65961456298828,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -42.71221160888672,
+      "rewards/margins": 0.9474003911018372,
+      "rewards/rejected": -43.65961456298828,
       "step": 3630
     },
     {
       "epoch": 1.945475832078943,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 0.00021147751416318505,
       "learning_rate": 3.312847403747883e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.003536224365234,
+      "logits/rejected": 8.807095527648926,
+      "logps/chosen": -41.30414962768555,
+      "logps/rejected": -42.76262283325195,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -41.30414962768555,
+      "rewards/margins": 1.4584693908691406,
+      "rewards/rejected": -42.76262283325195,
       "step": 3635
     },
     {
       "epoch": 1.9481518648603444,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.0298351919046356e-10,
       "learning_rate": 3.2981949950573733e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.763233184814453,
+      "logits/rejected": 8.888627052307129,
+      "logps/chosen": -42.269569396972656,
+      "logps/rejected": -43.727325439453125,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -42.269569396972656,
+      "rewards/margins": 1.4577535390853882,
+      "rewards/rejected": -43.727325439453125,
       "step": 3640
     },
     {
       "epoch": 1.9508278976417461,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.4858901545262147e-10,
       "learning_rate": 3.283559097286486e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.83544921875,
+      "logits/rejected": 8.6160888671875,
+      "logps/chosen": -42.627220153808594,
+      "logps/rejected": -44.04213333129883,
+      "loss": 0.0,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -42.627220153808594,
+      "rewards/margins": 1.4149081707000732,
+      "rewards/rejected": -44.04213333129883,
       "step": 3645
     },
     {
       "epoch": 1.9535039304231478,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.92707034353838e-12,
       "learning_rate": 3.268939852432765e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.995279312133789,
+      "logits/rejected": 8.886581420898438,
+      "logps/chosen": -42.7044677734375,
+      "logps/rejected": -44.215675354003906,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -42.7044677734375,
+      "rewards/margins": 1.511206865310669,
+      "rewards/rejected": -44.215675354003906,
       "step": 3650
     },
     {
       "epoch": 1.9561799632045491,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 0.010264740524516293,
       "learning_rate": 3.254337402332187e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.067689895629883,
+      "logits/rejected": 8.6453857421875,
+      "logps/chosen": -43.12929916381836,
+      "logps/rejected": -43.67707443237305,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -43.12929916381836,
+      "rewards/margins": 0.5477749705314636,
+      "rewards/rejected": -43.67707443237305,
       "step": 3655
     },
     {
       "epoch": 1.9588559959859508,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.314309570305559e-09,
       "learning_rate": 3.239751888657788e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.858835220336914,
+      "logits/rejected": 8.436338424682617,
+      "logps/chosen": -41.64826202392578,
+      "logps/rejected": -43.458641052246094,
+      "loss": 0.0,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -41.64826202392578,
+      "rewards/margins": 1.8103832006454468,
+      "rewards/rejected": -43.458641052246094,
       "step": 3660
     },
     {
       "epoch": 1.9615320287673526,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.4664841053984346e-10,
       "learning_rate": 3.2251834529182856e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.15099048614502,
+      "logits/rejected": 8.874698638916016,
+      "logps/chosen": -43.49369430541992,
+      "logps/rejected": -45.0528678894043,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -43.49369430541992,
+      "rewards/margins": 1.559175968170166,
+      "rewards/rejected": -45.0528678894043,
       "step": 3665
     },
     {
       "epoch": 1.9642080615487538,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.8363268309875536e-05,
       "learning_rate": 3.2106322364567075e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.21888542175293,
+      "logits/rejected": 8.769892692565918,
+      "logps/chosen": -41.953548431396484,
+      "logps/rejected": -43.543636322021484,
+      "loss": 0.0,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -41.953548431396484,
+      "rewards/margins": 1.5900886058807373,
+      "rewards/rejected": -43.543636322021484,
       "step": 3670
     },
     {
       "epoch": 1.9668840943301555,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.3062468486591577e-10,
       "learning_rate": 3.1960983804490183e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.015542984008789,
+      "logits/rejected": 8.798874855041504,
+      "logps/chosen": -42.58515548706055,
+      "logps/rejected": -44.5765380859375,
+      "loss": 0.0,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -42.58515548706055,
+      "rewards/margins": 1.9913829565048218,
+      "rewards/rejected": -44.5765380859375,
       "step": 3675
     },
     {
       "epoch": 1.9695601271115573,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.093880227717721e-08,
       "learning_rate": 3.1815820259027537e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.040103912353516,
+      "logits/rejected": 8.717150688171387,
+      "logps/chosen": -43.57097625732422,
+      "logps/rejected": -44.8014030456543,
+      "loss": 0.0,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -43.57097625732422,
+      "rewards/margins": 1.2304190397262573,
+      "rewards/rejected": -44.8014030456543,
       "step": 3680
     },
     {
       "epoch": 1.9722361598929585,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.86839826568376e-09,
       "learning_rate": 3.16708331365565e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.841609954833984,
+      "logits/rejected": 8.674072265625,
+      "logps/chosen": -42.47706985473633,
+      "logps/rejected": -43.49345016479492,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -42.47706985473633,
+      "rewards/margins": 1.0163809061050415,
+      "rewards/rejected": -43.49345016479492,
       "step": 3685
     },
     {
       "epoch": 1.9749121926743602,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 8.609434840677341e-05,
       "learning_rate": 3.152602384374275e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.062284469604492,
+      "logits/rejected": 8.479073524475098,
+      "logps/chosen": -41.911842346191406,
+      "logps/rejected": -43.35440444946289,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -41.911842346191406,
+      "rewards/margins": 1.4425618648529053,
+      "rewards/rejected": -43.35440444946289,
       "step": 3690
     },
     {
       "epoch": 1.977588225455762,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.2428544009127452e-13,
       "learning_rate": 3.1381393785526697e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.972543716430664,
+      "logits/rejected": 8.768500328063965,
+      "logps/chosen": -43.57020568847656,
+      "logps/rejected": -44.500789642333984,
+      "loss": 0.0,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -43.57020568847656,
+      "rewards/margins": 0.9305828213691711,
+      "rewards/rejected": -44.500789642333984,
       "step": 3695
     },
     {
       "epoch": 1.9802642582371635,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 0.000251858403495644,
       "learning_rate": 3.123694436510979e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.001150131225586,
+      "logits/rejected": 8.65253734588623,
+      "logps/chosen": -43.221378326416016,
+      "logps/rejected": -44.03514862060547,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5062500238418579,
+      "rewards/chosen": -43.221378326416016,
+      "rewards/margins": 0.8137704730033875,
+      "rewards/rejected": -44.03514862060547,
       "step": 3700
     },
     {
       "epoch": 1.982940291018565,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.1841891933569207e-12,
       "learning_rate": 3.1092676983940946e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.339698791503906,
+      "logits/rejected": 8.944598197937012,
+      "logps/chosen": -43.04975128173828,
+      "logps/rejected": -44.066368103027344,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -43.04975128173828,
+      "rewards/margins": 1.0166168212890625,
+      "rewards/rejected": -44.066368103027344,
       "step": 3705
     },
     {
       "epoch": 1.9856163237999667,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.79940201829462e-06,
       "learning_rate": 3.094859304170293e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.634775161743164,
+      "logits/rejected": 8.508970260620117,
+      "logps/chosen": -41.86668395996094,
+      "logps/rejected": -42.9535026550293,
+      "loss": 0.0,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -41.86668395996094,
+      "rewards/margins": 1.0868192911148071,
+      "rewards/rejected": -42.9535026550293,
       "step": 3710
     },
     {
       "epoch": 1.9882923565813682,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.4962069426964036e-11,
       "learning_rate": 3.0804693936298795e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.16926097869873,
+      "logits/rejected": 9.15269660949707,
+      "logps/chosen": -43.54453659057617,
+      "logps/rejected": -44.353599548339844,
+      "loss": 0.0,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -43.54453659057617,
+      "rewards/margins": 0.8090649843215942,
+      "rewards/rejected": -44.353599548339844,
       "step": 3715
     },
     {
       "epoch": 1.9909683893627697,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.6387255405814323e-09,
       "learning_rate": 3.066098106383826e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.042624473571777,
+      "logits/rejected": 8.616301536560059,
+      "logps/chosen": -42.257225036621094,
+      "logps/rejected": -42.584930419921875,
+      "loss": 0.0,
+      "rewards/accuracies": 0.4937500059604645,
+      "rewards/chosen": -42.257225036621094,
+      "rewards/margins": 0.3277064263820648,
+      "rewards/rejected": -42.584930419921875,
       "step": 3720
     },
     {
       "epoch": 1.9936444221441714,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.981580155066613e-09,
       "learning_rate": 3.0517455818624263e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.090360641479492,
+      "logits/rejected": 8.818116188049316,
+      "logps/chosen": -43.08317184448242,
+      "logps/rejected": -43.85124588012695,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -43.08317184448242,
+      "rewards/margins": 0.768067479133606,
+      "rewards/rejected": -43.85124588012695,
       "step": 3725
     },
     {
       "epoch": 1.9963204549255729,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.6685157142407223e-08,
       "learning_rate": 3.037411959313936e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.131718635559082,
+      "logits/rejected": 8.754563331604004,
+      "logps/chosen": -41.910179138183594,
+      "logps/rejected": -44.03877639770508,
+      "loss": 0.0,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -41.910179138183594,
+      "rewards/margins": 2.128592014312744,
+      "rewards/rejected": -44.03877639770508,
       "step": 3730
     },
     {
       "epoch": 1.9989964877069744,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.913456172737053e-08,
       "learning_rate": 3.023097377803224e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.980962753295898,
+      "logits/rejected": 8.703187942504883,
+      "logps/chosen": -43.56035614013672,
+      "logps/rejected": -43.69626235961914,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -43.56035614013672,
+      "rewards/margins": 0.13590554893016815,
+      "rewards/rejected": -43.69626235961914,
       "step": 3735
     },
     {
       "epoch": 2.001672520488376,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.4971899323164673e-10,
       "learning_rate": 3.008801976210423e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.113046646118164,
+      "logits/rejected": 8.793360710144043,
+      "logps/chosen": -42.32197952270508,
+      "logps/rejected": -41.993621826171875,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5062500238418579,
+      "rewards/chosen": -42.32197952270508,
+      "rewards/margins": -0.3283580541610718,
+      "rewards/rejected": -41.993621826171875,
       "step": 3740
     },
     {
       "epoch": 2.0043485532697773,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.039977757036747e-11,
       "learning_rate": 2.994525893229581e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.052238464355469,
+      "logits/rejected": 8.735589981079102,
+      "logps/chosen": -43.29452133178711,
+      "logps/rejected": -45.156394958496094,
+      "loss": 0.0,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -43.29452133178711,
+      "rewards/margins": 1.8618745803833008,
+      "rewards/rejected": -45.156394958496094,
       "step": 3745
     },
     {
       "epoch": 2.007024586051179,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.1388265880232565e-06,
       "learning_rate": 2.98026926736732e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.030289649963379,
+      "logits/rejected": 8.800799369812012,
+      "logps/chosen": -42.60272979736328,
+      "logps/rejected": -43.9164924621582,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -42.60272979736328,
+      "rewards/margins": 1.3137590885162354,
+      "rewards/rejected": -43.9164924621582,
       "step": 3750
     },
     {
       "epoch": 2.0097006188325808,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.817417636378003e-11,
       "learning_rate": 2.9660322369414846e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.969718933105469,
+      "logits/rejected": 8.599370956420898,
+      "logps/chosen": -41.9445686340332,
+      "logps/rejected": -43.792171478271484,
+      "loss": 0.0,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -41.9445686340332,
+      "rewards/margins": 1.847603440284729,
+      "rewards/rejected": -43.792171478271484,
       "step": 3755
     },
     {
       "epoch": 2.0123766516139825,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.82318020708712e-10,
       "learning_rate": 2.9518149400798063e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.258691787719727,
+      "logits/rejected": 9.018590927124023,
+      "logps/chosen": -41.618587493896484,
+      "logps/rejected": -42.92237091064453,
+      "loss": 0.0,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -41.618587493896484,
+      "rewards/margins": 1.3037803173065186,
+      "rewards/rejected": -42.92237091064453,
       "step": 3760
     },
     {
       "epoch": 2.0150526843953838,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.2625408529560704e-10,
       "learning_rate": 2.9376175147185633e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.354743003845215,
+      "logits/rejected": 8.688486099243164,
+      "logps/chosen": -43.16361618041992,
+      "logps/rejected": -44.503639221191406,
+      "loss": 0.0,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -43.16361618041992,
+      "rewards/margins": 1.3400237560272217,
+      "rewards/rejected": -44.503639221191406,
       "step": 3765
     },
     {
       "epoch": 2.0177287171767855,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.1563170913060482e-13,
       "learning_rate": 2.9234400986012376e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.151975631713867,
+      "logits/rejected": 8.601824760437012,
+      "logps/chosen": -42.729515075683594,
+      "logps/rejected": -44.817352294921875,
+      "loss": 0.0,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -42.729515075683594,
+      "rewards/margins": 2.0878400802612305,
+      "rewards/rejected": -44.817352294921875,
       "step": 3770
     },
     {
       "epoch": 2.020404749958187,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 8.980844137640453e-10,
       "learning_rate": 2.9092828292771817e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.241704940795898,
+      "logits/rejected": 9.087262153625488,
+      "logps/chosen": -43.182579040527344,
+      "logps/rejected": -44.141822814941406,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -43.182579040527344,
+      "rewards/margins": 0.9592512845993042,
+      "rewards/rejected": -44.141822814941406,
       "step": 3775
     },
     {
       "epoch": 2.0230807827395885,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 7.543454577515874e-10,
       "learning_rate": 2.8951458441002875e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.907041549682617,
+      "logits/rejected": 8.689894676208496,
+      "logps/chosen": -41.20647430419922,
+      "logps/rejected": -41.67596435546875,
+      "loss": 0.0,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -41.20647430419922,
+      "rewards/margins": 0.4694964289665222,
+      "rewards/rejected": -41.67596435546875,
       "step": 3780
     },
     {
       "epoch": 2.02575681552099,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.4743813651592606e-07,
       "learning_rate": 2.881029280227643e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.202451705932617,
+      "logits/rejected": 8.600340843200684,
+      "logps/chosen": -42.56684875488281,
+      "logps/rejected": -43.63859939575195,
+      "loss": 0.0,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -42.56684875488281,
+      "rewards/margins": 1.0717495679855347,
+      "rewards/rejected": -43.63859939575195,
       "step": 3785
     },
     {
       "epoch": 2.028432848302392,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.0150184532156505e-11,
       "learning_rate": 2.8669332746182177e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.287517547607422,
+      "logits/rejected": 8.754729270935059,
+      "logps/chosen": -42.62308883666992,
+      "logps/rejected": -44.30974578857422,
+      "loss": 0.0,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -42.62308883666992,
+      "rewards/margins": 1.6866579055786133,
+      "rewards/rejected": -44.30974578857422,
       "step": 3790
     },
     {
       "epoch": 2.031108881083793,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.0156104208237597e-05,
       "learning_rate": 2.8528579640315156e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.80391788482666,
+      "logits/rejected": 8.872693061828613,
+      "logps/chosen": -43.396141052246094,
+      "logps/rejected": -43.928218841552734,
+      "loss": 0.0003,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -43.396141052246094,
+      "rewards/margins": 0.5320850610733032,
+      "rewards/rejected": -43.928218841552734,
       "step": 3795
     },
     {
       "epoch": 2.033784913865195,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 6.576066294088537e-07,
       "learning_rate": 2.8388034850262646e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.98240852355957,
+      "logits/rejected": 8.529772758483887,
+      "logps/chosen": -43.47077560424805,
+      "logps/rejected": -44.37076950073242,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -43.47077560424805,
+      "rewards/margins": 0.8999907374382019,
+      "rewards/rejected": -44.37076950073242,
       "step": 3800
     },
     {
       "epoch": 2.0364609466465966,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 8.468351144692882e-09,
       "learning_rate": 2.824769973959079e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.085939407348633,
+      "logits/rejected": 8.778947830200195,
+      "logps/chosen": -42.20588684082031,
+      "logps/rejected": -44.11560821533203,
+      "loss": 0.0,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -42.20588684082031,
+      "rewards/margins": 1.9097156524658203,
+      "rewards/rejected": -44.11560821533203,
       "step": 3805
     },
     {
       "epoch": 2.039136979427998,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.0614687667194108e-05,
       "learning_rate": 2.81075756698315e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.683633804321289,
+      "logits/rejected": 8.692230224609375,
+      "logps/chosen": -43.65705108642578,
+      "logps/rejected": -45.27473449707031,
+      "loss": 0.0,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -43.65705108642578,
+      "rewards/margins": 1.6176856756210327,
+      "rewards/rejected": -45.27473449707031,
       "step": 3810
     },
     {
       "epoch": 2.0418130122093996,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 9.635794652609165e-13,
       "learning_rate": 2.7967664000469035e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.182882308959961,
+      "logits/rejected": 8.858095169067383,
+      "logps/chosen": -43.048606872558594,
+      "logps/rejected": -44.22917938232422,
+      "loss": 0.0005,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -43.048606872558594,
+      "rewards/margins": 1.180572271347046,
+      "rewards/rejected": -44.22917938232422,
       "step": 3815
     },
     {
       "epoch": 2.0444890449908013,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.295626604099632e-09,
       "learning_rate": 2.7827966088927095e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.295331001281738,
+      "logits/rejected": 8.602898597717285,
+      "logps/chosen": -43.672733306884766,
+      "logps/rejected": -45.004112243652344,
+      "loss": 0.0,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -43.672733306884766,
+      "rewards/margins": 1.3313796520233154,
+      "rewards/rejected": -45.004112243652344,
       "step": 3820
     },
     {
       "epoch": 2.0471650777722026,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.3777344597859796e-10,
       "learning_rate": 2.768848329055538e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.324398040771484,
+      "logits/rejected": 9.085660934448242,
+      "logps/chosen": -43.610008239746094,
+      "logps/rejected": -45.07090759277344,
+      "loss": 0.0,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -43.610008239746094,
+      "rewards/margins": 1.46089768409729,
+      "rewards/rejected": -45.07090759277344,
       "step": 3825
     },
     {
       "epoch": 2.0498411105536043,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.2595106314453214e-10,
       "learning_rate": 2.7549216958616657e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.324460983276367,
+      "logits/rejected": 8.689237594604492,
+      "logps/chosen": -42.19231414794922,
+      "logps/rejected": -43.17029571533203,
+      "loss": 0.0,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -42.19231414794922,
+      "rewards/margins": 0.9779860377311707,
+      "rewards/rejected": -43.17029571533203,
       "step": 3830
     },
     {
       "epoch": 2.052517143335006,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.211331088623185e-06,
       "learning_rate": 2.741016844427344e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.99532413482666,
+      "logits/rejected": 8.408125877380371,
+      "logps/chosen": -42.37995529174805,
+      "logps/rejected": -43.69165802001953,
+      "loss": 0.0,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -42.37995529174805,
+      "rewards/margins": 1.3117038011550903,
+      "rewards/rejected": -43.69165802001953,
       "step": 3835
     },
     {
       "epoch": 2.0551931761164073,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 8.06253437927408e-09,
       "learning_rate": 2.7271339096575073e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.014673233032227,
+      "logits/rejected": 8.615967750549316,
+      "logps/chosen": -42.85087585449219,
+      "logps/rejected": -44.701324462890625,
+      "loss": 0.0,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -42.85087585449219,
+      "rewards/margins": 1.8504464626312256,
+      "rewards/rejected": -44.701324462890625,
       "step": 3840
     },
     {
       "epoch": 2.057869208897809,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 5.721844444063363e-13,
       "learning_rate": 2.713273026244446e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.721975326538086,
+      "logits/rejected": 8.899737358093262,
+      "logps/chosen": -42.51079559326172,
+      "logps/rejected": -44.6930046081543,
+      "loss": 0.0,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -42.51079559326172,
+      "rewards/margins": 2.182206869125366,
+      "rewards/rejected": -44.6930046081543,
       "step": 3845
     },
     {
       "epoch": 2.0605452416792107,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.1517434797084815e-07,
       "learning_rate": 2.6994343286665156e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.298352241516113,
+      "logits/rejected": 8.801363945007324,
+      "logps/chosen": -41.07033157348633,
+      "logps/rejected": -43.37969970703125,
+      "loss": 0.0,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -41.07033157348633,
+      "rewards/margins": 2.3093700408935547,
+      "rewards/rejected": -43.37969970703125,
       "step": 3850
     },
     {
       "epoch": 2.063221274460612,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.5295456626605476e-05,
       "learning_rate": 2.6856179511868156e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.238503456115723,
+      "logits/rejected": 8.465252876281738,
+      "logps/chosen": -41.18201446533203,
+      "logps/rejected": -42.878196716308594,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -41.18201446533203,
+      "rewards/margins": 1.6961853504180908,
+      "rewards/rejected": -42.878196716308594,
       "step": 3855
     },
     {
       "epoch": 2.0658973072420137,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 6.60994330410182e-09,
       "learning_rate": 2.6718240278519056e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.798654556274414,
+      "logits/rejected": 8.263616561889648,
+      "logps/chosen": -41.15531539916992,
+      "logps/rejected": -43.32644271850586,
+      "loss": 0.0,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -41.15531539916992,
+      "rewards/margins": 2.171125888824463,
+      "rewards/rejected": -43.32644271850586,
       "step": 3860
     },
     {
       "epoch": 2.0685733400234154,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.128752490463152e-10,
       "learning_rate": 2.6580526924904866e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.404783248901367,
+      "logits/rejected": 8.739439964294434,
+      "logps/chosen": -43.65899658203125,
+      "logps/rejected": -45.323692321777344,
+      "loss": 0.001,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -43.65899658203125,
+      "rewards/margins": 1.6646976470947266,
+      "rewards/rejected": -45.323692321777344,
       "step": 3865
     },
     {
       "epoch": 2.0712493728048167,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 6.715896231778555e-08,
       "learning_rate": 2.6443040787121186e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.327392578125,
+      "logits/rejected": 8.963663101196289,
+      "logps/chosen": -43.72232437133789,
+      "logps/rejected": -44.819427490234375,
+      "loss": 0.0,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -43.72232437133789,
+      "rewards/margins": 1.097109079360962,
+      "rewards/rejected": -44.819427490234375,
       "step": 3870
     },
     {
       "epoch": 2.0739254055862184,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.004862467346365e-07,
       "learning_rate": 2.6305783199059084e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.901762962341309,
+      "logits/rejected": 8.742029190063477,
+      "logps/chosen": -42.300865173339844,
+      "logps/rejected": -44.190460205078125,
+      "loss": 0.0,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -42.300865173339844,
+      "rewards/margins": 1.8895998001098633,
+      "rewards/rejected": -44.190460205078125,
       "step": 3875
     },
     {
       "epoch": 2.07660143836762,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.8011284678424387e-13,
       "learning_rate": 2.6168755492392324e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.4299955368042,
+      "logits/rejected": 8.773306846618652,
+      "logps/chosen": -42.846839904785156,
+      "logps/rejected": -44.283424377441406,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -42.846839904785156,
+      "rewards/margins": 1.436584234237671,
+      "rewards/rejected": -44.283424377441406,
       "step": 3880
     },
     {
       "epoch": 2.0792774711490214,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 0.001467624750922948,
       "learning_rate": 2.6031958996564274e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.310066223144531,
+      "logits/rejected": 8.857309341430664,
+      "logps/chosen": -43.28919219970703,
+      "logps/rejected": -44.32575988769531,
+      "loss": 0.0,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -43.28919219970703,
+      "rewards/margins": 1.036568522453308,
+      "rewards/rejected": -44.32575988769531,
       "step": 3885
     },
     {
       "epoch": 2.081953503930423,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.576593305171687e-09,
       "learning_rate": 2.589539503877518e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.669404983520508,
+      "logits/rejected": 8.387873649597168,
+      "logps/chosen": -43.117332458496094,
+      "logps/rejected": -44.266868591308594,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -43.117332458496094,
+      "rewards/margins": 1.1495267152786255,
+      "rewards/rejected": -44.266868591308594,
       "step": 3890
     },
     {
       "epoch": 2.084629536711825,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.1491102099131159e-07,
       "learning_rate": 2.5759064943969125e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.045702934265137,
+      "logits/rejected": 8.297749519348145,
+      "logps/chosen": -42.43684005737305,
+      "logps/rejected": -44.2757568359375,
+      "loss": 0.0,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -42.43684005737305,
+      "rewards/margins": 1.8389114141464233,
+      "rewards/rejected": -44.2757568359375,
       "step": 3895
     },
     {
       "epoch": 2.087305569493226,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.741351840835168e-08,
       "learning_rate": 2.562297003482131e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.002426147460938,
+      "logits/rejected": 9.01283073425293,
+      "logps/chosen": -43.30495071411133,
+      "logps/rejected": -43.95606231689453,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5062500238418579,
+      "rewards/chosen": -43.30495071411133,
+      "rewards/margins": 0.6511114835739136,
+      "rewards/rejected": -43.95606231689453,
       "step": 3900
     },
     {
       "epoch": 2.089981602274628,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 8.817827689403988e-09,
       "learning_rate": 2.548711163172512e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.014253616333008,
+      "logits/rejected": 8.92760944366455,
+      "logps/chosen": -42.65181350708008,
+      "logps/rejected": -44.23887634277344,
+      "loss": 0.0,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -42.65181350708008,
+      "rewards/margins": 1.5870609283447266,
+      "rewards/rejected": -44.23887634277344,
       "step": 3905
     },
     {
       "epoch": 2.0926576350560295,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.1983228105477875e-10,
       "learning_rate": 2.53514910527794e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.045778274536133,
+      "logits/rejected": 8.59282398223877,
+      "logps/chosen": -43.62392044067383,
+      "logps/rejected": -44.38840103149414,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -43.62392044067383,
+      "rewards/margins": 0.7644834518432617,
+      "rewards/rejected": -44.38840103149414,
       "step": 3910
     },
     {
       "epoch": 2.095333667837431,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.5817597533685205e-11,
       "learning_rate": 2.5216109613775573e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.067813873291016,
+      "logits/rejected": 8.734966278076172,
+      "logps/chosen": -43.35368728637695,
+      "logps/rejected": -44.36429977416992,
+      "loss": 0.0,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -43.35368728637695,
+      "rewards/margins": 1.0106127262115479,
+      "rewards/rejected": -44.36429977416992,
       "step": 3915
     },
     {
       "epoch": 2.0980097006188325,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 7.103466358065859e-07,
       "learning_rate": 2.5080968628184993e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.974212646484375,
+      "logits/rejected": 8.480744361877441,
+      "logps/chosen": -42.90849685668945,
+      "logps/rejected": -44.19004440307617,
+      "loss": 0.0,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -42.90849685668945,
+      "rewards/margins": 1.281553864479065,
+      "rewards/rejected": -44.19004440307617,
       "step": 3920
     },
     {
       "epoch": 2.1006857334002342,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 7.472472465856852e-10,
       "learning_rate": 2.494606940714605e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.834748268127441,
+      "logits/rejected": 8.467195510864258,
+      "logps/chosen": -42.65045928955078,
+      "logps/rejected": -43.457786560058594,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -42.65045928955078,
+      "rewards/margins": 0.8073281049728394,
+      "rewards/rejected": -43.457786560058594,
       "step": 3925
     },
     {
       "epoch": 2.103361766181636,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 7.009659713125854e-08,
       "learning_rate": 2.4811413259451625e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.188947677612305,
+      "logits/rejected": 8.615166664123535,
+      "logps/chosen": -41.76762008666992,
+      "logps/rejected": -43.74374771118164,
+      "loss": 0.0,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -41.76762008666992,
+      "rewards/margins": 1.9761216640472412,
+      "rewards/rejected": -43.74374771118164,
       "step": 3930
     },
     {
       "epoch": 2.106037798963037,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.2265033106407376e-10,
       "learning_rate": 2.46770014915362e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.099128723144531,
+      "logits/rejected": 8.857921600341797,
+      "logps/chosen": -43.18244934082031,
+      "logps/rejected": -44.17724609375,
+      "loss": 0.0,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -43.18244934082031,
+      "rewards/margins": 0.9947935342788696,
+      "rewards/rejected": -44.17724609375,
       "step": 3935
     },
     {
       "epoch": 2.108713831744439,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.831122985380686e-12,
       "learning_rate": 2.45428354074634e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.956808090209961,
+      "logits/rejected": 8.80980396270752,
+      "logps/chosen": -43.34827423095703,
+      "logps/rejected": -45.02986526489258,
+      "loss": 0.0,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -43.34827423095703,
+      "rewards/margins": 1.6815884113311768,
+      "rewards/rejected": -45.02986526489258,
       "step": 3940
     },
     {
       "epoch": 2.1113898645258407,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 5.385973620598079e-09,
       "learning_rate": 2.4408916308913105e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.856588363647461,
+      "logits/rejected": 8.372272491455078,
+      "logps/chosen": -41.744720458984375,
+      "logps/rejected": -44.288475036621094,
+      "loss": 0.0,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -41.744720458984375,
+      "rewards/margins": 2.543750286102295,
+      "rewards/rejected": -44.288475036621094,
       "step": 3945
     },
     {
       "epoch": 2.114065897307242,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.2885550934086666e-11,
       "learning_rate": 2.4275245495169025e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.060127258300781,
+      "logits/rejected": 8.5348539352417,
+      "logps/chosen": -43.25366973876953,
+      "logps/rejected": -44.860923767089844,
+      "loss": 0.0,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -43.25366973876953,
+      "rewards/margins": 1.6072571277618408,
+      "rewards/rejected": -44.860923767089844,
       "step": 3950
     },
     {
       "epoch": 2.1167419300886436,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.0220418821000886e-08,
       "learning_rate": 2.414182426310597e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.218419075012207,
+      "logits/rejected": 9.203883171081543,
+      "logps/chosen": -43.59880828857422,
+      "logps/rejected": -44.50844955444336,
+      "loss": 0.0,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -43.59880828857422,
+      "rewards/margins": 0.9096387624740601,
+      "rewards/rejected": -44.50844955444336,
       "step": 3955
     },
     {
       "epoch": 2.1194179628700454,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.893393021176766e-08,
       "learning_rate": 2.400865390717734e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.099970817565918,
+      "logits/rejected": 8.677289009094238,
+      "logps/chosen": -42.886451721191406,
+      "logps/rejected": -44.28646469116211,
+      "loss": 0.0,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -42.886451721191406,
+      "rewards/margins": 1.4000129699707031,
+      "rewards/rejected": -44.28646469116211,
       "step": 3960
     },
     {
       "epoch": 2.1220939956514466,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 7.48536826683026e-07,
       "learning_rate": 2.3875735719402475e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.244757652282715,
+      "logits/rejected": 8.968472480773926,
+      "logps/chosen": -43.204368591308594,
+      "logps/rejected": -44.411964416503906,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -43.204368591308594,
+      "rewards/margins": 1.2075891494750977,
+      "rewards/rejected": -44.411964416503906,
       "step": 3965
     },
     {
       "epoch": 2.1247700284328483,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.523063610498576e-10,
       "learning_rate": 2.3743070989354258e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.296489715576172,
+      "logits/rejected": 9.018621444702148,
+      "logps/chosen": -43.32949447631836,
+      "logps/rejected": -45.09051513671875,
+      "loss": 0.0,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -43.32949447631836,
+      "rewards/margins": 1.7610194683074951,
+      "rewards/rejected": -45.09051513671875,
       "step": 3970
     },
     {
       "epoch": 2.12744606121425,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.3776824452666e-09,
       "learning_rate": 2.3610661004146454e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.050966262817383,
+      "logits/rejected": 8.661293029785156,
+      "logps/chosen": -42.240291595458984,
+      "logps/rejected": -43.407020568847656,
+      "loss": 0.0,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -42.240291595458984,
+      "rewards/margins": 1.1667273044586182,
+      "rewards/rejected": -43.407020568847656,
       "step": 3975
     },
     {
       "epoch": 2.1301220939956513,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.625804320271655e-05,
       "learning_rate": 2.3478507048421314e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.038867950439453,
+      "logits/rejected": 8.616304397583008,
+      "logps/chosen": -42.380035400390625,
+      "logps/rejected": -43.47580337524414,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -42.380035400390625,
+      "rewards/margins": 1.0957672595977783,
+      "rewards/rejected": -43.47580337524414,
       "step": 3980
     },
     {
       "epoch": 2.132798126777053,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.9259894518899858e-12,
       "learning_rate": 2.334661040433713e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.501883506774902,
+      "logits/rejected": 9.05534553527832,
+      "logps/chosen": -44.69688415527344,
+      "logps/rejected": -46.294898986816406,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -44.69688415527344,
+      "rewards/margins": 1.5980141162872314,
+      "rewards/rejected": -46.294898986816406,
       "step": 3985
     },
     {
       "epoch": 2.1354741595584548,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.2922434755463565e-10,
       "learning_rate": 2.321497235155568e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.28370475769043,
+      "logits/rejected": 8.744659423828125,
+      "logps/chosen": -42.807228088378906,
+      "logps/rejected": -44.01629638671875,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -42.807228088378906,
+      "rewards/margins": 1.2090747356414795,
+      "rewards/rejected": -44.01629638671875,
       "step": 3990
     },
     {
       "epoch": 2.138150192339856,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.4786456834887667e-10,
       "learning_rate": 2.3083594167229965e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.276188850402832,
+      "logits/rejected": 8.40411376953125,
+      "logps/chosen": -42.7730598449707,
+      "logps/rejected": -44.181884765625,
+      "loss": 0.0,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -42.7730598449707,
+      "rewards/margins": 1.4088283777236938,
+      "rewards/rejected": -44.181884765625,
       "step": 3995
     },
     {
       "epoch": 2.1408262251212578,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.3875723591921611e-08,
       "learning_rate": 2.295247712599167e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.039514541625977,
+      "logits/rejected": 8.849933624267578,
+      "logps/chosen": -43.12586212158203,
+      "logps/rejected": -44.19435501098633,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -43.12586212158203,
+      "rewards/margins": 1.0684914588928223,
+      "rewards/rejected": -44.19435501098633,
       "step": 4000
     },
     {
       "epoch": 2.1408262251212578,
-      "eval_logits/chosen": NaN,
-      "eval_logits/rejected": NaN,
-      "eval_logps/chosen": NaN,
-      "eval_logps/rejected": NaN,
-      "eval_loss": NaN,
-      "eval_rewards/accuracies": 0.0,
-      "eval_rewards/chosen": NaN,
-      "eval_rewards/margins": NaN,
-      "eval_rewards/rejected": NaN,
-      "eval_runtime": 35.0743,
-      "eval_samples_per_second": 38.347,
-      "eval_semantic_entropy": NaN,
-      "eval_steps_per_second": 9.608,
+      "eval_logits/chosen": 7.554755687713623,
+      "eval_logits/rejected": 7.255157947540283,
+      "eval_logps/chosen": -43.03852081298828,
+      "eval_logps/rejected": -44.147335052490234,
+      "eval_loss": 2.646533232564252e-07,
+      "eval_rewards/accuracies": 0.5697329640388489,
+      "eval_rewards/chosen": -43.03852081298828,
+      "eval_rewards/margins": 1.1088168621063232,
+      "eval_rewards/rejected": -44.147335052490234,
+      "eval_runtime": 40.3659,
+      "eval_samples_per_second": 33.32,
+      "eval_steps_per_second": 8.349,
       "step": 4000
     },
     {
       "epoch": 2.1435022579026595,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 8.014969882167829e-06,
       "learning_rate": 2.2821622499938948e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.994928359985352,
+      "logits/rejected": 8.230422973632812,
+      "logps/chosen": -42.28566360473633,
+      "logps/rejected": -43.514556884765625,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -42.28566360473633,
+      "rewards/margins": 1.228898286819458,
+      "rewards/rejected": -43.514556884765625,
       "step": 4005
     },
     {
       "epoch": 2.1461782906840607,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.517083628322829e-12,
       "learning_rate": 2.269103155862391e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.259135246276855,
+      "logits/rejected": 8.679240226745605,
+      "logps/chosen": -43.896732330322266,
+      "logps/rejected": -44.3692512512207,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -43.896732330322266,
+      "rewards/margins": 0.47251325845718384,
+      "rewards/rejected": -44.3692512512207,
       "step": 4010
     },
     {
       "epoch": 2.1488543234654625,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.3701373141855658e-07,
       "learning_rate": 2.2560705569040483e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.022557258605957,
+      "logits/rejected": 8.053827285766602,
+      "logps/chosen": -43.2159309387207,
+      "logps/rejected": -44.100799560546875,
+      "loss": 0.0,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -43.2159309387207,
+      "rewards/margins": 0.8848685026168823,
+      "rewards/rejected": -44.100799560546875,
       "step": 4015
     },
     {
       "epoch": 2.151530356246864,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 6.404427860088793e-07,
       "learning_rate": 2.2430645795611963e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.072053909301758,
+      "logits/rejected": 8.639387130737305,
+      "logps/chosen": -42.088523864746094,
+      "logps/rejected": -43.122013092041016,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -42.088523864746094,
+      "rewards/margins": 1.0334886312484741,
+      "rewards/rejected": -43.122013092041016,
       "step": 4020
     },
     {
       "epoch": 2.1542063890282654,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.186050331956477e-06,
       "learning_rate": 2.230085350017884e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.985651016235352,
+      "logits/rejected": 8.547399520874023,
+      "logps/chosen": -42.805233001708984,
+      "logps/rejected": -43.92900848388672,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -42.805233001708984,
+      "rewards/margins": 1.1237679719924927,
+      "rewards/rejected": -43.92900848388672,
       "step": 4025
     },
     {
       "epoch": 2.156882421809667,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 5.068366124293054e-14,
       "learning_rate": 2.2171329941986554e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.14936637878418,
+      "logits/rejected": 8.997502326965332,
+      "logps/chosen": -43.808631896972656,
+      "logps/rejected": -45.30188751220703,
+      "loss": 0.0,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": -43.808631896972656,
+      "rewards/margins": 1.493253231048584,
+      "rewards/rejected": -45.30188751220703,
       "step": 4030
     },
     {
       "epoch": 2.159558454591069,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.1810545762746e-10,
       "learning_rate": 2.2042076377673202e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.72819709777832,
+      "logits/rejected": 8.76697826385498,
+      "logps/chosen": -42.61308670043945,
+      "logps/rejected": -43.672000885009766,
+      "loss": 0.0,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -42.61308670043945,
+      "rewards/margins": 1.0589206218719482,
+      "rewards/rejected": -43.672000885009766,
       "step": 4035
     },
     {
       "epoch": 2.16223448737247,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.2177576135311725e-09,
       "learning_rate": 2.1913094061257476e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.976733207702637,
+      "logits/rejected": 8.85966682434082,
+      "logps/chosen": -43.48511505126953,
+      "logps/rejected": -44.28733444213867,
+      "loss": 0.0,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -43.48511505126953,
+      "rewards/margins": 0.8022161722183228,
+      "rewards/rejected": -44.28733444213867,
       "step": 4040
     },
     {
       "epoch": 2.164910520153872,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.733320928040242e-10,
       "learning_rate": 2.178438424412633e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.916051864624023,
+      "logits/rejected": 8.544427871704102,
+      "logps/chosen": -43.135459899902344,
+      "logps/rejected": -44.476280212402344,
+      "loss": 0.0,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -43.135459899902344,
+      "rewards/margins": 1.3408172130584717,
+      "rewards/rejected": -44.476280212402344,
       "step": 4045
     },
     {
       "epoch": 2.1675865529352736,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 6.924030041514536e-09,
       "learning_rate": 2.165594817502302e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.367121696472168,
+      "logits/rejected": 8.813446044921875,
+      "logps/chosen": -43.04452896118164,
+      "logps/rejected": -44.201663970947266,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -43.04452896118164,
+      "rewards/margins": 1.157134771347046,
+      "rewards/rejected": -44.201663970947266,
       "step": 4050
     },
     {
       "epoch": 2.170262585716675,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 5.88410062128727e-09,
       "learning_rate": 2.1527787100034806e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.773908615112305,
+      "logits/rejected": 8.69034194946289,
+      "logps/chosen": -42.485679626464844,
+      "logps/rejected": -43.780277252197266,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -42.485679626464844,
+      "rewards/margins": 1.2945966720581055,
+      "rewards/rejected": -43.780277252197266,
       "step": 4055
     },
     {
       "epoch": 2.1729386184980766,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 7.435395033790735e-08,
       "learning_rate": 2.1399902262581037e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.17042064666748,
+      "logits/rejected": 8.571090698242188,
+      "logps/chosen": -42.98744583129883,
+      "logps/rejected": -44.34369659423828,
+      "loss": 0.0,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -42.98744583129883,
+      "rewards/margins": 1.3562500476837158,
+      "rewards/rejected": -44.34369659423828,
       "step": 4060
     },
     {
       "epoch": 2.1756146512794783,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.1036007844510176e-10,
       "learning_rate": 2.127229490340094e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.034303665161133,
+      "logits/rejected": 8.611432075500488,
+      "logps/chosen": -43.0045166015625,
+      "logps/rejected": -44.0695686340332,
+      "loss": 0.0,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -43.0045166015625,
+      "rewards/margins": 1.0650520324707031,
+      "rewards/rejected": -44.0695686340332,
       "step": 4065
     },
     {
       "epoch": 2.1782906840608796,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.2912627928083715e-08,
       "learning_rate": 2.1144966260541698e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.093645095825195,
+      "logits/rejected": 8.371310234069824,
+      "logps/chosen": -42.680213928222656,
+      "logps/rejected": -43.944541931152344,
+      "loss": 0.0,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -42.680213928222656,
+      "rewards/margins": 1.264331579208374,
+      "rewards/rejected": -43.944541931152344,
       "step": 4070
     },
     {
       "epoch": 2.1809667168422813,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 6.547996878046198e-05,
       "learning_rate": 2.1017917569346332e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.030420303344727,
+      "logits/rejected": 8.363921165466309,
+      "logps/chosen": -42.09078598022461,
+      "logps/rejected": -43.690826416015625,
+      "loss": 0.0,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -42.09078598022461,
+      "rewards/margins": 1.6000381708145142,
+      "rewards/rejected": -43.690826416015625,
       "step": 4075
     },
     {
       "epoch": 2.183642749623683,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 0.00018978273997853653,
       "learning_rate": 2.0891150062441837e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.048044204711914,
+      "logits/rejected": 8.46544075012207,
+      "logps/chosen": -42.538658142089844,
+      "logps/rejected": -44.154579162597656,
+      "loss": 0.0,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -42.538658142089844,
+      "rewards/margins": 1.615917444229126,
+      "rewards/rejected": -44.154579162597656,
       "step": 4080
     },
     {
       "epoch": 2.1863187824050843,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 7.93023309397749e-12,
       "learning_rate": 2.0764664969727086e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.71142864227295,
+      "logits/rejected": 8.834856986999512,
+      "logps/chosen": -41.934146881103516,
+      "logps/rejected": -42.829708099365234,
+      "loss": 0.0,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -41.934146881103516,
+      "rewards/margins": 0.8955645561218262,
+      "rewards/rejected": -42.829708099365234,
       "step": 4085
     },
     {
       "epoch": 2.188994815186486,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.562234346281564e-09,
       "learning_rate": 2.0638463518361033e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.596474647521973,
+      "logits/rejected": 8.892104148864746,
+      "logps/chosen": -42.5695915222168,
+      "logps/rejected": -43.9444465637207,
+      "loss": 0.0001,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -42.5695915222168,
+      "rewards/margins": 1.3748515844345093,
+      "rewards/rejected": -43.9444465637207,
       "step": 4090
     },
     {
       "epoch": 2.1916708479678877,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 6.821408193988347e-11,
       "learning_rate": 2.0512546932750702e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.35791015625,
+      "logits/rejected": 9.129060745239258,
+      "logps/chosen": -42.6036376953125,
+      "logps/rejected": -43.65306854248047,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -42.6036376953125,
+      "rewards/margins": 1.0494320392608643,
+      "rewards/rejected": -43.65306854248047,
       "step": 4095
     },
     {
       "epoch": 2.194346880749289,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.196708590046929e-06,
       "learning_rate": 2.0386916434539343e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.000151634216309,
+      "logits/rejected": 8.360278129577637,
+      "logps/chosen": -44.51485061645508,
+      "logps/rejected": -45.5093994140625,
+      "loss": 0.0005,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -44.51485061645508,
+      "rewards/margins": 0.9945542216300964,
+      "rewards/rejected": -45.5093994140625,
       "step": 4100
     },
     {
       "epoch": 2.1970229135306907,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.4444548241558217e-10,
       "learning_rate": 2.0261573242594627e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.081033706665039,
+      "logits/rejected": 8.563546180725098,
+      "logps/chosen": -42.82344055175781,
+      "logps/rejected": -43.670806884765625,
+      "loss": 0.0,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -42.82344055175781,
+      "rewards/margins": 0.8473671078681946,
+      "rewards/rejected": -43.670806884765625,
       "step": 4105
     },
     {
       "epoch": 2.1996989463120924,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.007488682186981e-11,
       "learning_rate": 2.0136518572996724e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.229379653930664,
+      "logits/rejected": 8.714961051940918,
+      "logps/chosen": -43.3348274230957,
+      "logps/rejected": -44.753509521484375,
+      "loss": 0.0,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -43.3348274230957,
+      "rewards/margins": 1.4186785221099854,
+      "rewards/rejected": -44.753509521484375,
       "step": 4110
     },
     {
       "epoch": 2.202374979093494,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.4650077103015228e-09,
       "learning_rate": 2.0011753639026617e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.876505851745605,
+      "logits/rejected": 8.473733901977539,
+      "logps/chosen": -43.526954650878906,
+      "logps/rejected": -44.23854446411133,
+      "loss": 0.0,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -43.526954650878906,
+      "rewards/margins": 0.7115898728370667,
+      "rewards/rejected": -44.23854446411133,
       "step": 4115
     },
     {
       "epoch": 2.2050510118748954,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.2031412283640005e-09,
       "learning_rate": 1.988727965115421e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.417000770568848,
+      "logits/rejected": 8.949925422668457,
+      "logps/chosen": -43.354881286621094,
+      "logps/rejected": -44.204193115234375,
+      "loss": 0.0,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -43.354881286621094,
+      "rewards/margins": 0.8493131399154663,
+      "rewards/rejected": -44.204193115234375,
       "step": 4120
     },
     {
       "epoch": 2.207727044656297,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.2683891641049463e-09,
       "learning_rate": 1.9763097817026713e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.431020736694336,
+      "logits/rejected": 8.648829460144043,
+      "logps/chosen": -43.11323165893555,
+      "logps/rejected": -44.73983383178711,
+      "loss": 0.0,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -43.11323165893555,
+      "rewards/margins": 1.6266052722930908,
+      "rewards/rejected": -44.73983383178711,
       "step": 4125
     },
     {
       "epoch": 2.210403077437699,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 7.784828062105631e-12,
       "learning_rate": 1.9639209341456796e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.925825119018555,
+      "logits/rejected": 8.671640396118164,
+      "logps/chosen": -43.275718688964844,
+      "logps/rejected": -44.03488540649414,
+      "loss": 0.0,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -43.275718688964844,
+      "rewards/margins": 0.7591708302497864,
+      "rewards/rejected": -44.03488540649414,
       "step": 4130
     },
     {
       "epoch": 2.2130791102191,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.4506430999343263e-05,
       "learning_rate": 1.951561542641102e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.659500122070312,
+      "logits/rejected": 8.84306812286377,
+      "logps/chosen": -43.804786682128906,
+      "logps/rejected": -43.858951568603516,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -43.804786682128906,
+      "rewards/margins": 0.0541643388569355,
+      "rewards/rejected": -43.858951568603516,
       "step": 4135
     },
     {
       "epoch": 2.215755143000502,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 9.560065389393075e-10,
       "learning_rate": 1.939231727099806e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.69888973236084,
+      "logits/rejected": 9.543991088867188,
+      "logps/chosen": -42.596351623535156,
+      "logps/rejected": -43.82774353027344,
+      "loss": 0.0,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -42.596351623535156,
+      "rewards/margins": 1.23139226436615,
+      "rewards/rejected": -43.82774353027344,
       "step": 4140
     },
     {
       "epoch": 2.2184311757819035,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.779419934626799e-08,
       "learning_rate": 1.926931607145719e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.757194519042969,
+      "logits/rejected": 8.255291938781738,
+      "logps/chosen": -42.77762222290039,
+      "logps/rejected": -44.15127944946289,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -42.77762222290039,
+      "rewards/margins": 1.3736575841903687,
+      "rewards/rejected": -44.15127944946289,
       "step": 4145
     },
     {
       "epoch": 2.221107208563305,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.1968425553405376e-11,
       "learning_rate": 1.9146613021146564e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.280985832214355,
+      "logits/rejected": 8.816904067993164,
+      "logps/chosen": -43.721435546875,
+      "logps/rejected": -45.39226531982422,
+      "loss": 0.0,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -43.721435546875,
+      "rewards/margins": 1.670829176902771,
+      "rewards/rejected": -45.39226531982422,
       "step": 4150
     },
     {
       "epoch": 2.2237832413447065,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.360179443915149e-09,
       "learning_rate": 1.9024209310531736e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.317049026489258,
+      "logits/rejected": 9.267797470092773,
+      "logps/chosen": -43.670799255371094,
+      "logps/rejected": -43.69342041015625,
+      "loss": 0.0,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -43.670799255371094,
+      "rewards/margins": 0.02261648140847683,
+      "rewards/rejected": -43.69342041015625,
       "step": 4155
     },
     {
       "epoch": 2.2264592741261082,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.133369610360667e-05,
       "learning_rate": 1.890210612717401e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.052912712097168,
+      "logits/rejected": 8.901524543762207,
+      "logps/chosen": -42.55875778198242,
+      "logps/rejected": -44.060935974121094,
+      "loss": 0.0004,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -42.55875778198242,
+      "rewards/margins": 1.502176284790039,
+      "rewards/rejected": -44.060935974121094,
       "step": 4160
     },
     {
       "epoch": 2.2291353069075095,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.6609431491087264e-11,
       "learning_rate": 1.8780304655719054e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.408945083618164,
+      "logits/rejected": 8.934088706970215,
+      "logps/chosen": -43.247955322265625,
+      "logps/rejected": -44.729820251464844,
+      "loss": 0.0,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -43.247955322265625,
+      "rewards/margins": 1.481859564781189,
+      "rewards/rejected": -44.729820251464844,
       "step": 4165
     },
     {
       "epoch": 2.231811339688911,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 8.508534257493168e-12,
       "learning_rate": 1.865880607788523e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.797937393188477,
+      "logits/rejected": 8.62280559539795,
+      "logps/chosen": -43.60547637939453,
+      "logps/rejected": -44.05706024169922,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -43.60547637939453,
+      "rewards/margins": 0.45158448815345764,
+      "rewards/rejected": -44.05706024169922,
       "step": 4170
     },
     {
       "epoch": 2.234487372470313,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.238662880192874e-09,
       "learning_rate": 1.8537611572452316e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.218876838684082,
+      "logits/rejected": 8.86221981048584,
+      "logps/chosen": -42.84282302856445,
+      "logps/rejected": -43.70226287841797,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -42.84282302856445,
+      "rewards/margins": 0.859442412853241,
+      "rewards/rejected": -43.70226287841797,
       "step": 4175
     },
     {
       "epoch": 2.237163405251714,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.0667150759492616e-08,
       "learning_rate": 1.84167223152499e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.290166854858398,
+      "logits/rejected": 8.51961898803711,
+      "logps/chosen": -41.8538932800293,
+      "logps/rejected": -43.41326904296875,
+      "loss": 0.0012,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -41.8538932800293,
+      "rewards/margins": 1.559377670288086,
+      "rewards/rejected": -43.41326904296875,
       "step": 4180
     },
     {
       "epoch": 2.239839438033116,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.211804441308766e-10,
       "learning_rate": 1.8296139479146112e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.022268295288086,
+      "logits/rejected": 9.100817680358887,
+      "logps/chosen": -43.53455352783203,
+      "logps/rejected": -43.85361099243164,
+      "loss": 0.0,
+      "rewards/accuracies": 0.4937500059604645,
+      "rewards/chosen": -43.53455352783203,
+      "rewards/margins": 0.31906184554100037,
+      "rewards/rejected": -43.85361099243164,
       "step": 4185
     },
     {
       "epoch": 2.2425154708145176,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 8.513984700052057e-12,
       "learning_rate": 1.8175864234036132e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.85918140411377,
+      "logits/rejected": 8.641780853271484,
+      "logps/chosen": -43.27741241455078,
+      "logps/rejected": -44.291725158691406,
+      "loss": 0.0,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -43.27741241455078,
+      "rewards/margins": 1.014306664466858,
+      "rewards/rejected": -44.291725158691406,
       "step": 4190
     },
     {
       "epoch": 2.245191503595919,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.601210554763539e-11,
       "learning_rate": 1.805589774683094e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.456896781921387,
+      "logits/rejected": 9.032327651977539,
+      "logps/chosen": -43.820228576660156,
+      "logps/rejected": -44.67981719970703,
+      "loss": 0.0,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -43.820228576660156,
+      "rewards/margins": 0.8595897555351257,
+      "rewards/rejected": -44.67981719970703,
       "step": 4195
     },
     {
       "epoch": 2.2478675363773206,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.2462054780508733e-09,
       "learning_rate": 1.79362411814459e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.816685676574707,
+      "logits/rejected": 8.74372673034668,
+      "logps/chosen": -43.02033233642578,
+      "logps/rejected": -43.57416915893555,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -43.02033233642578,
+      "rewards/margins": 0.5538488626480103,
+      "rewards/rejected": -43.57416915893555,
       "step": 4200
     },
     {
       "epoch": 2.2505435691587223,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.0210239043090519e-10,
       "learning_rate": 1.7816895698789552e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.066399574279785,
+      "logits/rejected": 9.013555526733398,
+      "logps/chosen": -42.33242416381836,
+      "logps/rejected": -44.05732345581055,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -42.33242416381836,
+      "rewards/margins": 1.724898099899292,
+      "rewards/rejected": -44.05732345581055,
       "step": 4205
     },
     {
       "epoch": 2.2532196019401236,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.3869634917082e-11,
       "learning_rate": 1.7697862456752271e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.118454933166504,
+      "logits/rejected": 8.773681640625,
+      "logps/chosen": -42.39700698852539,
+      "logps/rejected": -43.76935577392578,
+      "loss": 0.0,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -42.39700698852539,
+      "rewards/margins": 1.372349500656128,
+      "rewards/rejected": -43.76935577392578,
       "step": 4210
     },
     {
       "epoch": 2.2558956347215253,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 7.532527352209602e-06,
       "learning_rate": 1.7579142610195124e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.82850456237793,
+      "logits/rejected": 8.392868041992188,
+      "logps/chosen": -43.370033264160156,
+      "logps/rejected": -44.02642059326172,
+      "loss": 0.0,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -43.370033264160156,
+      "rewards/margins": 0.6563886404037476,
+      "rewards/rejected": -44.02642059326172,
       "step": 4215
     },
     {
       "epoch": 2.258571667502927,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.5320614551379607e-10,
       "learning_rate": 1.7460737310938568e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.47398567199707,
+      "logits/rejected": 8.70628547668457,
+      "logps/chosen": -43.42267990112305,
+      "logps/rejected": -44.04685592651367,
+      "loss": 0.0,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -43.42267990112305,
+      "rewards/margins": 0.6241763830184937,
+      "rewards/rejected": -44.04685592651367,
       "step": 4220
     },
     {
       "epoch": 2.2612477002843283,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.3023732778125845e-10,
       "learning_rate": 1.734264770775133e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.031586647033691,
+      "logits/rejected": 8.20925521850586,
+      "logps/chosen": -42.42650604248047,
+      "logps/rejected": -44.3620719909668,
+      "loss": 0.0002,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -42.42650604248047,
+      "rewards/margins": 1.9355676174163818,
+      "rewards/rejected": -44.3620719909668,
       "step": 4225
     },
     {
       "epoch": 2.26392373306573,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.939435168885417e-09,
       "learning_rate": 1.7224874946339241e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.228693008422852,
+      "logits/rejected": 8.812799453735352,
+      "logps/chosen": -42.75141525268555,
+      "logps/rejected": -44.04648971557617,
+      "loss": 0.0,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": -42.75141525268555,
+      "rewards/margins": 1.295072317123413,
+      "rewards/rejected": -44.04648971557617,
       "step": 4230
     },
     {
       "epoch": 2.2665997658471317,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.402054977933239e-10,
       "learning_rate": 1.7107420169334186e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.116961479187012,
+      "logits/rejected": 8.883676528930664,
+      "logps/chosen": -43.965187072753906,
+      "logps/rejected": -44.140235900878906,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -43.965187072753906,
+      "rewards/margins": 0.17505022883415222,
+      "rewards/rejected": -44.140235900878906,
       "step": 4235
     },
     {
       "epoch": 2.269275798628533,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.1772358639588033e-08,
       "learning_rate": 1.6990284516282893e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.122404098510742,
+      "logits/rejected": 8.788342475891113,
+      "logps/chosen": -44.38725662231445,
+      "logps/rejected": -45.855464935302734,
+      "loss": 0.0,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -44.38725662231445,
+      "rewards/margins": 1.4682106971740723,
+      "rewards/rejected": -45.855464935302734,
       "step": 4240
     },
     {
       "epoch": 2.2719518314099347,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.1436833892050929e-06,
       "learning_rate": 1.687346912363602e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.333844184875488,
+      "logits/rejected": 8.777189254760742,
+      "logps/chosen": -43.29682922363281,
+      "logps/rejected": -44.67272186279297,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -43.29682922363281,
+      "rewards/margins": 1.3758885860443115,
+      "rewards/rejected": -44.67272186279297,
       "step": 4245
     },
     {
       "epoch": 2.2746278641913364,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.291591414056221e-06,
       "learning_rate": 1.675697512473697e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.850008010864258,
+      "logits/rejected": 8.159396171569824,
+      "logps/chosen": -42.38923645019531,
+      "logps/rejected": -43.2281379699707,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -42.38923645019531,
+      "rewards/margins": 0.8389037847518921,
+      "rewards/rejected": -43.2281379699707,
       "step": 4250
     },
     {
       "epoch": 2.2773038969727377,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.099846744219099e-05,
       "learning_rate": 1.6640803649811087e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.921536445617676,
+      "logits/rejected": 7.920711517333984,
+      "logps/chosen": -41.776832580566406,
+      "logps/rejected": -43.69265365600586,
+      "loss": 0.0,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -41.776832580566406,
+      "rewards/margins": 1.9158248901367188,
+      "rewards/rejected": -43.69265365600586,
       "step": 4255
     },
     {
       "epoch": 2.2799799297541394,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 8.506790364959634e-11,
       "learning_rate": 1.6524955825954472e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.039824485778809,
+      "logits/rejected": 8.914033889770508,
+      "logps/chosen": -43.91550827026367,
+      "logps/rejected": -44.114463806152344,
+      "loss": 0.0,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -43.91550827026367,
+      "rewards/margins": 0.19895082712173462,
+      "rewards/rejected": -44.114463806152344,
       "step": 4260
     },
     {
       "epoch": 2.282655962535541,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 7.449668752509071e-11,
       "learning_rate": 1.6409432777123277e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.453493118286133,
+      "logits/rejected": 8.80844783782959,
+      "logps/chosen": -42.1513786315918,
+      "logps/rejected": -43.998191833496094,
+      "loss": 0.0,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -42.1513786315918,
+      "rewards/margins": 1.846813440322876,
+      "rewards/rejected": -43.998191833496094,
       "step": 4265
     },
     {
       "epoch": 2.285331995316943,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 5.070532250693957e-07,
       "learning_rate": 1.6294235624122577e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.8313627243042,
+      "logits/rejected": 8.03431510925293,
+      "logps/chosen": -41.363250732421875,
+      "logps/rejected": -43.45961380004883,
+      "loss": 0.0,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -41.363250732421875,
+      "rewards/margins": 2.096357822418213,
+      "rewards/rejected": -43.45961380004883,
       "step": 4270
     },
     {
       "epoch": 2.288008028098344,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.97156346995716e-05,
       "learning_rate": 1.6179365484595697e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.09044361114502,
+      "logits/rejected": 8.647317886352539,
+      "logps/chosen": -42.99861526489258,
+      "logps/rejected": -44.119056701660156,
+      "loss": 0.0,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -42.99861526489258,
+      "rewards/margins": 1.1204428672790527,
+      "rewards/rejected": -44.119056701660156,
       "step": 4275
     },
     {
       "epoch": 2.290684060879746,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 6.238097462577515e-07,
       "learning_rate": 1.60648234730132e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.191560745239258,
+      "logits/rejected": 8.795047760009766,
+      "logps/chosen": -42.10997772216797,
+      "logps/rejected": -43.478328704833984,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -42.10997772216797,
+      "rewards/margins": 1.3683528900146484,
+      "rewards/rejected": -43.478328704833984,
       "step": 4280
     },
     {
       "epoch": 2.293360093661147,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.7994462064748028e-06,
       "learning_rate": 1.595061070066222e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.007406234741211,
+      "logits/rejected": 9.154937744140625,
+      "logps/chosen": -43.635414123535156,
+      "logps/rejected": -43.90370559692383,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5062500238418579,
+      "rewards/chosen": -43.635414123535156,
+      "rewards/margins": 0.26829349994659424,
+      "rewards/rejected": -43.90370559692383,
       "step": 4285
     },
     {
       "epoch": 2.296036126442549,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.331899347058868e-08,
       "learning_rate": 1.5836728275635542e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.42839241027832,
+      "logits/rejected": 8.707287788391113,
+      "logps/chosen": -43.73933029174805,
+      "logps/rejected": -45.089637756347656,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -43.73933029174805,
+      "rewards/margins": 1.3503055572509766,
+      "rewards/rejected": -45.089637756347656,
       "step": 4290
     },
     {
       "epoch": 2.2987121592239506,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.1261952875606925e-11,
       "learning_rate": 1.5723177302820984e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.148388862609863,
+      "logits/rejected": 8.937200546264648,
+      "logps/chosen": -42.85658645629883,
+      "logps/rejected": -44.64931106567383,
+      "loss": 0.0,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -42.85658645629883,
+      "rewards/margins": 1.7927252054214478,
+      "rewards/rejected": -44.64931106567383,
       "step": 4295
     },
     {
       "epoch": 2.3013881920053523,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 5.144613684375694e-10,
       "learning_rate": 1.5609958883890544e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.734636306762695,
+      "logits/rejected": 8.355271339416504,
+      "logps/chosen": -42.69615173339844,
+      "logps/rejected": -43.04405975341797,
+      "loss": 0.0,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -42.69615173339844,
+      "rewards/margins": 0.34790679812431335,
+      "rewards/rejected": -43.04405975341797,
       "step": 4300
     },
     {
       "epoch": 2.3040642247867535,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.9076431694630282e-13,
       "learning_rate": 1.5497074117289865e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.247693061828613,
+      "logits/rejected": 8.819948196411133,
+      "logps/chosen": -43.84877014160156,
+      "logps/rejected": -44.492122650146484,
+      "loss": 0.0,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -43.84877014160156,
+      "rewards/margins": 0.6433536410331726,
+      "rewards/rejected": -44.492122650146484,
       "step": 4305
     },
     {
       "epoch": 2.3067402575681553,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 7.636399860451499e-08,
       "learning_rate": 1.5384524098227402e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.422225952148438,
+      "logits/rejected": 8.724987983703613,
+      "logps/chosen": -43.411964416503906,
+      "logps/rejected": -44.942100524902344,
+      "loss": 0.0,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -43.411964416503906,
+      "rewards/margins": 1.530137300491333,
+      "rewards/rejected": -44.942100524902344,
       "step": 4310
     },
     {
       "epoch": 2.3094162903495565,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.4998625370364907e-09,
       "learning_rate": 1.5272309918663974e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.082725524902344,
+      "logits/rejected": 8.595535278320312,
+      "logps/chosen": -43.05567932128906,
+      "logps/rejected": -44.44702911376953,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -43.05567932128906,
+      "rewards/margins": 1.3913497924804688,
+      "rewards/rejected": -44.44702911376953,
       "step": 4315
     },
     {
       "epoch": 2.3120923231309582,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.6689369136725444e-09,
       "learning_rate": 1.516043266730201e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.244811058044434,
+      "logits/rejected": 8.717084884643555,
+      "logps/chosen": -43.39812088012695,
+      "logps/rejected": -44.62095260620117,
+      "loss": 0.0,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -43.39812088012695,
+      "rewards/margins": 1.2228361368179321,
+      "rewards/rejected": -44.62095260620117,
       "step": 4320
     },
     {
       "epoch": 2.31476835591236,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.100455603489451e-08,
       "learning_rate": 1.504889342957512e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.180676460266113,
+      "logits/rejected": 8.67148208618164,
+      "logps/chosen": -43.18669891357422,
+      "logps/rejected": -44.49002456665039,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -43.18669891357422,
+      "rewards/margins": 1.3033275604248047,
+      "rewards/rejected": -44.49002456665039,
       "step": 4325
     },
     {
       "epoch": 2.3174443886937617,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.5867202340473305e-07,
       "learning_rate": 1.4937693287637453e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.144622802734375,
+      "logits/rejected": 8.766639709472656,
+      "logps/chosen": -43.558021545410156,
+      "logps/rejected": -44.061065673828125,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -43.558021545410156,
+      "rewards/margins": 0.5030478835105896,
+      "rewards/rejected": -44.061065673828125,
       "step": 4330
     },
     {
       "epoch": 2.320120421475163,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.4873206498981649e-10,
       "learning_rate": 1.4826833320353305e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.09286880493164,
+      "logits/rejected": 8.727540969848633,
+      "logps/chosen": -42.87596130371094,
+      "logps/rejected": -43.59950256347656,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -42.87596130371094,
+      "rewards/margins": 0.7235434651374817,
+      "rewards/rejected": -43.59950256347656,
       "step": 4335
     },
     {
       "epoch": 2.3227964542565647,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.27176365555468e-08,
       "learning_rate": 1.4716314603286528e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.23253345489502,
+      "logits/rejected": 8.481476783752441,
+      "logps/chosen": -42.807376861572266,
+      "logps/rejected": -44.712669372558594,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -42.807376861572266,
+      "rewards/margins": 1.9052963256835938,
+      "rewards/rejected": -44.712669372558594,
       "step": 4340
     },
     {
       "epoch": 2.3254724870379664,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.2941480635934895e-07,
       "learning_rate": 1.4606138208690233e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.741349220275879,
+      "logits/rejected": 8.724069595336914,
+      "logps/chosen": -43.336788177490234,
+      "logps/rejected": -44.0677490234375,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -43.336788177490234,
+      "rewards/margins": 0.7309597730636597,
+      "rewards/rejected": -44.0677490234375,
       "step": 4345
     },
     {
       "epoch": 2.3281485198193677,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.2287024403415583e-11,
       "learning_rate": 1.4496305205496251e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.90785026550293,
+      "logits/rejected": 8.737528800964355,
+      "logps/chosen": -44.29743194580078,
+      "logps/rejected": -44.66332244873047,
+      "loss": 0.0,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -44.29743194580078,
+      "rewards/margins": 0.36587873101234436,
+      "rewards/rejected": -44.66332244873047,
       "step": 4350
     },
     {
       "epoch": 2.3308245526007694,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 8.668890570301739e-09,
       "learning_rate": 1.4386816659304895e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.492953300476074,
+      "logits/rejected": 8.876031875610352,
+      "logps/chosen": -44.036006927490234,
+      "logps/rejected": -44.11336898803711,
+      "loss": 0.0011,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -44.036006927490234,
+      "rewards/margins": 0.07736074924468994,
+      "rewards/rejected": -44.11336898803711,
       "step": 4355
     },
     {
       "epoch": 2.333500585382171,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.262696609355777e-05,
       "learning_rate": 1.4277673632374492e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.042167663574219,
+      "logits/rejected": 8.144396781921387,
+      "logps/chosen": -42.3798828125,
+      "logps/rejected": -44.473358154296875,
+      "loss": 0.0,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -42.3798828125,
+      "rewards/margins": 2.093472957611084,
+      "rewards/rejected": -44.473358154296875,
       "step": 4360
     },
     {
       "epoch": 2.3361766181635724,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 8.912362764443241e-06,
       "learning_rate": 1.416887718361119e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.104842185974121,
+      "logits/rejected": 9.123126029968262,
+      "logps/chosen": -44.14275360107422,
+      "logps/rejected": -45.2035026550293,
+      "loss": 0.0,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": -44.14275360107422,
+      "rewards/margins": 1.0607534646987915,
+      "rewards/rejected": -45.2035026550293,
       "step": 4365
     },
     {
       "epoch": 2.338852650944974,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.064646509395895e-11,
       "learning_rate": 1.406042836855859e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.455392837524414,
+      "logits/rejected": 9.015202522277832,
+      "logps/chosen": -44.02656555175781,
+      "logps/rejected": -44.52220916748047,
+      "loss": 0.0,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": -44.02656555175781,
+      "rewards/margins": 0.4956429898738861,
+      "rewards/rejected": -44.52220916748047,
       "step": 4370
     },
     {
       "epoch": 2.341528683726376,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.937515268137528e-07,
       "learning_rate": 1.3952328239387595e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.14968490600586,
+      "logits/rejected": 8.300264358520508,
+      "logps/chosen": -42.67992401123047,
+      "logps/rejected": -43.92139434814453,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -42.67992401123047,
+      "rewards/margins": 1.2414720058441162,
+      "rewards/rejected": -43.92139434814453,
       "step": 4375
     },
     {
       "epoch": 2.344204716507777,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.4018277863105718e-08,
       "learning_rate": 1.3844577844886109e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.254443168640137,
+      "logits/rejected": 8.511683464050293,
+      "logps/chosen": -41.791778564453125,
+      "logps/rejected": -43.45569610595703,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -41.791778564453125,
+      "rewards/margins": 1.6639143228530884,
+      "rewards/rejected": -43.45569610595703,
       "step": 4380
     },
     {
       "epoch": 2.346880749289179,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.051836029427001e-12,
       "learning_rate": 1.3737178230448955e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.885170936584473,
+      "logits/rejected": 8.619671821594238,
+      "logps/chosen": -43.519752502441406,
+      "logps/rejected": -44.45296859741211,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -43.519752502441406,
+      "rewards/margins": 0.9332124590873718,
+      "rewards/rejected": -44.45296859741211,
       "step": 4385
     },
     {
       "epoch": 2.3495567820705805,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 6.280604241054e-13,
       "learning_rate": 1.363013043806764e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.265891075134277,
+      "logits/rejected": 8.933459281921387,
+      "logps/chosen": -43.056278228759766,
+      "logps/rejected": -44.31239318847656,
+      "loss": 0.0,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -43.056278228759766,
+      "rewards/margins": 1.256115198135376,
+      "rewards/rejected": -44.31239318847656,
       "step": 4390
     },
     {
       "epoch": 2.3522328148519818,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.1861349984204927e-08,
       "learning_rate": 1.352343550632034e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.280359268188477,
+      "logits/rejected": 8.73161506652832,
+      "logps/chosen": -42.6974983215332,
+      "logps/rejected": -44.89132308959961,
+      "loss": 0.0,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -42.6974983215332,
+      "rewards/margins": 2.1938223838806152,
+      "rewards/rejected": -44.89132308959961,
       "step": 4395
     },
     {
       "epoch": 2.3549088476333835,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.2432950582382938e-09,
       "learning_rate": 1.3417094470361722e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.069659233093262,
+      "logits/rejected": 8.624251365661621,
+      "logps/chosen": -42.985801696777344,
+      "logps/rejected": -44.3646125793457,
+      "loss": 0.0,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -42.985801696777344,
+      "rewards/margins": 1.378806233406067,
+      "rewards/rejected": -44.3646125793457,
       "step": 4400
     },
     {
       "epoch": 2.3549088476333835,
-      "eval_logits/chosen": NaN,
-      "eval_logits/rejected": NaN,
-      "eval_logps/chosen": NaN,
-      "eval_logps/rejected": NaN,
-      "eval_loss": NaN,
-      "eval_rewards/accuracies": 0.0,
-      "eval_rewards/chosen": NaN,
-      "eval_rewards/margins": NaN,
-      "eval_rewards/rejected": NaN,
-      "eval_runtime": 34.9067,
-      "eval_samples_per_second": 38.531,
-      "eval_semantic_entropy": NaN,
-      "eval_steps_per_second": 9.654,
+      "eval_logits/chosen": 7.58363676071167,
+      "eval_logits/rejected": 7.291767597198486,
+      "eval_logps/chosen": -42.94476318359375,
+      "eval_logps/rejected": -44.052490234375,
+      "eval_loss": 2.09151721719536e-07,
+      "eval_rewards/accuracies": 0.5704748034477234,
+      "eval_rewards/chosen": -42.94476318359375,
+      "eval_rewards/margins": 1.1077219247817993,
+      "eval_rewards/rejected": -44.052490234375,
+      "eval_runtime": 40.5432,
+      "eval_samples_per_second": 33.174,
+      "eval_steps_per_second": 8.312,
       "step": 4400
     },
     {
       "epoch": 2.357584880414785,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.137727206451965e-07,
       "learning_rate": 1.3311108361913015e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.142969131469727,
+      "logits/rejected": 9.181938171386719,
+      "logps/chosen": -44.06669235229492,
+      "logps/rejected": -44.470558166503906,
+      "loss": 0.0,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -44.06669235229492,
+      "rewards/margins": 0.40386858582496643,
+      "rewards/rejected": -44.470558166503906,
       "step": 4405
     },
     {
       "epoch": 2.3602609131961865,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.2430418719987272e-06,
       "learning_rate": 1.3205478209251874e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.302844047546387,
+      "logits/rejected": 9.121850967407227,
+      "logps/chosen": -43.989173889160156,
+      "logps/rejected": -44.57482147216797,
+      "loss": 0.0,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -43.989173889160156,
+      "rewards/margins": 0.5856438875198364,
+      "rewards/rejected": -44.57482147216797,
       "step": 4410
     },
     {
       "epoch": 2.362936945977588,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.6140402659209794e-08,
       "learning_rate": 1.310020503720254e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.872774124145508,
+      "logits/rejected": 8.172953605651855,
+      "logps/chosen": -42.81657028198242,
+      "logps/rejected": -44.09730911254883,
+      "loss": 0.0002,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -42.81657028198242,
+      "rewards/margins": 1.2807376384735107,
+      "rewards/rejected": -44.09730911254883,
       "step": 4415
     },
     {
       "epoch": 2.36561297875899,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.5868902326904634e-12,
       "learning_rate": 1.2995289867125752e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.153031349182129,
+      "logits/rejected": 8.872515678405762,
+      "logps/chosen": -44.24485778808594,
+      "logps/rejected": -45.25538635253906,
+      "loss": 0.0,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -44.24485778808594,
+      "rewards/margins": 1.0105345249176025,
+      "rewards/rejected": -45.25538635253906,
       "step": 4420
     },
     {
       "epoch": 2.368289011540391,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.363046327658823e-09,
       "learning_rate": 1.2890733716908986e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.967996597290039,
+      "logits/rejected": 9.00118637084961,
+      "logps/chosen": -42.25114822387695,
+      "logps/rejected": -43.24909591674805,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -42.25114822387695,
+      "rewards/margins": 0.9979490041732788,
+      "rewards/rejected": -43.24909591674805,
       "step": 4425
     },
     {
       "epoch": 2.370965044321793,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 5.64828628500646e-11,
       "learning_rate": 1.2786537600956454e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.335803985595703,
+      "logits/rejected": 8.578765869140625,
+      "logps/chosen": -42.637569427490234,
+      "logps/rejected": -44.36760711669922,
+      "loss": 0.0,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -42.637569427490234,
+      "rewards/margins": 1.7300386428833008,
+      "rewards/rejected": -44.36760711669922,
       "step": 4430
     },
     {
       "epoch": 2.3736410771031946,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.7188667113474353e-08,
       "learning_rate": 1.268270253017933e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.207649230957031,
+      "logits/rejected": 8.488981246948242,
+      "logps/chosen": -43.47707748413086,
+      "logps/rejected": -44.49448013305664,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -43.47707748413086,
+      "rewards/margins": 1.0174052715301514,
+      "rewards/rejected": -44.49448013305664,
       "step": 4435
     },
     {
       "epoch": 2.376317109884596,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.0235296188135633e-08,
       "learning_rate": 1.257922951198591e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.092676162719727,
+      "logits/rejected": 8.0695219039917,
+      "logps/chosen": -42.53782653808594,
+      "logps/rejected": -43.3267822265625,
+      "loss": 0.0,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -42.53782653808594,
+      "rewards/margins": 0.7889548540115356,
+      "rewards/rejected": -43.3267822265625,
       "step": 4440
     },
     {
       "epoch": 2.3789931426659976,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 5.959831703014962e-09,
       "learning_rate": 1.24761195502719e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.072807312011719,
+      "logits/rejected": 8.297175407409668,
+      "logps/chosen": -43.0941047668457,
+      "logps/rejected": -44.660926818847656,
+      "loss": 0.0,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -43.0941047668457,
+      "rewards/margins": 1.5668278932571411,
+      "rewards/rejected": -44.660926818847656,
       "step": 4445
     },
     {
       "epoch": 2.3816691754473993,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 0.0006455802202963956,
       "learning_rate": 1.2373373645410573e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.012727737426758,
+      "logits/rejected": 8.577821731567383,
+      "logps/chosen": -44.019378662109375,
+      "logps/rejected": -44.51078414916992,
+      "loss": 0.0,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -44.019378662109375,
+      "rewards/margins": 0.49140363931655884,
+      "rewards/rejected": -44.51078414916992,
       "step": 4450
     },
     {
       "epoch": 2.384345208228801,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.178148998749168e-10,
       "learning_rate": 1.2270992794243175e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.32244873046875,
+      "logits/rejected": 8.821152687072754,
+      "logps/chosen": -44.0055046081543,
+      "logps/rejected": -44.87663269042969,
+      "loss": 0.0,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -44.0055046081543,
+      "rewards/margins": 0.8711255192756653,
+      "rewards/rejected": -44.87663269042969,
       "step": 4455
     },
     {
       "epoch": 2.3870212410102023,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.3801590406911339e-08,
       "learning_rate": 1.2168977990069147e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.49370288848877,
+      "logits/rejected": 8.74947452545166,
+      "logps/chosen": -43.95867156982422,
+      "logps/rejected": -44.999305725097656,
+      "loss": 0.0,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -43.95867156982422,
+      "rewards/margins": 1.0406330823898315,
+      "rewards/rejected": -44.999305725097656,
       "step": 4460
     },
     {
       "epoch": 2.389697273791604,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 8.824963748286137e-10,
       "learning_rate": 1.206733022263659e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.082025527954102,
+      "logits/rejected": 8.632207870483398,
+      "logps/chosen": -42.01361846923828,
+      "logps/rejected": -43.52204895019531,
+      "loss": 0.0,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -42.01361846923828,
+      "rewards/margins": 1.5084317922592163,
+      "rewards/rejected": -43.52204895019531,
       "step": 4465
     },
     {
       "epoch": 2.3923733065730053,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.3673137598535915e-08,
       "learning_rate": 1.1966050478132572e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.137206077575684,
+      "logits/rejected": 8.792137145996094,
+      "logps/chosen": -44.097599029541016,
+      "logps/rejected": -44.59095764160156,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -44.097599029541016,
+      "rewards/margins": 0.4933616518974304,
+      "rewards/rejected": -44.59095764160156,
       "step": 4470
     },
     {
       "epoch": 2.395049339354407,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 7.719831477307347e-08,
       "learning_rate": 1.1865139739173635e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.21369457244873,
+      "logits/rejected": 8.449313163757324,
+      "logps/chosen": -45.10253143310547,
+      "logps/rejected": -45.31608200073242,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -45.10253143310547,
+      "rewards/margins": 0.21355172991752625,
+      "rewards/rejected": -45.31608200073242,
       "step": 4475
     },
     {
       "epoch": 2.3977253721358087,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 8.309703240889619e-08,
       "learning_rate": 1.1764598984796187e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.55870246887207,
+      "logits/rejected": 8.969377517700195,
+      "logps/chosen": -43.30122756958008,
+      "logps/rejected": -44.269256591796875,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -43.30122756958008,
+      "rewards/margins": 0.9680284261703491,
+      "rewards/rejected": -44.269256591796875,
       "step": 4480
     },
     {
       "epoch": 2.4004014049172104,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.707401359543554e-10,
       "learning_rate": 1.1664429190447095e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.745327949523926,
+      "logits/rejected": 8.617340087890625,
+      "logps/chosen": -42.18756866455078,
+      "logps/rejected": -44.07577896118164,
+      "loss": 0.0,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -42.18756866455078,
+      "rewards/margins": 1.888204574584961,
+      "rewards/rejected": -44.07577896118164,
       "step": 4485
     },
     {
       "epoch": 2.4030774376986117,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.184333434134072e-09,
       "learning_rate": 1.1564631327974122e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.120786666870117,
+      "logits/rejected": 8.588520050048828,
+      "logps/chosen": -42.65946578979492,
+      "logps/rejected": -44.833744049072266,
+      "loss": 0.0,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -42.65946578979492,
+      "rewards/margins": 2.1742758750915527,
+      "rewards/rejected": -44.833744049072266,
       "step": 4490
     },
     {
       "epoch": 2.4057534704800134,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 9.60038359522924e-10,
       "learning_rate": 1.1465206365616587e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.228276252746582,
+      "logits/rejected": 8.587491035461426,
+      "logps/chosen": -43.101383209228516,
+      "logps/rejected": -44.66340255737305,
+      "loss": 0.0,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -43.101383209228516,
+      "rewards/margins": 1.562018871307373,
+      "rewards/rejected": -44.66340255737305,
       "step": 4495
     },
     {
       "epoch": 2.408429503261415,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.097560065575246e-12,
       "learning_rate": 1.1366155267995887e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.23318099975586,
+      "logits/rejected": 9.171361923217773,
+      "logps/chosen": -44.490020751953125,
+      "logps/rejected": -45.20877456665039,
+      "loss": 0.0,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -44.490020751953125,
+      "rewards/margins": 0.7187555432319641,
+      "rewards/rejected": -45.20877456665039,
       "step": 4500
     },
     {
       "epoch": 2.4111055360428164,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 6.109008494100467e-06,
       "learning_rate": 1.1267478996106228e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.225412368774414,
+      "logits/rejected": 8.4717435836792,
+      "logps/chosen": -42.46234130859375,
+      "logps/rejected": -43.9901123046875,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -42.46234130859375,
+      "rewards/margins": 1.5277734994888306,
+      "rewards/rejected": -43.9901123046875,
       "step": 4505
     },
     {
       "epoch": 2.413781568824218,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.621012448107174e-08,
       "learning_rate": 1.116917850730521e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.25502872467041,
+      "logits/rejected": 8.600152969360352,
+      "logps/chosen": -43.92424392700195,
+      "logps/rejected": -43.80561828613281,
+      "loss": 0.0,
+      "rewards/accuracies": 0.4937500059604645,
+      "rewards/chosen": -43.92424392700195,
+      "rewards/margins": -0.118623748421669,
+      "rewards/rejected": -43.80561828613281,
       "step": 4510
     },
     {
       "epoch": 2.41645760160562,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 9.10958713801315e-08,
       "learning_rate": 1.1071254755304637e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.845125198364258,
+      "logits/rejected": 8.52183723449707,
+      "logps/chosen": -42.841552734375,
+      "logps/rejected": -43.588497161865234,
+      "loss": 0.0,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -42.841552734375,
+      "rewards/margins": 0.7469453811645508,
+      "rewards/rejected": -43.588497161865234,
       "step": 4515
     },
     {
       "epoch": 2.419133634387021,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.761370057397501e-10,
       "learning_rate": 1.0973708690161143e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.203384399414062,
+      "logits/rejected": 8.749646186828613,
+      "logps/chosen": -41.85224151611328,
+      "logps/rejected": -43.942543029785156,
+      "loss": 0.0,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -41.85224151611328,
+      "rewards/margins": 2.090305805206299,
+      "rewards/rejected": -43.942543029785156,
       "step": 4520
     },
     {
       "epoch": 2.421809667168423,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.1468272897694934e-10,
       "learning_rate": 1.0876541258267119e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.081708908081055,
+      "logits/rejected": 8.44297981262207,
+      "logps/chosen": -42.12873840332031,
+      "logps/rejected": -42.96746826171875,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -42.12873840332031,
+      "rewards/margins": 0.8387306332588196,
+      "rewards/rejected": -42.96746826171875,
       "step": 4525
     },
     {
       "epoch": 2.4244856999498245,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.2739414547381716e-06,
       "learning_rate": 1.0779753402341379e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.346722602844238,
+      "logits/rejected": 8.936014175415039,
+      "logps/chosen": -42.54629898071289,
+      "logps/rejected": -43.609249114990234,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5062500238418579,
+      "rewards/chosen": -42.54629898071289,
+      "rewards/margins": 1.0629475116729736,
+      "rewards/rejected": -43.609249114990234,
       "step": 4530
     },
     {
       "epoch": 2.427161732731226,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.3633868479780688e-08,
       "learning_rate": 1.0683346061420157e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.418802261352539,
+      "logits/rejected": 8.973909378051758,
+      "logps/chosen": -43.503509521484375,
+      "logps/rejected": -45.15605926513672,
+      "loss": 0.0,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -43.503509521484375,
+      "rewards/margins": 1.6525547504425049,
+      "rewards/rejected": -45.15605926513672,
       "step": 4535
     },
     {
       "epoch": 2.4298377655126275,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.468131527788943e-07,
       "learning_rate": 1.0587320170847874e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.299947738647461,
+      "logits/rejected": 8.888525009155273,
+      "logps/chosen": -44.736328125,
+      "logps/rejected": -44.92253112792969,
+      "loss": 0.0,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": -44.736328125,
+      "rewards/margins": 0.1862012892961502,
+      "rewards/rejected": -44.92253112792969,
       "step": 4540
     },
     {
       "epoch": 2.4325137982940293,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 8.094425573062365e-08,
       "learning_rate": 1.0491676662268156e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.102094650268555,
+      "logits/rejected": 8.735517501831055,
+      "logps/chosen": -44.068443298339844,
+      "logps/rejected": -45.31291961669922,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -44.068443298339844,
+      "rewards/margins": 1.2444796562194824,
+      "rewards/rejected": -45.31291961669922,
       "step": 4545
     },
     {
       "epoch": 2.4351898310754305,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 9.565163343963566e-07,
       "learning_rate": 1.0396416463614732e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.991244316101074,
+      "logits/rejected": 8.889235496520996,
+      "logps/chosen": -42.168846130371094,
+      "logps/rejected": -43.555091857910156,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -42.168846130371094,
+      "rewards/margins": 1.3862473964691162,
+      "rewards/rejected": -43.555091857910156,
       "step": 4550
     },
     {
       "epoch": 2.4378658638568322,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 6.569198730149456e-11,
       "learning_rate": 1.0301540499102479e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.95669174194336,
+      "logits/rejected": 8.926271438598633,
+      "logps/chosen": -43.23931121826172,
+      "logps/rejected": -44.545623779296875,
+      "loss": 0.0,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -43.23931121826172,
+      "rewards/margins": 1.306312084197998,
+      "rewards/rejected": -44.545623779296875,
       "step": 4555
     },
     {
       "epoch": 2.440541896638234,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 5.905434562500467e-08,
       "learning_rate": 1.0207049689218405e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.12828254699707,
+      "logits/rejected": 8.25415325164795,
+      "logps/chosen": -42.28589630126953,
+      "logps/rejected": -44.01564025878906,
+      "loss": 0.0,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -42.28589630126953,
+      "rewards/margins": 1.7297471761703491,
+      "rewards/rejected": -44.01564025878906,
       "step": 4560
     },
     {
       "epoch": 2.4432179294196352,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 0.19605552077566635,
       "learning_rate": 1.0112944950712782e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.721559524536133,
+      "logits/rejected": 8.356966018676758,
+      "logps/chosen": -42.18324279785156,
+      "logps/rejected": -43.19923400878906,
+      "loss": 0.0006,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -42.18324279785156,
+      "rewards/margins": 1.0159876346588135,
+      "rewards/rejected": -43.19923400878906,
       "step": 4565
     },
     {
       "epoch": 2.445893962201037,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.159598733161546e-07,
       "learning_rate": 1.0019227196590174e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.34399127960205,
+      "logits/rejected": 8.758341789245605,
+      "logps/chosen": -43.436161041259766,
+      "logps/rejected": -45.19282913208008,
+      "loss": 0.0,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -43.436161041259766,
+      "rewards/margins": 1.7566697597503662,
+      "rewards/rejected": -45.19282913208008,
       "step": 4570
     },
     {
       "epoch": 2.4485699949824387,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.8619680091328555e-11,
       "learning_rate": 9.925897336100664e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.596741676330566,
+      "logits/rejected": 8.583913803100586,
+      "logps/chosen": -44.03644943237305,
+      "logps/rejected": -44.815696716308594,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -44.03644943237305,
+      "rewards/margins": 0.7792443037033081,
+      "rewards/rejected": -44.815696716308594,
       "step": 4575
     },
     {
       "epoch": 2.45124602776384,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.2907628883556757e-09,
       "learning_rate": 9.832956274730946e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.77738094329834,
+      "logits/rejected": 8.592802047729492,
+      "logps/chosen": -42.731876373291016,
+      "logps/rejected": -44.014312744140625,
+      "loss": 0.0,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -42.731876373291016,
+      "rewards/margins": 1.2824397087097168,
+      "rewards/rejected": -44.014312744140625,
       "step": 4580
     },
     {
       "epoch": 2.4539220605452416,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.1553071307698722e-06,
       "learning_rate": 9.740404914195633e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.135787010192871,
+      "logits/rejected": 8.799894332885742,
+      "logps/chosen": -43.23937225341797,
+      "logps/rejected": -43.849246978759766,
+      "loss": 0.0,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -43.23937225341797,
+      "rewards/margins": 0.6098742485046387,
+      "rewards/rejected": -43.849246978759766,
       "step": 4585
     },
     {
       "epoch": 2.4565980933266434,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.478421466199099e-09,
       "learning_rate": 9.648244152428392e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.190836906433105,
+      "logits/rejected": 8.90406608581543,
+      "logps/chosen": -43.241981506347656,
+      "logps/rejected": -44.98204040527344,
+      "loss": 0.0,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -43.241981506347656,
+      "rewards/margins": 1.740060567855835,
+      "rewards/rejected": -44.98204040527344,
       "step": 4590
     },
     {
       "epoch": 2.4592741261080446,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.902457079906106e-06,
       "learning_rate": 9.556474883573379e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.186389923095703,
+      "logits/rejected": 8.845205307006836,
+      "logps/chosen": -43.44452667236328,
+      "logps/rejected": -45.01343536376953,
+      "loss": 0.0,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -43.44452667236328,
+      "rewards/margins": 1.5689066648483276,
+      "rewards/rejected": -45.01343536376953,
       "step": 4595
     },
     {
       "epoch": 2.4619501588894463,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.090602261307582e-13,
       "learning_rate": 9.465097997976412e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.1355562210083,
+      "logits/rejected": 8.413407325744629,
+      "logps/chosen": -42.12215042114258,
+      "logps/rejected": -43.92107009887695,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -42.12215042114258,
+      "rewards/margins": 1.79892098903656,
+      "rewards/rejected": -43.92107009887695,
       "step": 4600
     },
     {
       "epoch": 2.464626191670848,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 5.643186007673075e-09,
       "learning_rate": 9.374114382176457e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.422418594360352,
+      "logits/rejected": 8.689146041870117,
+      "logps/chosen": -43.20514678955078,
+      "logps/rejected": -43.85572052001953,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5062500238418579,
+      "rewards/chosen": -43.20514678955078,
+      "rewards/margins": 0.6505736112594604,
+      "rewards/rejected": -43.85572052001953,
       "step": 4605
     },
     {
       "epoch": 2.46730222445225,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 0.0005032262561090332,
       "learning_rate": 9.283524918896945e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.425394058227539,
+      "logits/rejected": 8.85915470123291,
+      "logps/chosen": -43.71038818359375,
+      "logps/rejected": -45.225032806396484,
+      "loss": 0.0,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -43.71038818359375,
+      "rewards/margins": 1.5146474838256836,
+      "rewards/rejected": -45.225032806396484,
       "step": 4610
     },
     {
       "epoch": 2.469978257233651,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 7.370446268456402e-13,
       "learning_rate": 9.193330487037232e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.175050735473633,
+      "logits/rejected": 8.738815307617188,
+      "logps/chosen": -43.13275909423828,
+      "logps/rejected": -44.79536819458008,
+      "loss": 0.0,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -43.13275909423828,
+      "rewards/margins": 1.6626050472259521,
+      "rewards/rejected": -44.79536819458008,
       "step": 4615
     },
     {
       "epoch": 2.4726542900150528,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.1216955098910236e-13,
       "learning_rate": 9.103531961664118e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.888827323913574,
+      "logits/rejected": 8.366586685180664,
+      "logps/chosen": -43.796363830566406,
+      "logps/rejected": -45.556129455566406,
+      "loss": 0.0,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -43.796363830566406,
+      "rewards/margins": 1.7597652673721313,
+      "rewards/rejected": -45.556129455566406,
       "step": 4620
     },
     {
       "epoch": 2.475330322796454,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.4830532008993502e-10,
       "learning_rate": 9.014130214003269e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.817378044128418,
+      "logits/rejected": 8.828986167907715,
+      "logps/chosen": -42.25873565673828,
+      "logps/rejected": -43.57822799682617,
+      "loss": 0.0,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -42.25873565673828,
+      "rewards/margins": 1.319486141204834,
+      "rewards/rejected": -43.57822799682617,
       "step": 4625
     },
     {
       "epoch": 2.4780063555778558,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.854193146921064e-08,
       "learning_rate": 8.925126111430848e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.268295288085938,
+      "logits/rejected": 8.089634895324707,
+      "logps/chosen": -42.152252197265625,
+      "logps/rejected": -43.164939880371094,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -42.152252197265625,
+      "rewards/margins": 1.0126903057098389,
+      "rewards/rejected": -43.164939880371094,
       "step": 4630
     },
     {
       "epoch": 2.4806823883592575,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 6.156027836353993e-08,
       "learning_rate": 8.83652051746504e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.214881896972656,
+      "logits/rejected": 8.371711730957031,
+      "logps/chosen": -43.5845947265625,
+      "logps/rejected": -43.76476287841797,
+      "loss": 0.0,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -43.5845947265625,
+      "rewards/margins": 0.18017558753490448,
+      "rewards/rejected": -43.76476287841797,
       "step": 4635
     },
     {
       "epoch": 2.483358421140659,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.1780720295974752e-11,
       "learning_rate": 8.748314291757696e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.744614601135254,
+      "logits/rejected": 8.606954574584961,
+      "logps/chosen": -42.13679122924805,
+      "logps/rejected": -44.540260314941406,
+      "loss": 0.0,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -42.13679122924805,
+      "rewards/margins": 2.4034669399261475,
+      "rewards/rejected": -44.540260314941406,
       "step": 4640
     },
     {
       "epoch": 2.4860344539220605,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 0.00046056731037930645,
       "learning_rate": 8.660508290086032e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.15407657623291,
+      "logits/rejected": 8.441764831542969,
+      "logps/chosen": -42.433143615722656,
+      "logps/rejected": -43.31450653076172,
+      "loss": 0.0,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -42.433143615722656,
+      "rewards/margins": 0.8813636898994446,
+      "rewards/rejected": -43.31450653076172,
       "step": 4645
     },
     {
       "epoch": 2.488710486703462,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.4789977070032176e-09,
       "learning_rate": 8.573103364344231e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.336080551147461,
+      "logits/rejected": 8.67591381072998,
+      "logps/chosen": -42.2813835144043,
+      "logps/rejected": -44.98908615112305,
+      "loss": 0.0,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -42.2813835144043,
+      "rewards/margins": 2.7077033519744873,
+      "rewards/rejected": -44.98908615112305,
       "step": 4650
     },
     {
       "epoch": 2.4913865194848634,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.0095563167156927e-10,
       "learning_rate": 8.486100362535292e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.016302108764648,
+      "logits/rejected": 8.631976127624512,
+      "logps/chosen": -42.29338836669922,
+      "logps/rejected": -43.491416931152344,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -42.29338836669922,
+      "rewards/margins": 1.198028326034546,
+      "rewards/rejected": -43.491416931152344,
       "step": 4655
     },
     {
       "epoch": 2.494062552266265,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.1782215295645974e-05,
       "learning_rate": 8.399500128762693e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.841638565063477,
+      "logits/rejected": 8.438671112060547,
+      "logps/chosen": -43.53633117675781,
+      "logps/rejected": -44.23197555541992,
+      "loss": 0.0,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -43.53633117675781,
+      "rewards/margins": 0.6956467628479004,
+      "rewards/rejected": -44.23197555541992,
       "step": 4660
     },
     {
       "epoch": 2.496738585047667,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.3249847462314841e-08,
       "learning_rate": 8.313303503222313e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.326006889343262,
+      "logits/rejected": 8.895910263061523,
+      "logps/chosen": -43.320247650146484,
+      "logps/rejected": -42.704132080078125,
+      "loss": 0.0,
+      "rewards/accuracies": 0.46875,
+      "rewards/chosen": -43.320247650146484,
+      "rewards/margins": -0.6161131858825684,
+      "rewards/rejected": -42.704132080078125,
       "step": 4665
     },
     {
       "epoch": 2.4994146178290686,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.7773475599607546e-07,
       "learning_rate": 8.227511322194164e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.266204833984375,
+      "logits/rejected": 8.7720947265625,
+      "logps/chosen": -42.21365737915039,
+      "logps/rejected": -43.435455322265625,
+      "loss": 0.0,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -42.21365737915039,
+      "rewards/margins": 1.2218005657196045,
+      "rewards/rejected": -43.435455322265625,
       "step": 4670
     },
     {
       "epoch": 2.50209065061047,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.7695469312968915e-09,
       "learning_rate": 8.142124418034385e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.141852378845215,
+      "logits/rejected": 8.288122177124023,
+      "logps/chosen": -43.673988342285156,
+      "logps/rejected": -44.82506561279297,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -43.673988342285156,
+      "rewards/margins": 1.151077389717102,
+      "rewards/rejected": -44.82506561279297,
       "step": 4675
     },
     {
       "epoch": 2.5047666833918716,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.980043802572538e-10,
       "learning_rate": 8.057143619167073e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.759631156921387,
+      "logits/rejected": 8.601361274719238,
+      "logps/chosen": -43.098846435546875,
+      "logps/rejected": -44.34484100341797,
+      "loss": 0.0,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -43.098846435546875,
+      "rewards/margins": 1.2459893226623535,
+      "rewards/rejected": -44.34484100341797,
       "step": 4680
     },
     {
       "epoch": 2.507442716173273,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.619037474401766e-10,
       "learning_rate": 7.97256975007633e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.121816635131836,
+      "logits/rejected": 8.402771949768066,
+      "logps/chosen": -42.12567138671875,
+      "logps/rejected": -43.63753890991211,
+      "loss": 0.0,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -42.12567138671875,
+      "rewards/margins": 1.5118703842163086,
+      "rewards/rejected": -43.63753890991211,
       "step": 4685
     },
     {
       "epoch": 2.5101187489546746,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 0.09768188870803572,
       "learning_rate": 7.888403631298186e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.689119338989258,
+      "logits/rejected": 8.800952911376953,
+      "logps/chosen": -42.88166427612305,
+      "logps/rejected": -44.109832763671875,
+      "loss": 0.0002,
+      "rewards/accuracies": 0.48124998807907104,
+      "rewards/chosen": -42.88166427612305,
+      "rewards/margins": 1.228164792060852,
+      "rewards/rejected": -44.109832763671875,
       "step": 4690
     },
     {
       "epoch": 2.5127947817360763,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.538492359074244e-09,
       "learning_rate": 7.804646079412719e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.839693069458008,
+      "logits/rejected": 8.37195110321045,
+      "logps/chosen": -42.27249526977539,
+      "logps/rejected": -43.85920715332031,
+      "loss": 0.0,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -42.27249526977539,
+      "rewards/margins": 1.5867151021957397,
+      "rewards/rejected": -43.85920715332031,
       "step": 4695
     },
     {
       "epoch": 2.515470814517478,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.7601590985472095e-10,
       "learning_rate": 7.72129790703604e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.232236862182617,
+      "logits/rejected": 8.781030654907227,
+      "logps/chosen": -42.43456268310547,
+      "logps/rejected": -43.71471405029297,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -42.43456268310547,
+      "rewards/margins": 1.280151128768921,
+      "rewards/rejected": -43.71471405029297,
       "step": 4700
     },
     {
       "epoch": 2.5181468472988793,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.567319141993614e-05,
       "learning_rate": 7.638359922812504e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.958989143371582,
+      "logits/rejected": 8.665552139282227,
+      "logps/chosen": -43.41173553466797,
+      "logps/rejected": -43.88212585449219,
+      "loss": 0.0,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -43.41173553466797,
+      "rewards/margins": 0.47039252519607544,
+      "rewards/rejected": -43.88212585449219,
       "step": 4705
     },
     {
       "epoch": 2.520822880080281,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.4408883132354555e-07,
       "learning_rate": 7.555832931406774e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.118542671203613,
+      "logits/rejected": 8.412516593933105,
+      "logps/chosen": -42.43720245361328,
+      "logps/rejected": -44.114192962646484,
+      "loss": 0.0,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -42.43720245361328,
+      "rewards/margins": 1.6769895553588867,
+      "rewards/rejected": -44.114192962646484,
       "step": 4710
     },
     {
       "epoch": 2.5234989128616827,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 6.101686581962118e-13,
       "learning_rate": 7.47371773349611e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.923929214477539,
+      "logits/rejected": 8.91518783569336,
+      "logps/chosen": -42.96770095825195,
+      "logps/rejected": -44.249420166015625,
+      "loss": 0.0,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -42.96770095825195,
+      "rewards/margins": 1.2817184925079346,
+      "rewards/rejected": -44.249420166015625,
       "step": 4715
     },
     {
       "epoch": 2.526174945643084,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 6.473310034087893e-06,
       "learning_rate": 7.392015125762496e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.949080467224121,
+      "logits/rejected": 8.5923490524292,
+      "logps/chosen": -43.18629837036133,
+      "logps/rejected": -44.24211883544922,
+      "loss": 0.0,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -43.18629837036133,
+      "rewards/margins": 1.0558218955993652,
+      "rewards/rejected": -44.24211883544922,
       "step": 4720
     },
     {
       "epoch": 2.5288509784244857,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 7.948510177439446e-07,
       "learning_rate": 7.310725900885018e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.120261192321777,
+      "logits/rejected": 9.004095077514648,
+      "logps/chosen": -43.56443786621094,
+      "logps/rejected": -43.506187438964844,
+      "loss": 0.0,
+      "rewards/accuracies": 0.4937500059604645,
+      "rewards/chosen": -43.56443786621094,
+      "rewards/margins": -0.05825188010931015,
+      "rewards/rejected": -43.506187438964844,
       "step": 4725
     },
     {
       "epoch": 2.5315270112058874,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 9.222741127483986e-10,
       "learning_rate": 7.229850847532076e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.934659004211426,
+      "logits/rejected": 8.441221237182617,
+      "logps/chosen": -43.583702087402344,
+      "logps/rejected": -44.898075103759766,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -43.583702087402344,
+      "rewards/margins": 1.314375877380371,
+      "rewards/rejected": -44.898075103759766,
       "step": 4730
     },
     {
       "epoch": 2.5342030439872887,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.4504388257674365e-09,
       "learning_rate": 7.149390750353779e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.475000381469727,
+      "logits/rejected": 8.893838882446289,
+      "logps/chosen": -42.61485290527344,
+      "logps/rejected": -43.29773712158203,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -42.61485290527344,
+      "rewards/margins": 0.6828866004943848,
+      "rewards/rejected": -43.29773712158203,
       "step": 4735
     },
     {
       "epoch": 2.5368790767686904,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.551103097822002e-11,
       "learning_rate": 7.069346389974374e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.223368644714355,
+      "logits/rejected": 8.584945678710938,
+      "logps/chosen": -42.723670959472656,
+      "logps/rejected": -44.25645446777344,
+      "loss": 0.0,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -42.723670959472656,
+      "rewards/margins": 1.5327866077423096,
+      "rewards/rejected": -44.25645446777344,
       "step": 4740
     },
     {
       "epoch": 2.539555109550092,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.799314031257049e-08,
       "learning_rate": 6.989718542984563e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.01329517364502,
+      "logits/rejected": 9.00218391418457,
+      "logps/chosen": -43.31816864013672,
+      "logps/rejected": -44.022430419921875,
+      "loss": 0.0,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -43.31816864013672,
+      "rewards/margins": 0.7042635679244995,
+      "rewards/rejected": -44.022430419921875,
       "step": 4745
     },
     {
       "epoch": 2.5422311423314934,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.013353981105687e-11,
       "learning_rate": 6.9105079819341e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.109373092651367,
+      "logits/rejected": 7.922883033752441,
+      "logps/chosen": -43.286582946777344,
+      "logps/rejected": -44.85076904296875,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -43.286582946777344,
+      "rewards/margins": 1.5641835927963257,
+      "rewards/rejected": -44.85076904296875,
       "step": 4750
     },
     {
       "epoch": 2.544907175112895,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.363193649779592e-10,
       "learning_rate": 6.831715475324163e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.389158248901367,
+      "logits/rejected": 8.734638214111328,
+      "logps/chosen": -43.528594970703125,
+      "logps/rejected": -44.85797882080078,
+      "loss": 0.0,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -43.528594970703125,
+      "rewards/margins": 1.3293819427490234,
+      "rewards/rejected": -44.85797882080078,
       "step": 4755
     },
     {
       "epoch": 2.547583207894297,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 6.73110666216195e-08,
       "learning_rate": 6.753341787600026e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.577058792114258,
+      "logits/rejected": 9.203988075256348,
+      "logps/chosen": -43.840457916259766,
+      "logps/rejected": -44.656455993652344,
+      "loss": 0.0,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -43.840457916259766,
+      "rewards/margins": 0.8160001635551453,
+      "rewards/rejected": -44.656455993652344,
       "step": 4760
     },
     {
       "epoch": 2.5502592406756985,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 8.597062998286907e-11,
       "learning_rate": 6.67538767914353e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.302873611450195,
+      "logits/rejected": 8.718219757080078,
+      "logps/chosen": -43.999595642089844,
+      "logps/rejected": -45.015052795410156,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -43.999595642089844,
+      "rewards/margins": 1.0154579877853394,
+      "rewards/rejected": -45.015052795410156,
       "step": 4765
     },
     {
       "epoch": 2.5529352734571,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 8.902378285635942e-06,
       "learning_rate": 6.597853906265793e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.784675598144531,
+      "logits/rejected": 8.37619686126709,
+      "logps/chosen": -42.801353454589844,
+      "logps/rejected": -44.62187576293945,
+      "loss": 0.0,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -42.801353454589844,
+      "rewards/margins": 1.8205153942108154,
+      "rewards/rejected": -44.62187576293945,
       "step": 4770
     },
     {
       "epoch": 2.5556113062385015,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.6874402235025962e-11,
       "learning_rate": 6.5207412211998e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.940374374389648,
+      "logits/rejected": 8.66877269744873,
+      "logps/chosen": -43.24928665161133,
+      "logps/rejected": -44.148502349853516,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -43.24928665161133,
+      "rewards/margins": 0.8992165327072144,
+      "rewards/rejected": -44.148502349853516,
       "step": 4775
     },
     {
       "epoch": 2.558287339019903,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.947176296804689e-07,
       "learning_rate": 6.444050372093186e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.13990592956543,
+      "logits/rejected": 8.96580696105957,
+      "logps/chosen": -43.406429290771484,
+      "logps/rejected": -44.29347229003906,
+      "loss": 0.0,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -43.406429290771484,
+      "rewards/margins": 0.8870447278022766,
+      "rewards/rejected": -44.29347229003906,
       "step": 4780
     },
     {
       "epoch": 2.5609633718013045,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.61470305220332e-10,
       "learning_rate": 6.367782103000873e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.231401443481445,
+      "logits/rejected": 8.809134483337402,
+      "logps/chosen": -43.69819641113281,
+      "logps/rejected": -44.2975959777832,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -43.69819641113281,
+      "rewards/margins": 0.5994040966033936,
+      "rewards/rejected": -44.2975959777832,
       "step": 4785
     },
     {
       "epoch": 2.5636394045827062,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.206934296860541e-11,
       "learning_rate": 6.29193715387798e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.969747543334961,
+      "logits/rejected": 8.523025512695312,
+      "logps/chosen": -42.60356903076172,
+      "logps/rejected": -43.568359375,
+      "loss": 0.0005,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -42.60356903076172,
+      "rewards/margins": 0.9647868275642395,
+      "rewards/rejected": -43.568359375,
       "step": 4790
     },
     {
       "epoch": 2.566315437364108,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.7486743581350295e-12,
       "learning_rate": 6.216516260572502e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.140700340270996,
+      "logits/rejected": 8.706997871398926,
+      "logps/chosen": -43.59941864013672,
+      "logps/rejected": -44.77802276611328,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -43.59941864013672,
+      "rewards/margins": 1.178605556488037,
+      "rewards/rejected": -44.77802276611328,
       "step": 4795
     },
     {
       "epoch": 2.568991470145509,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.05459567632616e-08,
       "learning_rate": 6.141520154818297e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.3609037399292,
+      "logits/rejected": 8.919872283935547,
+      "logps/chosen": -43.40058898925781,
+      "logps/rejected": -44.294105529785156,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -43.40058898925781,
+      "rewards/margins": 0.8935127258300781,
+      "rewards/rejected": -44.294105529785156,
       "step": 4800
     },
     {
       "epoch": 2.568991470145509,
-      "eval_logits/chosen": NaN,
-      "eval_logits/rejected": NaN,
-      "eval_logps/chosen": NaN,
-      "eval_logps/rejected": NaN,
-      "eval_loss": NaN,
-      "eval_rewards/accuracies": 0.0,
-      "eval_rewards/chosen": NaN,
-      "eval_rewards/margins": NaN,
-      "eval_rewards/rejected": NaN,
-      "eval_runtime": 34.7803,
-      "eval_samples_per_second": 38.671,
-      "eval_semantic_entropy": NaN,
-      "eval_steps_per_second": 9.689,
+      "eval_logits/chosen": 7.669034481048584,
+      "eval_logits/rejected": 7.379368305206299,
+      "eval_logps/chosen": -43.0767936706543,
+      "eval_logps/rejected": -44.17669677734375,
+      "eval_loss": 2.33503300250959e-07,
+      "eval_rewards/accuracies": 0.5675074458122253,
+      "eval_rewards/chosen": -43.0767936706543,
+      "eval_rewards/margins": 1.099900245666504,
+      "eval_rewards/rejected": -44.17669677734375,
+      "eval_runtime": 40.4965,
+      "eval_samples_per_second": 33.213,
+      "eval_steps_per_second": 8.322,
       "step": 4800
     },
     {
       "epoch": 2.571667502926911,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.3818118511475883e-05,
       "learning_rate": 6.066949564227897e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.336973190307617,
+      "logits/rejected": 8.907478332519531,
+      "logps/chosen": -42.392398834228516,
+      "logps/rejected": -43.604225158691406,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -42.392398834228516,
+      "rewards/margins": 1.2118293046951294,
+      "rewards/rejected": -43.604225158691406,
       "step": 4805
     },
     {
       "epoch": 2.574343535708312,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 7.631184559034695e-07,
       "learning_rate": 5.992805212285523e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.003918647766113,
+      "logits/rejected": 8.74857234954834,
+      "logps/chosen": -42.74648666381836,
+      "logps/rejected": -44.06636428833008,
+      "loss": 0.0,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -42.74648666381836,
+      "rewards/margins": 1.3198761940002441,
+      "rewards/rejected": -44.06636428833008,
       "step": 4810
     },
     {
       "epoch": 2.577019568489714,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 8.208916666334157e-09,
       "learning_rate": 5.9190878183399684e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.21426010131836,
+      "logits/rejected": 8.546037673950195,
+      "logps/chosen": -43.744224548339844,
+      "logps/rejected": -45.308162689208984,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -43.744224548339844,
+      "rewards/margins": 1.563937783241272,
+      "rewards/rejected": -45.308162689208984,
       "step": 4815
     },
     {
       "epoch": 2.5796956012711156,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 5.66179956353687e-07,
       "learning_rate": 5.845798097597748e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.841906547546387,
+      "logits/rejected": 8.718778610229492,
+      "logps/chosen": -43.10283660888672,
+      "logps/rejected": -44.05834197998047,
+      "loss": 0.0,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -43.10283660888672,
+      "rewards/margins": 0.9555104970932007,
+      "rewards/rejected": -44.05834197998047,
       "step": 4820
     },
     {
       "epoch": 2.5823716340525174,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.7991797546243745e-09,
       "learning_rate": 5.772936761116026e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.157719612121582,
+      "logits/rejected": 8.6575288772583,
+      "logps/chosen": -43.11156463623047,
+      "logps/rejected": -44.14491653442383,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -43.11156463623047,
+      "rewards/margins": 1.0333516597747803,
+      "rewards/rejected": -44.14491653442383,
       "step": 4825
     },
     {
       "epoch": 2.5850476668339186,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 8.051157589167438e-12,
       "learning_rate": 5.700504515795829e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.240645408630371,
+      "logits/rejected": 8.733817100524902,
+      "logps/chosen": -42.72593307495117,
+      "logps/rejected": -44.708248138427734,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -42.72593307495117,
+      "rewards/margins": 1.9823105335235596,
+      "rewards/rejected": -44.708248138427734,
       "step": 4830
     },
     {
       "epoch": 2.5877236996153203,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.1256045419031086e-12,
       "learning_rate": 5.628502064375101e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.201604843139648,
+      "logits/rejected": 8.701864242553711,
+      "logps/chosen": -43.109962463378906,
+      "logps/rejected": -45.54670333862305,
+      "loss": 0.0,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -43.109962463378906,
+      "rewards/margins": 2.436739683151245,
+      "rewards/rejected": -45.54670333862305,
       "step": 4835
     },
     {
       "epoch": 2.5903997323967216,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.305228283750965e-09,
       "learning_rate": 5.55693010542197e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.403355598449707,
+      "logits/rejected": 8.471258163452148,
+      "logps/chosen": -43.800025939941406,
+      "logps/rejected": -44.85037612915039,
+      "loss": 0.0,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -43.800025939941406,
+      "rewards/margins": 1.0503461360931396,
+      "rewards/rejected": -44.85037612915039,
       "step": 4840
     },
     {
       "epoch": 2.5930757651781233,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 5.82498911742165e-14,
       "learning_rate": 5.485789333327856e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.017709732055664,
+      "logits/rejected": 8.868873596191406,
+      "logps/chosen": -43.83453369140625,
+      "logps/rejected": -45.265838623046875,
+      "loss": 0.0,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -43.83453369140625,
+      "rewards/margins": 1.4313052892684937,
+      "rewards/rejected": -45.265838623046875,
       "step": 4845
     },
     {
       "epoch": 2.595751797959525,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.770058159357817e-09,
       "learning_rate": 5.4150804383008675e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.21560001373291,
+      "logits/rejected": 8.71939468383789,
+      "logps/chosen": -42.4356689453125,
+      "logps/rejected": -44.408084869384766,
+      "loss": 0.0,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -42.4356689453125,
+      "rewards/margins": 1.9724204540252686,
+      "rewards/rejected": -44.408084869384766,
       "step": 4850
     },
     {
       "epoch": 2.5984278307409268,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.6432633084823913e-06,
       "learning_rate": 5.344804106359002e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.005030632019043,
+      "logits/rejected": 8.43646240234375,
+      "logps/chosen": -43.049781799316406,
+      "logps/rejected": -44.541805267333984,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -43.049781799316406,
+      "rewards/margins": 1.492021083831787,
+      "rewards/rejected": -44.541805267333984,
       "step": 4855
     },
     {
       "epoch": 2.601103863522328,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 0.00014917573958102027,
       "learning_rate": 5.274961019323559e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.91691780090332,
+      "logits/rejected": 8.580900192260742,
+      "logps/chosen": -42.616825103759766,
+      "logps/rejected": -44.42259216308594,
+      "loss": 0.0,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -42.616825103759766,
+      "rewards/margins": 1.8057644367218018,
+      "rewards/rejected": -44.42259216308594,
       "step": 4860
     },
     {
       "epoch": 2.6037798963037297,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.087445973846156e-05,
       "learning_rate": 5.205551854812451e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.534845352172852,
+      "logits/rejected": 9.058837890625,
+      "logps/chosen": -42.86024856567383,
+      "logps/rejected": -43.009254455566406,
+      "loss": 0.0,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": -42.86024856567383,
+      "rewards/margins": 0.1490052044391632,
+      "rewards/rejected": -43.009254455566406,
       "step": 4865
     },
     {
       "epoch": 2.606455929085131,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.996685825953934e-11,
       "learning_rate": 5.1365772862337177e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.968287467956543,
+      "logits/rejected": 8.726919174194336,
+      "logps/chosen": -43.56601333618164,
+      "logps/rejected": -45.56330108642578,
+      "loss": 0.0,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -43.56601333618164,
+      "rewards/margins": 1.997291922569275,
+      "rewards/rejected": -45.56330108642578,
       "step": 4870
     },
     {
       "epoch": 2.6091319618665327,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.458635185533661e-05,
       "learning_rate": 5.068037982778905e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.992135047912598,
+      "logits/rejected": 8.74323844909668,
+      "logps/chosen": -43.59288024902344,
+      "logps/rejected": -44.756614685058594,
+      "loss": 0.0,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -43.59288024902344,
+      "rewards/margins": 1.163736343383789,
+      "rewards/rejected": -44.756614685058594,
       "step": 4875
     },
     {
       "epoch": 2.6118079946479344,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.509076287800465e-11,
       "learning_rate": 4.999934609416656e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.304932594299316,
+      "logits/rejected": 8.768881797790527,
+      "logps/chosen": -43.59864044189453,
+      "logps/rejected": -45.254886627197266,
+      "loss": 0.0,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -43.59864044189453,
+      "rewards/margins": 1.656252145767212,
+      "rewards/rejected": -45.254886627197266,
       "step": 4880
     },
     {
       "epoch": 2.614484027429336,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.284089324467653e-06,
       "learning_rate": 4.932267826886183e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.986671447753906,
+      "logits/rejected": 8.93264102935791,
+      "logps/chosen": -42.908203125,
+      "logps/rejected": -43.92797088623047,
+      "loss": 0.0,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -42.908203125,
+      "rewards/margins": 1.019768476486206,
+      "rewards/rejected": -43.92797088623047,
       "step": 4885
     },
     {
       "epoch": 2.6171600602107374,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 7.56573416378216e-12,
       "learning_rate": 4.8650382916909206e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.429896354675293,
+      "logits/rejected": 8.764261245727539,
+      "logps/chosen": -43.6498908996582,
+      "logps/rejected": -44.962730407714844,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -43.6498908996582,
+      "rewards/margins": 1.3128423690795898,
+      "rewards/rejected": -44.962730407714844,
       "step": 4890
     },
     {
       "epoch": 2.619836092992139,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 0.1268813660563405,
       "learning_rate": 4.7982466560920976e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.970413208007812,
+      "logits/rejected": 8.704388618469238,
+      "logps/chosen": -43.230384826660156,
+      "logps/rejected": -42.64691925048828,
+      "loss": 0.0009,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": -43.230384826660156,
+      "rewards/margins": -0.5834633111953735,
+      "rewards/rejected": -42.64691925048828,
       "step": 4895
     },
     {
       "epoch": 2.622512125773541,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.930897497825209e-05,
       "learning_rate": 4.7318935681024685e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.904264450073242,
+      "logits/rejected": 8.47325611114502,
+      "logps/chosen": -42.75983810424805,
+      "logps/rejected": -43.8365478515625,
+      "loss": 0.0,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -42.75983810424805,
+      "rewards/margins": 1.0767098665237427,
+      "rewards/rejected": -43.8365478515625,
       "step": 4900
     },
     {
       "epoch": 2.625188158554942,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.3387832181807664e-06,
       "learning_rate": 4.6659796714799745e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.630779266357422,
+      "logits/rejected": 8.169549942016602,
+      "logps/chosen": -42.07412338256836,
+      "logps/rejected": -43.108097076416016,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -42.07412338256836,
+      "rewards/margins": 1.0339674949645996,
+      "rewards/rejected": -43.108097076416016,
       "step": 4905
     },
     {
       "epoch": 2.627864191336344,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.1770521225739604e-09,
       "learning_rate": 4.60050560572155e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.008570671081543,
+      "logits/rejected": 9.138814926147461,
+      "logps/chosen": -44.2877082824707,
+      "logps/rejected": -44.840511322021484,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -44.2877082824707,
+      "rewards/margins": 0.552801251411438,
+      "rewards/rejected": -44.840511322021484,
       "step": 4910
     },
     {
       "epoch": 2.6305402241177456,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.6869112562877167e-10,
       "learning_rate": 4.535472006056834e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.11001968383789,
+      "logits/rejected": 8.620874404907227,
+      "logps/chosen": -43.35248565673828,
+      "logps/rejected": -44.16017150878906,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -43.35248565673828,
+      "rewards/margins": 0.8076847195625305,
+      "rewards/rejected": -44.16017150878906,
       "step": 4915
     },
     {
       "epoch": 2.6332162568991473,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 5.560416415014575e-13,
       "learning_rate": 4.470879503442132e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.919614791870117,
+      "logits/rejected": 8.703848838806152,
+      "logps/chosen": -43.5286865234375,
+      "logps/rejected": -44.891990661621094,
+      "loss": 0.0,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -43.5286865234375,
+      "rewards/margins": 1.363307237625122,
+      "rewards/rejected": -44.891990661621094,
       "step": 4920
     },
     {
       "epoch": 2.6358922896805486,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.5325250170831703e-13,
       "learning_rate": 4.406728724554154e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.218072891235352,
+      "logits/rejected": 8.42087173461914,
+      "logps/chosen": -42.3083381652832,
+      "logps/rejected": -44.92945098876953,
+      "loss": 0.0,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -42.3083381652832,
+      "rewards/margins": 2.621107578277588,
+      "rewards/rejected": -44.92945098876953,
       "step": 4925
     },
     {
       "epoch": 2.6385683224619503,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.8884796417525246e-05,
       "learning_rate": 4.3430202917840664e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.42890739440918,
+      "logits/rejected": 8.808436393737793,
+      "logps/chosen": -44.56859588623047,
+      "logps/rejected": -46.297515869140625,
+      "loss": 0.0,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -44.56859588623047,
+      "rewards/margins": 1.728915810585022,
+      "rewards/rejected": -46.297515869140625,
       "step": 4930
     },
     {
       "epoch": 2.6412443552433515,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.395523534731467e-07,
       "learning_rate": 4.279754823231346e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.074231147766113,
+      "logits/rejected": 8.443303108215332,
+      "logps/chosen": -41.81409454345703,
+      "logps/rejected": -43.53511428833008,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -41.81409454345703,
+      "rewards/margins": 1.7210206985473633,
+      "rewards/rejected": -43.53511428833008,
       "step": 4935
     },
     {
       "epoch": 2.6439203880247533,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.885682021598219e-12,
       "learning_rate": 4.216932932697859e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.230489730834961,
+      "logits/rejected": 8.875797271728516,
+      "logps/chosen": -43.60871124267578,
+      "logps/rejected": -45.03398895263672,
+      "loss": 0.0,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -43.60871124267578,
+      "rewards/margins": 1.4252796173095703,
+      "rewards/rejected": -45.03398895263672,
       "step": 4940
     },
     {
       "epoch": 2.646596420806155,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.7150421522036104e-08,
       "learning_rate": 4.154555229681844e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.929789543151855,
+      "logits/rejected": 8.41897201538086,
+      "logps/chosen": -43.34040451049805,
+      "logps/rejected": -44.39496612548828,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -43.34040451049805,
+      "rewards/margins": 1.0545607805252075,
+      "rewards/rejected": -44.39496612548828,
       "step": 4945
     },
     {
       "epoch": 2.6492724535875567,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.8184685541067486e-09,
       "learning_rate": 4.092622319372069e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.147625923156738,
+      "logits/rejected": 8.522071838378906,
+      "logps/chosen": -43.4959602355957,
+      "logps/rejected": -44.520790100097656,
+      "loss": 0.0,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": -43.4959602355957,
+      "rewards/margins": 1.024835467338562,
+      "rewards/rejected": -44.520790100097656,
       "step": 4950
     },
     {
       "epoch": 2.651948486368958,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 6.386522240649272e-08,
       "learning_rate": 4.031134802641889e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.138346672058105,
+      "logits/rejected": 9.06105899810791,
+      "logps/chosen": -43.88892364501953,
+      "logps/rejected": -43.99091720581055,
+      "loss": 0.0,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -43.88892364501953,
+      "rewards/margins": 0.1019887924194336,
+      "rewards/rejected": -43.99091720581055,
       "step": 4955
     },
     {
       "epoch": 2.6546245191503597,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.3663886471168935e-11,
       "learning_rate": 3.970093276043468e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.051881790161133,
+      "logits/rejected": 8.854409217834473,
+      "logps/chosen": -43.49562454223633,
+      "logps/rejected": -44.75468826293945,
+      "loss": 0.0,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -43.49562454223633,
+      "rewards/margins": 1.2590659856796265,
+      "rewards/rejected": -44.75468826293945,
       "step": 4960
     },
     {
       "epoch": 2.657300551931761,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.6079622630085887e-08,
       "learning_rate": 3.9094983318019584e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.301385879516602,
+      "logits/rejected": 8.678624153137207,
+      "logps/chosen": -43.20638656616211,
+      "logps/rejected": -44.19391632080078,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -43.20638656616211,
+      "rewards/margins": 0.9875224232673645,
+      "rewards/rejected": -44.19391632080078,
       "step": 4965
     },
     {
       "epoch": 2.6599765847131627,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.0649162510362826e-09,
       "learning_rate": 3.849350557809789e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.169889450073242,
+      "logits/rejected": 9.004755973815918,
+      "logps/chosen": -43.71628189086914,
+      "logps/rejected": -44.48175048828125,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -43.71628189086914,
+      "rewards/margins": 0.7654660940170288,
+      "rewards/rejected": -44.48175048828125,
       "step": 4970
     },
     {
       "epoch": 2.6626526174945644,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.2666338350319802e-12,
       "learning_rate": 3.789650537620903e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.083078384399414,
+      "logits/rejected": 8.865434646606445,
+      "logps/chosen": -44.71297836303711,
+      "logps/rejected": -45.582454681396484,
+      "loss": 0.0,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -44.71297836303711,
+      "rewards/margins": 0.8694734573364258,
+      "rewards/rejected": -45.582454681396484,
       "step": 4975
     },
     {
       "epoch": 2.665328650275966,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.971531394870502e-07,
       "learning_rate": 3.730398850445182e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.873515129089355,
+      "logits/rejected": 8.76361083984375,
+      "logps/chosen": -43.24320602416992,
+      "logps/rejected": -43.78880310058594,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -43.24320602416992,
+      "rewards/margins": 0.545599639415741,
+      "rewards/rejected": -43.78880310058594,
       "step": 4980
     },
     {
       "epoch": 2.6680046830573674,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.5357519756630864e-11,
       "learning_rate": 3.671596071142735e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.174957275390625,
+      "logits/rejected": 8.634170532226562,
+      "logps/chosen": -43.926204681396484,
+      "logps/rejected": -45.98197937011719,
+      "loss": 0.0,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -43.926204681396484,
+      "rewards/margins": 2.055774688720703,
+      "rewards/rejected": -45.98197937011719,
       "step": 4985
     },
     {
       "epoch": 2.670680715838769,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 7.355422836845194e-09,
       "learning_rate": 3.6132427702183996e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.440366744995117,
+      "logits/rejected": 8.885251998901367,
+      "logps/chosen": -43.57441711425781,
+      "logps/rejected": -44.79595184326172,
+      "loss": 0.0,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -43.57441711425781,
+      "rewards/margins": 1.2215327024459839,
+      "rewards/rejected": -44.79595184326172,
       "step": 4990
     },
     {
       "epoch": 2.6733567486201704,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.0379499255378706e-12,
       "learning_rate": 3.555339513816147e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.349838256835938,
+      "logits/rejected": 9.28815746307373,
+      "logps/chosen": -45.09845733642578,
+      "logps/rejected": -44.304115295410156,
+      "loss": 0.0,
+      "rewards/accuracies": 0.44999998807907104,
+      "rewards/chosen": -45.09845733642578,
+      "rewards/margins": -0.7943447828292847,
+      "rewards/rejected": -44.304115295410156,
       "step": 4995
     },
     {
       "epoch": 2.676032781401572,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.2105664935416367e-08,
       "learning_rate": 3.497886863713639e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.391852378845215,
+      "logits/rejected": 9.19373893737793,
+      "logps/chosen": -45.093528747558594,
+      "logps/rejected": -44.81812286376953,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -45.093528747558594,
+      "rewards/margins": -0.27539995312690735,
+      "rewards/rejected": -44.81812286376953,
       "step": 5000
     },
     {
       "epoch": 2.678708814182974,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.991499452668781e-09,
       "learning_rate": 3.440885377316721e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.918176651000977,
+      "logits/rejected": 8.692453384399414,
+      "logps/chosen": -43.148006439208984,
+      "logps/rejected": -43.84690475463867,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -43.148006439208984,
+      "rewards/margins": 0.6988977789878845,
+      "rewards/rejected": -43.84690475463867,
       "step": 5005
     },
     {
       "epoch": 2.6813848469643755,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 8.902621115785179e-10,
       "learning_rate": 3.384335607654082e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.754938125610352,
+      "logits/rejected": 8.3749418258667,
+      "logps/chosen": -43.56318283081055,
+      "logps/rejected": -44.35714340209961,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -43.56318283081055,
+      "rewards/margins": 0.7939629554748535,
+      "rewards/rejected": -44.35714340209961,
       "step": 5010
     },
     {
       "epoch": 2.684060879745777,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 0.00010227740881515844,
       "learning_rate": 3.328238103371811e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.126741409301758,
+      "logits/rejected": 8.923644065856934,
+      "logps/chosen": -44.1834831237793,
+      "logps/rejected": -44.94786834716797,
+      "loss": 0.0,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -44.1834831237793,
+      "rewards/margins": 0.7643880248069763,
+      "rewards/rejected": -44.94786834716797,
       "step": 5015
     },
     {
       "epoch": 2.6867369125271785,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.3713092844715224e-12,
       "learning_rate": 3.272593408728169e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.143709182739258,
+      "logits/rejected": 8.276471138000488,
+      "logps/chosen": -43.83470916748047,
+      "logps/rejected": -45.55449676513672,
+      "loss": 0.0,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -43.83470916748047,
+      "rewards/margins": 1.719788908958435,
+      "rewards/rejected": -45.55449676513672,
       "step": 5020
     },
     {
       "epoch": 2.6894129453085798,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.689941290693259e-11,
       "learning_rate": 3.217402063588204e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.257261276245117,
+      "logits/rejected": 8.785032272338867,
+      "logps/chosen": -43.62569808959961,
+      "logps/rejected": -44.92503356933594,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -43.62569808959961,
+      "rewards/margins": 1.2993357181549072,
+      "rewards/rejected": -44.92503356933594,
       "step": 5025
     },
     {
       "epoch": 2.6920889780899815,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.713641675516236e-10,
       "learning_rate": 3.162664603418608e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.411273956298828,
+      "logits/rejected": 9.02724552154541,
+      "logps/chosen": -44.0105094909668,
+      "logps/rejected": -45.17673110961914,
+      "loss": 0.0,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -44.0105094909668,
+      "rewards/margins": 1.1662204265594482,
+      "rewards/rejected": -45.17673110961914,
       "step": 5030
     },
     {
       "epoch": 2.694765010871383,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 5.56789214960003e-10,
       "learning_rate": 3.1083815592824416e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.001391410827637,
+      "logits/rejected": 8.826278686523438,
+      "logps/chosen": -43.243263244628906,
+      "logps/rejected": -44.79766082763672,
+      "loss": 0.0,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -43.243263244628906,
+      "rewards/margins": 1.5543973445892334,
+      "rewards/rejected": -44.79766082763672,
       "step": 5035
     },
     {
       "epoch": 2.697441043652785,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.1635836104737904e-08,
       "learning_rate": 3.054553457834053e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.783475875854492,
+      "logits/rejected": 8.934747695922852,
+      "logps/chosen": -42.744544982910156,
+      "logps/rejected": -43.472389221191406,
+      "loss": 0.0,
+      "rewards/accuracies": 0.4625000059604645,
+      "rewards/chosen": -42.744544982910156,
+      "rewards/margins": 0.7278450727462769,
+      "rewards/rejected": -43.472389221191406,
       "step": 5040
     },
     {
       "epoch": 2.700117076434186,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.2921655467812956e-07,
       "learning_rate": 3.0011808213139036e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.854515075683594,
+      "logits/rejected": 9.011656761169434,
+      "logps/chosen": -43.90321350097656,
+      "logps/rejected": -44.64320373535156,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -43.90321350097656,
+      "rewards/margins": 0.7399893999099731,
+      "rewards/rejected": -44.64320373535156,
       "step": 5045
     },
     {
       "epoch": 2.702793109215588,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.7148536393523832e-08,
       "learning_rate": 2.948264167543568e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.019979476928711,
+      "logits/rejected": 8.739091873168945,
+      "logps/chosen": -44.38550567626953,
+      "logps/rejected": -44.7628059387207,
+      "loss": 0.0,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": -44.38550567626953,
+      "rewards/margins": 0.377299964427948,
+      "rewards/rejected": -44.7628059387207,
       "step": 5050
     },
     {
       "epoch": 2.7054691419969896,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.888795611051064e-10,
       "learning_rate": 2.8958040099206216e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.428553581237793,
+      "logits/rejected": 9.128547668457031,
+      "logps/chosen": -44.93156051635742,
+      "logps/rejected": -45.50389099121094,
+      "loss": 0.0,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -44.93156051635742,
+      "rewards/margins": 0.5723268389701843,
+      "rewards/rejected": -45.50389099121094,
       "step": 5055
     },
     {
       "epoch": 2.708145174778391,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 5.1054413534218807e-11,
       "learning_rate": 2.843800857413775e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.756775856018066,
+      "logits/rejected": 8.636309623718262,
+      "logps/chosen": -43.08277130126953,
+      "logps/rejected": -43.82593536376953,
+      "loss": 0.0,
+      "rewards/accuracies": 0.4625000059604645,
+      "rewards/chosen": -43.08277130126953,
+      "rewards/margins": 0.7431662678718567,
+      "rewards/rejected": -43.82593536376953,
       "step": 5060
     },
     {
       "epoch": 2.7108212075597926,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.5235207796454951e-10,
       "learning_rate": 2.7922552145578203e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.130325317382812,
+      "logits/rejected": 8.246222496032715,
+      "logps/chosen": -43.436309814453125,
+      "logps/rejected": -44.406898498535156,
+      "loss": 0.0,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -43.436309814453125,
+      "rewards/margins": 0.9705861210823059,
+      "rewards/rejected": -44.406898498535156,
       "step": 5065
     },
     {
       "epoch": 2.7134972403411943,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.407476836458239e-10,
       "learning_rate": 2.7411675814488277e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.94955062866211,
+      "logits/rejected": 8.54890251159668,
+      "logps/chosen": -43.280242919921875,
+      "logps/rejected": -43.54806900024414,
+      "loss": 0.0,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -43.280242919921875,
+      "rewards/margins": 0.26782605051994324,
+      "rewards/rejected": -43.54806900024414,
       "step": 5070
     },
     {
       "epoch": 2.7161732731225956,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.947587094236316e-06,
       "learning_rate": 2.690538453739216e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.172134399414062,
+      "logits/rejected": 8.903919219970703,
+      "logps/chosen": -42.591148376464844,
+      "logps/rejected": -43.65777587890625,
+      "loss": 0.0,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": -42.591148376464844,
+      "rewards/margins": 1.066626787185669,
+      "rewards/rejected": -43.65777587890625,
       "step": 5075
     },
     {
       "epoch": 2.7188493059039973,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.6267342835275215e-09,
       "learning_rate": 2.6403683226330298e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.345294952392578,
+      "logits/rejected": 8.793591499328613,
+      "logps/chosen": -43.54670715332031,
+      "logps/rejected": -45.21221160888672,
+      "loss": 0.0,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -43.54670715332031,
+      "rewards/margins": 1.6655025482177734,
+      "rewards/rejected": -45.21221160888672,
       "step": 5080
     },
     {
       "epoch": 2.721525338685399,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 7.970079157218425e-11,
       "learning_rate": 2.5906576748810804e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.353070259094238,
+      "logits/rejected": 8.84656047821045,
+      "logps/chosen": -43.939056396484375,
+      "logps/rejected": -45.37128829956055,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -43.939056396484375,
+      "rewards/margins": 1.4322341680526733,
+      "rewards/rejected": -45.37128829956055,
       "step": 5085
     },
     {
       "epoch": 2.7242013714668003,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.1930341711764064e-05,
       "learning_rate": 2.5414069927763016e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.542497634887695,
+      "logits/rejected": 8.837213516235352,
+      "logps/chosen": -42.942108154296875,
+      "logps/rejected": -43.854305267333984,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -42.942108154296875,
+      "rewards/margins": 0.9121963381767273,
+      "rewards/rejected": -43.854305267333984,
       "step": 5090
     },
     {
       "epoch": 2.726877404248202,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.0450888032888466e-09,
       "learning_rate": 2.4926167541490185e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.348544120788574,
+      "logits/rejected": 8.513127326965332,
+      "logps/chosen": -42.83620071411133,
+      "logps/rejected": -44.66901397705078,
+      "loss": 0.0,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -42.83620071411133,
+      "rewards/margins": 1.8328173160552979,
+      "rewards/rejected": -44.66901397705078,
       "step": 5095
     },
     {
       "epoch": 2.7295534370296037,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 6.433083452140862e-10,
       "learning_rate": 2.4442874323623574e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.062211036682129,
+      "logits/rejected": 8.49016284942627,
+      "logps/chosen": -42.86086654663086,
+      "logps/rejected": -44.317771911621094,
+      "loss": 0.0,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -42.86086654663086,
+      "rewards/margins": 1.456904649734497,
+      "rewards/rejected": -44.317771911621094,
       "step": 5100
     },
     {
       "epoch": 2.7322294698110055,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.2046151819761444e-08,
       "learning_rate": 2.396419496307589e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.877019882202148,
+      "logits/rejected": 8.395615577697754,
+      "logps/chosen": -42.771446228027344,
+      "logps/rejected": -44.42991256713867,
+      "loss": 0.0,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -42.771446228027344,
+      "rewards/margins": 1.6584646701812744,
+      "rewards/rejected": -44.42991256713867,
       "step": 5105
     },
     {
       "epoch": 2.7349055025924067,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.4199026468186926e-13,
       "learning_rate": 2.349013410399653e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.8574857711792,
+      "logits/rejected": 8.671666145324707,
+      "logps/chosen": -43.167930603027344,
+      "logps/rejected": -44.862247467041016,
+      "loss": 0.0,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -43.167930603027344,
+      "rewards/margins": 1.694319486618042,
+      "rewards/rejected": -44.862247467041016,
       "step": 5110
     },
     {
       "epoch": 2.7375815353738084,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.279877509417109e-11,
       "learning_rate": 2.3020696345725954e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.263761520385742,
+      "logits/rejected": 8.411904335021973,
+      "logps/chosen": -42.049312591552734,
+      "logps/rejected": -43.55626678466797,
+      "loss": 0.0,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -42.049312591552734,
+      "rewards/margins": 1.506955862045288,
+      "rewards/rejected": -43.55626678466797,
       "step": 5115
     },
     {
       "epoch": 2.7402575681552097,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.846755253120891e-06,
       "learning_rate": 2.2555886242751398e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.307716369628906,
+      "logits/rejected": 9.046968460083008,
+      "logps/chosen": -42.930198669433594,
+      "logps/rejected": -43.85496520996094,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -42.930198669433594,
+      "rewards/margins": 0.924761176109314,
+      "rewards/rejected": -43.85496520996094,
       "step": 5120
     },
     {
       "epoch": 2.7429336009366114,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.00782934623768e-09,
       "learning_rate": 2.2095708304662453e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.613089561462402,
+      "logits/rejected": 8.96796703338623,
+      "logps/chosen": -44.13003921508789,
+      "logps/rejected": -45.09586715698242,
+      "loss": 0.0,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -44.13003921508789,
+      "rewards/margins": 0.9658284187316895,
+      "rewards/rejected": -45.09586715698242,
       "step": 5125
     },
     {
       "epoch": 2.745609633718013,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.345035529209351e-07,
       "learning_rate": 2.16401669961076e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.60748291015625,
+      "logits/rejected": 8.8858003616333,
+      "logps/chosen": -42.96638488769531,
+      "logps/rejected": -44.450279235839844,
+      "loss": 0.0,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -42.96638488769531,
+      "rewards/margins": 1.4838933944702148,
+      "rewards/rejected": -44.450279235839844,
       "step": 5130
     },
     {
       "epoch": 2.748285666499415,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.0046950769199335e-09,
       "learning_rate": 2.1189266736750532e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.073997497558594,
+      "logits/rejected": 8.833829879760742,
+      "logps/chosen": -44.25225830078125,
+      "logps/rejected": -44.91228485107422,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -44.25225830078125,
+      "rewards/margins": 0.6600229144096375,
+      "rewards/rejected": -44.91228485107422,
       "step": 5135
     },
     {
       "epoch": 2.750961699280816,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.025038714399654e-06,
       "learning_rate": 2.0743011901227623e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.187370300292969,
+      "logits/rejected": 8.696545600891113,
+      "logps/chosen": -41.95415496826172,
+      "logps/rejected": -42.715797424316406,
+      "loss": 0.0,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -41.95415496826172,
+      "rewards/margins": 0.7616451978683472,
+      "rewards/rejected": -42.715797424316406,
       "step": 5140
     },
     {
       "epoch": 2.753637732062218,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.6195640357372744e-06,
       "learning_rate": 2.030140681910508e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.051790237426758,
+      "logits/rejected": 8.603446960449219,
+      "logps/chosen": -43.37526321411133,
+      "logps/rejected": -44.78037643432617,
+      "loss": 0.0,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -43.37526321411133,
+      "rewards/margins": 1.4051129817962646,
+      "rewards/rejected": -44.78037643432617,
       "step": 5145
     },
     {
       "epoch": 2.756313764843619,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.105252712389415e-10,
       "learning_rate": 1.986445577483753e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.452394485473633,
+      "logits/rejected": 8.909395217895508,
+      "logps/chosen": -43.29167938232422,
+      "logps/rejected": -43.958072662353516,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -43.29167938232422,
+      "rewards/margins": 0.666393518447876,
+      "rewards/rejected": -43.958072662353516,
       "step": 5150
     },
     {
       "epoch": 2.758989797625021,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.97753562415548e-12,
       "learning_rate": 1.9432163007725765e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.33098316192627,
+      "logits/rejected": 9.138799667358398,
+      "logps/chosen": -43.150264739990234,
+      "logps/rejected": -44.80061721801758,
+      "loss": 0.0,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -43.150264739990234,
+      "rewards/margins": 1.6503534317016602,
+      "rewards/rejected": -44.80061721801758,
       "step": 5155
     },
     {
       "epoch": 2.7616658304064226,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.292591427401099e-06,
       "learning_rate": 1.9004532711876297e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.967911720275879,
+      "logits/rejected": 9.11368465423584,
+      "logps/chosen": -42.864418029785156,
+      "logps/rejected": -43.31922149658203,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -42.864418029785156,
+      "rewards/margins": 0.45479750633239746,
+      "rewards/rejected": -43.31922149658203,
       "step": 5160
     },
     {
       "epoch": 2.7643418631878243,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 5.76518072920128e-07,
       "learning_rate": 1.8581569036159928e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.069539070129395,
+      "logits/rejected": 8.464866638183594,
+      "logps/chosen": -42.33643341064453,
+      "logps/rejected": -44.70124816894531,
+      "loss": 0.0,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -42.33643341064453,
+      "rewards/margins": 2.3648159503936768,
+      "rewards/rejected": -44.70124816894531,
       "step": 5165
     },
     {
       "epoch": 2.7670178959692255,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.6939526449981929e-07,
       "learning_rate": 1.8163276084172285e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.361352920532227,
+      "logits/rejected": 9.145605087280273,
+      "logps/chosen": -42.27976989746094,
+      "logps/rejected": -43.554710388183594,
+      "loss": 0.0,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -42.27976989746094,
+      "rewards/margins": 1.2749345302581787,
+      "rewards/rejected": -43.554710388183594,
       "step": 5170
     },
     {
       "epoch": 2.7696939287506273,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.3088187296791866e-06,
       "learning_rate": 1.7749657914193194e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.967318534851074,
+      "logits/rejected": 8.859064102172852,
+      "logps/chosen": -43.4511833190918,
+      "logps/rejected": -44.335533142089844,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -43.4511833190918,
+      "rewards/margins": 0.8843482732772827,
+      "rewards/rejected": -44.335533142089844,
       "step": 5175
     },
     {
       "epoch": 2.7723699615320285,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.2041138659599504e-09,
       "learning_rate": 1.7340718539148203e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.051706314086914,
+      "logits/rejected": 8.825541496276855,
+      "logps/chosen": -43.35496139526367,
+      "logps/rejected": -44.44926834106445,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -43.35496139526367,
+      "rewards/margins": 1.094308614730835,
+      "rewards/rejected": -44.44926834106445,
       "step": 5180
     },
     {
       "epoch": 2.7750459943134302,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 5.282325397021928e-11,
       "learning_rate": 1.6936461926568724e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.262999534606934,
+      "logits/rejected": 8.572446823120117,
+      "logps/chosen": -43.500022888183594,
+      "logps/rejected": -45.655517578125,
+      "loss": 0.0012,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -43.500022888183594,
+      "rewards/margins": 2.1554973125457764,
+      "rewards/rejected": -45.655517578125,
       "step": 5185
     },
     {
       "epoch": 2.777722027094832,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 5.0751135263184296e-11,
       "learning_rate": 1.6536891998554346e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.05966854095459,
+      "logits/rejected": 8.62863540649414,
+      "logps/chosen": -42.68769836425781,
+      "logps/rejected": -44.32209777832031,
+      "loss": 0.0001,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -42.68769836425781,
+      "rewards/margins": 1.6343997716903687,
+      "rewards/rejected": -44.32209777832031,
       "step": 5190
     },
     {
       "epoch": 2.7803980598762337,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.265507782643499e-07,
       "learning_rate": 1.6142012631734093e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.037896156311035,
+      "logits/rejected": 8.6544189453125,
+      "logps/chosen": -43.891319274902344,
+      "logps/rejected": -44.3411750793457,
+      "loss": 0.0,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -43.891319274902344,
+      "rewards/margins": 0.4498554766178131,
+      "rewards/rejected": -44.3411750793457,
       "step": 5195
     },
     {
       "epoch": 2.783074092657635,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.1932820135416846e-11,
       "learning_rate": 1.575182765722949e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.161626815795898,
+      "logits/rejected": 8.609975814819336,
+      "logps/chosen": -43.45505905151367,
+      "logps/rejected": -45.027748107910156,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -43.45505905151367,
+      "rewards/margins": 1.5726863145828247,
+      "rewards/rejected": -45.027748107910156,
       "step": 5200
     },
     {
       "epoch": 2.783074092657635,
-      "eval_logits/chosen": NaN,
-      "eval_logits/rejected": NaN,
-      "eval_logps/chosen": NaN,
-      "eval_logps/rejected": NaN,
-      "eval_loss": NaN,
-      "eval_rewards/accuracies": 0.0,
-      "eval_rewards/chosen": NaN,
-      "eval_rewards/margins": NaN,
-      "eval_rewards/rejected": NaN,
-      "eval_runtime": 34.8842,
-      "eval_samples_per_second": 38.556,
-      "eval_semantic_entropy": NaN,
-      "eval_steps_per_second": 9.661,
+      "eval_logits/chosen": 7.59325647354126,
+      "eval_logits/rejected": 7.296000003814697,
+      "eval_logps/chosen": -43.122711181640625,
+      "eval_logps/rejected": -44.229122161865234,
+      "eval_loss": 2.156311182943682e-07,
+      "eval_rewards/accuracies": 0.5689911246299744,
+      "eval_rewards/chosen": -43.122711181640625,
+      "eval_rewards/margins": 1.106404423713684,
+      "eval_rewards/rejected": -44.229122161865234,
+      "eval_runtime": 40.4053,
+      "eval_samples_per_second": 33.288,
+      "eval_steps_per_second": 8.34,
       "step": 5200
     },
     {
       "epoch": 2.7857501254390367,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.3313276506519999e-07,
       "learning_rate": 1.536634086061672e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.959929466247559,
+      "logits/rejected": 8.747356414794922,
+      "logps/chosen": -42.357261657714844,
+      "logps/rejected": -42.94976043701172,
+      "loss": 0.0,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -42.357261657714844,
+      "rewards/margins": 0.5924930572509766,
+      "rewards/rejected": -42.94976043701172,
       "step": 5205
     },
     {
       "epoch": 2.788426158220438,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.441210445552706e-08,
       "learning_rate": 1.4985555981890495e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.194665908813477,
+      "logits/rejected": 8.970166206359863,
+      "logps/chosen": -43.786033630371094,
+      "logps/rejected": -45.51081466674805,
+      "loss": 0.0,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -43.786033630371094,
+      "rewards/margins": 1.7247825860977173,
+      "rewards/rejected": -45.51081466674805,
       "step": 5210
     },
     {
       "epoch": 2.7911021910018396,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.0256292204523195e-11,
       "learning_rate": 1.4609476715427226e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.315170288085938,
+      "logits/rejected": 8.941254615783691,
+      "logps/chosen": -42.23725509643555,
+      "logps/rejected": -43.261512756347656,
+      "loss": 0.0,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -42.23725509643555,
+      "rewards/margins": 1.0242549180984497,
+      "rewards/rejected": -43.261512756347656,
       "step": 5215
     },
     {
       "epoch": 2.7937782237832414,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.557174067438597e-07,
       "learning_rate": 1.4238106709949792e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.222701072692871,
+      "logits/rejected": 8.876856803894043,
+      "logps/chosen": -43.900089263916016,
+      "logps/rejected": -43.921295166015625,
+      "loss": 0.0,
+      "rewards/accuracies": 0.4937500059604645,
+      "rewards/chosen": -43.900089263916016,
+      "rewards/margins": 0.021208714693784714,
+      "rewards/rejected": -43.921295166015625,
       "step": 5220
     },
     {
       "epoch": 2.796454256564643,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.1313447034546127e-08,
       "learning_rate": 1.3871449568491511e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.978532791137695,
+      "logits/rejected": 8.517302513122559,
+      "logps/chosen": -44.594696044921875,
+      "logps/rejected": -45.526573181152344,
+      "loss": 0.0,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -44.594696044921875,
+      "rewards/margins": 0.9318796396255493,
+      "rewards/rejected": -45.526573181152344,
       "step": 5225
     },
     {
       "epoch": 2.7991302893460444,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.160158799261448e-08,
       "learning_rate": 1.3509508848361606e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.403702735900879,
+      "logits/rejected": 8.873385429382324,
+      "logps/chosen": -43.27454376220703,
+      "logps/rejected": -44.70935821533203,
+      "loss": 0.0,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -43.27454376220703,
+      "rewards/margins": 1.434807538986206,
+      "rewards/rejected": -44.70935821533203,
       "step": 5230
     },
     {
       "epoch": 2.801806322127446,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 7.820727053378319e-06,
       "learning_rate": 1.3152288061110517e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.201998710632324,
+      "logits/rejected": 8.765900611877441,
+      "logps/chosen": -42.991729736328125,
+      "logps/rejected": -44.02509689331055,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -42.991729736328125,
+      "rewards/margins": 1.033368706703186,
+      "rewards/rejected": -44.02509689331055,
       "step": 5235
     },
     {
       "epoch": 2.804482354908848,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.7902472447983859e-10,
       "learning_rate": 1.2799790672495814e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.463716506958008,
+      "logits/rejected": 8.737053871154785,
+      "logps/chosen": -43.7899055480957,
+      "logps/rejected": -45.37424850463867,
+      "loss": 0.0,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -43.7899055480957,
+      "rewards/margins": 1.5843477249145508,
+      "rewards/rejected": -45.37424850463867,
       "step": 5240
     },
     {
       "epoch": 2.807158387690249,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.8736349170294136e-12,
       "learning_rate": 1.2452020102448835e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.046751022338867,
+      "logits/rejected": 8.856473922729492,
+      "logps/chosen": -42.971893310546875,
+      "logps/rejected": -43.707008361816406,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -42.971893310546875,
+      "rewards/margins": 0.7351185083389282,
+      "rewards/rejected": -43.707008361816406,
       "step": 5245
     },
     {
       "epoch": 2.8098344204716508,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.8362881789696736e-10,
       "learning_rate": 1.2108979725041103e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.25361442565918,
+      "logits/rejected": 8.988142967224121,
+      "logps/chosen": -42.874149322509766,
+      "logps/rejected": -43.95782470703125,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -42.874149322509766,
+      "rewards/margins": 1.0836725234985352,
+      "rewards/rejected": -43.95782470703125,
       "step": 5250
     },
     {
       "epoch": 2.8125104532530525,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 6.89174656477691e-08,
       "learning_rate": 1.1770672868451958e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.159260749816895,
+      "logits/rejected": 8.536584854125977,
+      "logps/chosen": -42.166114807128906,
+      "logps/rejected": -43.92035675048828,
+      "loss": 0.0,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -42.166114807128906,
+      "rewards/margins": 1.7542474269866943,
+      "rewards/rejected": -43.92035675048828,
       "step": 5255
     },
     {
       "epoch": 2.8151864860344538,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 9.116662477275866e-08,
       "learning_rate": 1.1437102814935872e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.747381210327148,
+      "logits/rejected": 8.502212524414062,
+      "logps/chosen": -43.68171691894531,
+      "logps/rejected": -43.559906005859375,
+      "loss": 0.0002,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -43.68171691894531,
+      "rewards/margins": -0.1218104362487793,
+      "rewards/rejected": -43.559906005859375,
       "step": 5260
     },
     {
       "epoch": 2.8178625188158555,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 6.907126013085905e-08,
       "learning_rate": 1.1108272800791018e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.698392868041992,
+      "logits/rejected": 8.744329452514648,
+      "logps/chosen": -42.207672119140625,
+      "logps/rejected": -44.34783935546875,
+      "loss": 0.0,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -42.207672119140625,
+      "rewards/margins": 2.140167474746704,
+      "rewards/rejected": -44.34783935546875,
       "step": 5265
     },
     {
       "epoch": 2.820538551597257,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 6.983112331400282e-09,
       "learning_rate": 1.078418601632769e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.046021461486816,
+      "logits/rejected": 8.639293670654297,
+      "logps/chosen": -42.759029388427734,
+      "logps/rejected": -43.713539123535156,
+      "loss": 0.0,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -42.759029388427734,
+      "rewards/margins": 0.9545109868049622,
+      "rewards/rejected": -43.713539123535156,
       "step": 5270
     },
     {
       "epoch": 2.8232145843786585,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.4433310476687422e-10,
       "learning_rate": 1.0464845605837159e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.016949653625488,
+      "logits/rejected": 8.667830467224121,
+      "logps/chosen": -43.385009765625,
+      "logps/rejected": -44.63142776489258,
+      "loss": 0.0,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -43.385009765625,
+      "rewards/margins": 1.246415376663208,
+      "rewards/rejected": -44.63142776489258,
       "step": 5275
     },
     {
       "epoch": 2.82589061716006,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 0.00019095416438492458,
       "learning_rate": 1.0150254667561642e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.099616050720215,
+      "logits/rejected": 8.377941131591797,
+      "logps/chosen": -43.24871826171875,
+      "logps/rejected": -44.87980651855469,
+      "loss": 0.0023,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -43.24871826171875,
+      "rewards/margins": 1.6310901641845703,
+      "rewards/rejected": -44.87980651855469,
       "step": 5280
     },
     {
       "epoch": 2.828566649941462,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.527120312796905e-09,
       "learning_rate": 9.840416253663719e-09,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.420157432556152,
+      "logits/rejected": 9.100214004516602,
+      "logps/chosen": -43.914451599121094,
+      "logps/rejected": -45.43206787109375,
+      "loss": 0.0,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -43.914451599121094,
+      "rewards/margins": 1.5176138877868652,
+      "rewards/rejected": -45.43206787109375,
       "step": 5285
     },
     {
       "epoch": 2.8312426827228636,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.5039508246119844e-13,
       "learning_rate": 9.535333370197074e-09,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.187828063964844,
+      "logits/rejected": 8.611418724060059,
+      "logps/chosen": -43.67717361450195,
+      "logps/rejected": -44.80834197998047,
+      "loss": 0.0,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -43.67717361450195,
+      "rewards/margins": 1.1311668157577515,
+      "rewards/rejected": -44.80834197998047,
       "step": 5290
     },
     {
       "epoch": 2.833918715504265,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 5.882714809599216e-07,
       "learning_rate": 9.23500897707713e-09,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.386972427368164,
+      "logits/rejected": 8.771082878112793,
+      "logps/chosen": -43.3702507019043,
+      "logps/rejected": -44.68819046020508,
+      "loss": 0.0,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -43.3702507019043,
+      "rewards/margins": 1.3179371356964111,
+      "rewards/rejected": -44.68819046020508,
       "step": 5295
     },
     {
       "epoch": 2.8365947482856666,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 6.470387932983341e-11,
       "learning_rate": 8.939445988052574e-09,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.07305908203125,
+      "logits/rejected": 8.82376766204834,
+      "logps/chosen": -43.485137939453125,
+      "logps/rejected": -43.979042053222656,
+      "loss": 0.0,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": -43.485137939453125,
+      "rewards/margins": 0.4939079284667969,
+      "rewards/rejected": -43.979042053222656,
       "step": 5300
     },
     {
       "epoch": 2.839270781067068,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.6241058592339942e-09,
       "learning_rate": 8.648647270676656e-09,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.844246864318848,
+      "logits/rejected": 8.372082710266113,
+      "logps/chosen": -42.29883575439453,
+      "logps/rejected": -43.836509704589844,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -42.29883575439453,
+      "rewards/margins": 1.537675380706787,
+      "rewards/rejected": -43.836509704589844,
       "step": 5305
     },
     {
       "epoch": 2.8419468138484696,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.849104408977363e-14,
       "learning_rate": 8.362615646279991e-09,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.875150680541992,
+      "logits/rejected": 8.860733032226562,
+      "logps/chosen": -44.44951248168945,
+      "logps/rejected": -45.70476150512695,
+      "loss": 0.0,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -44.44951248168945,
+      "rewards/margins": 1.255252480506897,
+      "rewards/rejected": -45.70476150512695,
       "step": 5310
     },
     {
       "epoch": 2.8446228466298713,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.9994326140079131e-10,
       "learning_rate": 8.081353889942466e-09,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.169601440429688,
+      "logits/rejected": 8.663008689880371,
+      "logps/chosen": -43.47755432128906,
+      "logps/rejected": -45.06774139404297,
+      "loss": 0.0,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -43.47755432128906,
+      "rewards/margins": 1.5901862382888794,
+      "rewards/rejected": -45.06774139404297,
       "step": 5315
     },
     {
       "epoch": 2.847298879411273,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.2209391931202481e-09,
       "learning_rate": 7.804864730467042e-09,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.036087989807129,
+      "logits/rejected": 8.97172737121582,
+      "logps/chosen": -43.215675354003906,
+      "logps/rejected": -44.50541687011719,
+      "loss": 0.0,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -43.215675354003906,
+      "rewards/margins": 1.2897390127182007,
+      "rewards/rejected": -44.50541687011719,
       "step": 5320
     },
     {
       "epoch": 2.8499749121926743,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.1100234859355303e-09,
       "learning_rate": 7.533150850352665e-09,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.126693725585938,
+      "logits/rejected": 8.463194847106934,
+      "logps/chosen": -44.90612030029297,
+      "logps/rejected": -44.989322662353516,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -44.90612030029297,
+      "rewards/margins": 0.08319838345050812,
+      "rewards/rejected": -44.989322662353516,
       "step": 5325
     },
     {
       "epoch": 2.852650944974076,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 5.4843370305422565e-09,
       "learning_rate": 7.2662148857686175e-09,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.544363021850586,
+      "logits/rejected": 8.329607009887695,
+      "logps/chosen": -42.94819259643555,
+      "logps/rejected": -43.7733154296875,
+      "loss": 0.0004,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -42.94819259643555,
+      "rewards/margins": 0.8251211047172546,
+      "rewards/rejected": -43.7733154296875,
       "step": 5330
     },
     {
       "epoch": 2.8553269777554773,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.143171486734352e-09,
       "learning_rate": 7.0040594265287635e-09,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.722869873046875,
+      "logits/rejected": 8.899015426635742,
+      "logps/chosen": -43.56779098510742,
+      "logps/rejected": -44.39909744262695,
+      "loss": 0.0,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -43.56779098510742,
+      "rewards/margins": 0.8313077688217163,
+      "rewards/rejected": -44.39909744262695,
       "step": 5335
     },
     {
       "epoch": 2.858003010536879,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 5.4126539382297734e-12,
       "learning_rate": 6.746687016066566e-09,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.215690612792969,
+      "logits/rejected": 9.012933731079102,
+      "logps/chosen": -43.967124938964844,
+      "logps/rejected": -44.7640495300293,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -43.967124938964844,
+      "rewards/margins": 0.7969287037849426,
+      "rewards/rejected": -44.7640495300293,
       "step": 5340
     },
     {
       "epoch": 2.8606790433182807,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.0171548077762303e-08,
       "learning_rate": 6.494100151410276e-09,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.249914169311523,
+      "logits/rejected": 8.525673866271973,
+      "logps/chosen": -42.68964385986328,
+      "logps/rejected": -44.34496307373047,
+      "loss": 0.0,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -42.68964385986328,
+      "rewards/margins": 1.655322790145874,
+      "rewards/rejected": -44.34496307373047,
       "step": 5345
     },
     {
       "epoch": 2.8633550760996824,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.4436930405833195e-08,
       "learning_rate": 6.246301283158728e-09,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.896940231323242,
+      "logits/rejected": 8.847696304321289,
+      "logps/chosen": -43.179649353027344,
+      "logps/rejected": -43.715843200683594,
+      "loss": 0.0,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": -43.179649353027344,
+      "rewards/margins": 0.5361932516098022,
+      "rewards/rejected": -43.715843200683594,
       "step": 5350
     },
     {
       "epoch": 2.8660311088810837,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.3190865904739134e-09,
       "learning_rate": 6.0032928154576944e-09,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.236913681030273,
+      "logits/rejected": 9.100135803222656,
+      "logps/chosen": -42.778934478759766,
+      "logps/rejected": -44.05268096923828,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -42.778934478759766,
+      "rewards/margins": 1.2737449407577515,
+      "rewards/rejected": -44.05268096923828,
       "step": 5355
     },
     {
       "epoch": 2.8687071416624854,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.5797463887560218e-09,
       "learning_rate": 5.76507710597629e-09,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.130548477172852,
+      "logits/rejected": 8.44378662109375,
+      "logps/chosen": -42.159114837646484,
+      "logps/rejected": -44.34185028076172,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -42.159114837646484,
+      "rewards/margins": 2.1827340126037598,
+      "rewards/rejected": -44.34185028076172,
       "step": 5360
     },
     {
       "epoch": 2.8713831744438867,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.2612782631810013e-11,
       "learning_rate": 5.531656465884438e-09,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.986358642578125,
+      "logits/rejected": 8.461038589477539,
+      "logps/chosen": -42.87377166748047,
+      "logps/rejected": -44.58510971069336,
+      "loss": 0.0,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -42.87377166748047,
+      "rewards/margins": 1.7113420963287354,
+      "rewards/rejected": -44.58510971069336,
       "step": 5365
     },
     {
       "epoch": 2.8740592072252884,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.2347539369113793e-11,
       "learning_rate": 5.303033159830217e-09,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.828839302062988,
+      "logits/rejected": 8.831342697143555,
+      "logps/chosen": -42.833251953125,
+      "logps/rejected": -44.327335357666016,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -42.833251953125,
+      "rewards/margins": 1.494086503982544,
+      "rewards/rejected": -44.327335357666016,
       "step": 5370
     },
     {
       "epoch": 2.87673524000669,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.1706719966774322e-13,
       "learning_rate": 5.079209405917939e-09,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.819482803344727,
+      "logits/rejected": 8.773767471313477,
+      "logps/chosen": -43.446250915527344,
+      "logps/rejected": -45.201904296875,
+      "loss": 0.0,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -43.446250915527344,
+      "rewards/margins": 1.7556512355804443,
+      "rewards/rejected": -45.201904296875,
       "step": 5375
     },
     {
       "epoch": 2.879411272788092,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 5.11554878723892e-10,
       "learning_rate": 4.860187375686664e-09,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.458517074584961,
+      "logits/rejected": 8.459425926208496,
+      "logps/chosen": -42.429019927978516,
+      "logps/rejected": -44.0147590637207,
+      "loss": 0.0,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -42.429019927978516,
+      "rewards/margins": 1.5857442617416382,
+      "rewards/rejected": -44.0147590637207,
       "step": 5380
     },
     {
       "epoch": 2.882087305569493,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 7.844521320531377e-12,
       "learning_rate": 4.64596919408905e-09,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.818659782409668,
+      "logits/rejected": 8.798497200012207,
+      "logps/chosen": -41.92902374267578,
+      "logps/rejected": -43.45621871948242,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -41.92902374267578,
+      "rewards/margins": 1.5271942615509033,
+      "rewards/rejected": -43.45621871948242,
       "step": 5385
     },
     {
       "epoch": 2.884763338350895,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.961375735543267e-06,
       "learning_rate": 4.436556939470814e-09,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.121790885925293,
+      "logits/rejected": 8.664724349975586,
+      "logps/chosen": -44.26715087890625,
+      "logps/rejected": -44.96699905395508,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -44.26715087890625,
+      "rewards/margins": 0.6998492479324341,
+      "rewards/rejected": -44.96699905395508,
       "step": 5390
     },
     {
       "epoch": 2.887439371132296,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 6.458950029941331e-07,
       "learning_rate": 4.23195264355064e-09,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.128458023071289,
+      "logits/rejected": 8.465438842773438,
+      "logps/chosen": -42.92496109008789,
+      "logps/rejected": -44.21221923828125,
+      "loss": 0.0,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -42.92496109008789,
+      "rewards/margins": 1.2872591018676758,
+      "rewards/rejected": -44.21221923828125,
       "step": 5395
     },
     {
       "epoch": 2.890115403913698,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 9.296836917399921e-07,
       "learning_rate": 4.032158291400245e-09,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.210552215576172,
+      "logits/rejected": 8.299796104431152,
+      "logps/chosen": -42.495174407958984,
+      "logps/rejected": -44.12283706665039,
+      "loss": 0.0,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -42.495174407958984,
+      "rewards/margins": 1.6276592016220093,
+      "rewards/rejected": -44.12283706665039,
       "step": 5400
     },
     {
       "epoch": 2.8927914366950995,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 0.00015403173900330994,
       "learning_rate": 3.837175821425398e-09,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.843391418457031,
+      "logits/rejected": 8.573896408081055,
+      "logps/chosen": -43.014854431152344,
+      "logps/rejected": -43.65604019165039,
+      "loss": 0.0,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": -43.014854431152344,
+      "rewards/margins": 0.6411841511726379,
+      "rewards/rejected": -43.65604019165039,
       "step": 5405
     },
     {
       "epoch": 2.8954674694765012,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 8.547261274400296e-07,
       "learning_rate": 3.6470071253467683e-09,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.146087646484375,
+      "logits/rejected": 8.660335540771484,
+      "logps/chosen": -42.8497314453125,
+      "logps/rejected": -44.10491180419922,
+      "loss": 0.0,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -42.8497314453125,
+      "rewards/margins": 1.255183219909668,
+      "rewards/rejected": -44.10491180419922,
       "step": 5410
     },
     {
       "epoch": 2.8981435022579025,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.3361023416888584e-10,
       "learning_rate": 3.461654048181939e-09,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.334836959838867,
+      "logits/rejected": 8.748196601867676,
+      "logps/chosen": -43.159141540527344,
+      "logps/rejected": -44.73859786987305,
+      "loss": 0.0,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -43.159141540527344,
+      "rewards/margins": 1.5794521570205688,
+      "rewards/rejected": -44.73859786987305,
       "step": 5415
     },
     {
       "epoch": 2.9008195350393042,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.6339961467233275e-11,
       "learning_rate": 3.281118388227255e-09,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.09322452545166,
+      "logits/rejected": 8.83458423614502,
+      "logps/chosen": -43.47234344482422,
+      "logps/rejected": -43.72627639770508,
+      "loss": 0.0,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -43.47234344482422,
+      "rewards/margins": 0.25393232703208923,
+      "rewards/rejected": -43.72627639770508,
       "step": 5420
     },
     {
       "epoch": 2.903495567820706,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 9.958094605143179e-11,
       "learning_rate": 3.1054018970405048e-09,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.123544692993164,
+      "logits/rejected": 8.559076309204102,
+      "logps/chosen": -43.154457092285156,
+      "logps/rejected": -44.35981750488281,
+      "loss": 0.0,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -43.154457092285156,
+      "rewards/margins": 1.2053630352020264,
+      "rewards/rejected": -44.35981750488281,
       "step": 5425
     },
     {
       "epoch": 2.906171600602107,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.4276863648133436e-06,
       "learning_rate": 2.9345062794238207e-09,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.339111328125,
+      "logits/rejected": 8.790606498718262,
+      "logps/chosen": -43.166709899902344,
+      "logps/rejected": -44.22993850708008,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -43.166709899902344,
+      "rewards/margins": 1.0632320642471313,
+      "rewards/rejected": -44.22993850708008,
       "step": 5430
     },
     {
       "epoch": 2.908847633383509,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.2759251064601855e-13,
       "learning_rate": 2.7684331934072492e-09,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.134610176086426,
+      "logits/rejected": 8.85561752319336,
+      "logps/chosen": -42.64212417602539,
+      "logps/rejected": -43.90853500366211,
+      "loss": 0.0,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -42.64212417602539,
+      "rewards/margins": 1.2664151191711426,
+      "rewards/rejected": -43.90853500366211,
       "step": 5435
     },
     {
       "epoch": 2.9115236661649107,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.105596137804807e-08,
       "learning_rate": 2.6071842502326526e-09,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.124176025390625,
+      "logits/rejected": 8.62415885925293,
+      "logps/chosen": -42.54281234741211,
+      "logps/rejected": -43.394012451171875,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -42.54281234741211,
+      "rewards/margins": 0.8511954545974731,
+      "rewards/rejected": -43.394012451171875,
       "step": 5440
     },
     {
       "epoch": 2.9141996989463124,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 6.404810854747346e-06,
       "learning_rate": 2.450761014337888e-09,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.63016414642334,
+      "logits/rejected": 8.456235885620117,
+      "logps/chosen": -42.82942199707031,
+      "logps/rejected": -43.71613311767578,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -42.82942199707031,
+      "rewards/margins": 0.886710524559021,
+      "rewards/rejected": -43.71613311767578,
       "step": 5445
     },
     {
       "epoch": 2.9168757317277136,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 5.547656492059674e-07,
       "learning_rate": 2.299165003341985e-09,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.547208786010742,
+      "logits/rejected": 8.238048553466797,
+      "logps/chosen": -42.25947189331055,
+      "logps/rejected": -42.96728515625,
+      "loss": 0.0,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -42.25947189331055,
+      "rewards/margins": 0.7078157067298889,
+      "rewards/rejected": -42.96728515625,
       "step": 5450
     },
     {
       "epoch": 2.9195517645091154,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 6.663820388596482e-06,
       "learning_rate": 2.1523976880299945e-09,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.08293342590332,
+      "logits/rejected": 8.70988655090332,
+      "logps/chosen": -42.87151336669922,
+      "logps/rejected": -44.23230743408203,
+      "loss": 0.0,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -42.87151336669922,
+      "rewards/margins": 1.3607940673828125,
+      "rewards/rejected": -44.23230743408203,
       "step": 5455
     },
     {
       "epoch": 2.9222277972905166,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.6349918700593994e-12,
       "learning_rate": 2.010460492339161e-09,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.973346710205078,
+      "logits/rejected": 8.453689575195312,
+      "logps/chosen": -42.803321838378906,
+      "logps/rejected": -44.63463592529297,
+      "loss": 0.0,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -42.803321838378906,
+      "rewards/margins": 1.8313194513320923,
+      "rewards/rejected": -44.63463592529297,
       "step": 5460
     },
     {
       "epoch": 2.9249038300719183,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.86117986466598e-05,
       "learning_rate": 1.8733547933446614e-09,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.281246185302734,
+      "logits/rejected": 8.57530689239502,
+      "logps/chosen": -42.82166290283203,
+      "logps/rejected": -43.41827392578125,
+      "loss": 0.0,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -42.82166290283203,
+      "rewards/margins": 0.596615731716156,
+      "rewards/rejected": -43.41827392578125,
       "step": 5465
     },
     {
       "epoch": 2.92757986285332,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.4918971089722022e-09,
       "learning_rate": 1.7410819212467231e-09,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.121832847595215,
+      "logits/rejected": 8.893570899963379,
+      "logps/chosen": -44.752296447753906,
+      "logps/rejected": -45.21686553955078,
+      "loss": 0.0,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -44.752296447753906,
+      "rewards/margins": 0.4645637571811676,
+      "rewards/rejected": -45.21686553955078,
       "step": 5470
     },
     {
       "epoch": 2.9302558956347218,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.1491199409456674e-11,
       "learning_rate": 1.613643159357192e-09,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.142816543579102,
+      "logits/rejected": 9.115776062011719,
+      "logps/chosen": -43.880592346191406,
+      "logps/rejected": -44.0962028503418,
+      "loss": 0.0,
+      "rewards/accuracies": 0.48750001192092896,
+      "rewards/chosen": -43.880592346191406,
+      "rewards/margins": 0.2156103551387787,
+      "rewards/rejected": -44.0962028503418,
       "step": 5475
     },
     {
       "epoch": 2.932931928416123,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.1122140933028381e-08,
       "learning_rate": 1.4910397440875967e-09,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.083062171936035,
+      "logits/rejected": 8.739364624023438,
+      "logps/chosen": -43.33385467529297,
+      "logps/rejected": -44.41044616699219,
+      "loss": 0.0,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": -43.33385467529297,
+      "rewards/margins": 1.0765869617462158,
+      "rewards/rejected": -44.41044616699219,
       "step": 5480
     },
     {
       "epoch": 2.9356079611975248,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 7.680496960003502e-10,
       "learning_rate": 1.3732728649368253e-09,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.916650772094727,
+      "logits/rejected": 8.185681343078613,
+      "logps/chosen": -43.41474914550781,
+      "logps/rejected": -44.25482940673828,
+      "loss": 0.0,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -43.41474914550781,
+      "rewards/margins": 0.8400863409042358,
+      "rewards/rejected": -44.25482940673828,
       "step": 5485
     },
     {
       "epoch": 2.938283993978926,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.41509761269715e-10,
       "learning_rate": 1.260343664479524e-09,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.641057968139648,
+      "logits/rejected": 8.601696968078613,
+      "logps/chosen": -43.54811477661133,
+      "logps/rejected": -43.978782653808594,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5062500238418579,
+      "rewards/chosen": -43.54811477661133,
+      "rewards/margins": 0.43066921830177307,
+      "rewards/rejected": -43.978782653808594,
       "step": 5490
     },
     {
       "epoch": 2.9409600267603278,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 6.619161244267428e-06,
       "learning_rate": 1.1522532383554384e-09,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.290199279785156,
+      "logits/rejected": 8.573363304138184,
+      "logps/chosen": -43.536781311035156,
+      "logps/rejected": -44.43586349487305,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -43.536781311035156,
+      "rewards/margins": 0.8990872502326965,
+      "rewards/rejected": -44.43586349487305,
       "step": 5495
     },
     {
       "epoch": 2.9436360595417295,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.2023568582106304e-10,
       "learning_rate": 1.049002635258256e-09,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.85502815246582,
+      "logits/rejected": 8.50245475769043,
+      "logps/chosen": -43.516456604003906,
+      "logps/rejected": -44.108821868896484,
+      "loss": 0.0,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -43.516456604003906,
+      "rewards/margins": 0.5923608541488647,
+      "rewards/rejected": -44.108821868896484,
       "step": 5500
     },
     {
       "epoch": 2.946312092323131,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 7.862844538251259e-05,
       "learning_rate": 9.505928569258358e-10,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.895758628845215,
+      "logits/rejected": 8.956740379333496,
+      "logps/chosen": -43.34341812133789,
+      "logps/rejected": -44.25432586669922,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -43.34341812133789,
+      "rewards/margins": 0.9109099507331848,
+      "rewards/rejected": -44.25432586669922,
       "step": 5505
     },
     {
       "epoch": 2.9489881251045325,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.3511889003294474e-11,
       "learning_rate": 8.57024858130273e-10,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.06314754486084,
+      "logits/rejected": 8.55306625366211,
+      "logps/chosen": -43.68933868408203,
+      "logps/rejected": -44.623023986816406,
+      "loss": 0.0,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -43.68933868408203,
+      "rewards/margins": 0.9336822628974915,
+      "rewards/rejected": -44.623023986816406,
       "step": 5510
     },
     {
       "epoch": 2.951664157885934,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.091503354543624e-12,
       "learning_rate": 7.682995466686826e-10,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.43451976776123,
+      "logits/rejected": 8.934001922607422,
+      "logps/chosen": -43.223777770996094,
+      "logps/rejected": -44.37071990966797,
+      "loss": 0.0,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -43.223777770996094,
+      "rewards/margins": 1.1469380855560303,
+      "rewards/rejected": -44.37071990966797,
       "step": 5515
     },
     {
       "epoch": 2.9543401906673354,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.620901653731338e-10,
       "learning_rate": 6.844177833543741e-10,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.181394577026367,
+      "logits/rejected": 8.99818229675293,
+      "logps/chosen": -43.271820068359375,
+      "logps/rejected": -44.982200622558594,
+      "loss": 0.0,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -43.271820068359375,
+      "rewards/margins": 1.7103798389434814,
+      "rewards/rejected": -44.982200622558594,
       "step": 5520
     },
     {
       "epoch": 2.957016223448737,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 7.344292945024075e-13,
       "learning_rate": 6.053803820087467e-10,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.069437026977539,
+      "logits/rejected": 8.609970092773438,
+      "logps/chosen": -44.08431625366211,
+      "logps/rejected": -45.02243423461914,
+      "loss": 0.0,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -44.08431625366211,
+      "rewards/margins": 0.9381159543991089,
+      "rewards/rejected": -45.02243423461914,
       "step": 5525
     },
     {
       "epoch": 2.959692256230139,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.768078971846132e-08,
       "learning_rate": 5.311881094528514e-10,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.123296737670898,
+      "logits/rejected": 8.427828788757324,
+      "logps/chosen": -43.0762825012207,
+      "logps/rejected": -44.399314880371094,
+      "loss": 0.0,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -43.0762825012207,
+      "rewards/margins": 1.3230338096618652,
+      "rewards/rejected": -44.399314880371094,
       "step": 5530
     },
     {
       "epoch": 2.9623682890115406,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.89638696647581e-05,
       "learning_rate": 4.6184168550050806e-10,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.684124946594238,
+      "logits/rejected": 8.624351501464844,
+      "logps/chosen": -42.91204071044922,
+      "logps/rejected": -43.33562088012695,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -42.91204071044922,
+      "rewards/margins": 0.4235750138759613,
+      "rewards/rejected": -43.33562088012695,
       "step": 5535
     },
     {
       "epoch": 2.965044321792942,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.7424728723119498e-09,
       "learning_rate": 3.973417829510328e-10,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.220220565795898,
+      "logits/rejected": 8.75816822052002,
+      "logps/chosen": -42.318565368652344,
+      "logps/rejected": -43.9902229309082,
+      "loss": 0.0,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -42.318565368652344,
+      "rewards/margins": 1.671654462814331,
+      "rewards/rejected": -43.9902229309082,
       "step": 5540
     },
     {
       "epoch": 2.9677203545743436,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.9638784506663666e-11,
       "learning_rate": 3.3768902758274377e-10,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.967167854309082,
+      "logits/rejected": 8.66887378692627,
+      "logps/chosen": -41.842193603515625,
+      "logps/rejected": -43.09015655517578,
+      "loss": 0.0,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -41.842193603515625,
+      "rewards/margins": 1.2479666471481323,
+      "rewards/rejected": -43.09015655517578,
       "step": 5545
     },
     {
       "epoch": 2.970396387355745,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.0907062190140083e-07,
       "learning_rate": 2.8288399814691e-10,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.862554550170898,
+      "logits/rejected": 8.886552810668945,
+      "logps/chosen": -43.24170684814453,
+      "logps/rejected": -44.441734313964844,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -43.24170684814453,
+      "rewards/margins": 1.200028657913208,
+      "rewards/rejected": -44.441734313964844,
       "step": 5550
     },
     {
       "epoch": 2.9730724201371466,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.474885916988663e-09,
       "learning_rate": 2.3292722636220066e-10,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.124627113342285,
+      "logits/rejected": 8.516072273254395,
+      "logps/chosen": -42.52320098876953,
+      "logps/rejected": -44.84089279174805,
+      "loss": 0.0,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -42.52320098876953,
+      "rewards/margins": 2.317686080932617,
+      "rewards/rejected": -44.84089279174805,
       "step": 5555
     },
     {
       "epoch": 2.9757484529185483,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 6.353003881160805e-11,
       "learning_rate": 1.8781919690946668e-10,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.577714920043945,
+      "logits/rejected": 8.480096817016602,
+      "logps/chosen": -44.2556266784668,
+      "logps/rejected": -44.30500411987305,
+      "loss": 0.0,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -44.2556266784668,
+      "rewards/margins": 0.04937465116381645,
+      "rewards/rejected": -44.30500411987305,
       "step": 5560
     },
     {
       "epoch": 2.97842448569995,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 8.746418768118171e-08,
       "learning_rate": 1.4756034742696711e-10,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.16465950012207,
+      "logits/rejected": 8.840929985046387,
+      "logps/chosen": -42.676368713378906,
+      "logps/rejected": -43.760292053222656,
+      "loss": 0.0,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -42.676368713378906,
+      "rewards/margins": 1.0839216709136963,
+      "rewards/rejected": -43.760292053222656,
       "step": 5565
     },
     {
       "epoch": 2.9811005184813513,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.1753167945445568e-08,
       "learning_rate": 1.12151068506261e-10,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.012102127075195,
+      "logits/rejected": 8.687840461730957,
+      "logps/chosen": -43.252235412597656,
+      "logps/rejected": -44.85663986206055,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -43.252235412597656,
+      "rewards/margins": 1.6044065952301025,
+      "rewards/rejected": -44.85663986206055,
       "step": 5570
     },
     {
       "epoch": 2.983776551262753,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.5292187555844035e-09,
       "learning_rate": 8.159170368826629e-11,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.963420867919922,
+      "logits/rejected": 8.400527000427246,
+      "logps/chosen": -42.38309097290039,
+      "logps/rejected": -44.467132568359375,
+      "loss": 0.0,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -42.38309097290039,
+      "rewards/margins": 2.0840423107147217,
+      "rewards/rejected": -44.467132568359375,
       "step": 5575
     },
     {
       "epoch": 2.9864525840441547,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.941827391079566e-09,
       "learning_rate": 5.588254946015114e-11,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.213991165161133,
+      "logits/rejected": 8.347885131835938,
+      "logps/chosen": -42.460777282714844,
+      "logps/rejected": -44.171546936035156,
+      "loss": 0.0,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -42.460777282714844,
+      "rewards/margins": 1.710771918296814,
+      "rewards/rejected": -44.171546936035156,
       "step": 5580
     },
     {
       "epoch": 2.989128616825556,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 8.721330057929216e-12,
       "learning_rate": 3.502385525216978e-11,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.794221878051758,
+      "logits/rejected": 8.234247207641602,
+      "logps/chosen": -43.3101921081543,
+      "logps/rejected": -44.4868049621582,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -43.3101921081543,
+      "rewards/margins": 1.176613450050354,
+      "rewards/rejected": -44.4868049621582,
       "step": 5585
     },
     {
       "epoch": 2.9918046496069577,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.3914969676161003e-09,
       "learning_rate": 1.901582343555308e-11,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.635357856750488,
+      "logits/rejected": 8.538853645324707,
+      "logps/chosen": -42.34468078613281,
+      "logps/rejected": -43.14370346069336,
+      "loss": 0.0,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -42.34468078613281,
+      "rewards/margins": 0.7990256547927856,
+      "rewards/rejected": -43.14370346069336,
       "step": 5590
     },
     {
       "epoch": 2.9944806823883594,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.1978369083992432e-08,
       "learning_rate": 7.858609320232634e-12,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 8.80547046661377,
+      "logits/rejected": 8.196683883666992,
+      "logps/chosen": -43.26044845581055,
+      "logps/rejected": -45.38365936279297,
+      "loss": 0.0,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -43.26044845581055,
+      "rewards/margins": 2.123211622238159,
+      "rewards/rejected": -45.38365936279297,
       "step": 5595
     },
     {
       "epoch": 2.9971567151697607,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 6.715830344165064e-13,
       "learning_rate": 1.5523211535639624e-12,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 9.053936958312988,
+      "logits/rejected": 8.509064674377441,
+      "logps/chosen": -43.21258544921875,
+      "logps/rejected": -44.620113372802734,
+      "loss": 0.0,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -43.21258544921875,
+      "rewards/margins": 1.4075287580490112,
+      "rewards/rejected": -44.620113372802734,
       "step": 5600
     },
     {
       "epoch": 2.9971567151697607,
-      "eval_logits/chosen": NaN,
-      "eval_logits/rejected": NaN,
-      "eval_logps/chosen": NaN,
-      "eval_logps/rejected": NaN,
-      "eval_loss": NaN,
-      "eval_rewards/accuracies": 0.0,
-      "eval_rewards/chosen": NaN,
-      "eval_rewards/margins": NaN,
-      "eval_rewards/rejected": NaN,
-      "eval_runtime": 34.7507,
-      "eval_samples_per_second": 38.704,
-      "eval_semantic_entropy": NaN,
-      "eval_steps_per_second": 9.698,
+      "eval_logits/chosen": 7.539320468902588,
+      "eval_logits/rejected": 7.256679534912109,
+      "eval_logps/chosen": -42.77936935424805,
+      "eval_logps/rejected": -43.914947509765625,
+      "eval_loss": 1.999994623247403e-07,
+      "eval_rewards/accuracies": 0.5667656064033508,
+      "eval_rewards/chosen": -42.77936935424805,
+      "eval_rewards/margins": 1.1355794668197632,
+      "eval_rewards/rejected": -43.914947509765625,
+      "eval_runtime": 41.286,
+      "eval_samples_per_second": 32.578,
+      "eval_steps_per_second": 8.163,
       "step": 5600
     },
     {
       "epoch": 2.999297541394882,
       "step": 5604,
       "total_flos": 0.0,
-      "train_loss": 0.024124226235730746,
-      "train_runtime": 28949.9436,
-      "train_samples_per_second": 6.196,
-      "train_steps_per_second": 0.194
+      "train_loss": 0.010826070491851839,
+      "train_runtime": 30412.7115,
+      "train_samples_per_second": 5.898,
+      "train_steps_per_second": 0.184
     }
   ],
   "logging_steps": 5,