{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 0.9990600241708071,
  "eval_steps": 100000,
  "global_step": 465,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.002148516181012488,
      "grad_norm": 4.815295005445406,
      "learning_rate": 1.0638297872340425e-08,
      "logits/chosen": -11.7161283493042,
      "logits/rejected": -11.149711608886719,
      "logps/chosen": -155.27786254882812,
      "logps/rejected": -143.72726440429688,
      "loss": 0.6931,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 1
    },
    {
      "epoch": 0.010742580905062441,
      "grad_norm": 4.665144316910627,
      "learning_rate": 5.3191489361702123e-08,
      "logits/chosen": -10.514410018920898,
      "logits/rejected": -10.558334350585938,
      "logps/chosen": -176.54898071289062,
      "logps/rejected": -177.0060577392578,
      "loss": 0.6932,
      "rewards/accuracies": 0.390625,
      "rewards/chosen": 0.0011347520630806684,
      "rewards/margins": 0.0005755239981226623,
      "rewards/rejected": 0.0005592280067503452,
      "step": 5
    },
    {
      "epoch": 0.021485161810124883,
      "grad_norm": 4.631928897966055,
      "learning_rate": 1.0638297872340425e-07,
      "logits/chosen": -11.231101989746094,
      "logits/rejected": -11.33354377746582,
      "logps/chosen": -200.58383178710938,
      "logps/rejected": -198.61624145507812,
      "loss": 0.693,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.0011275215074419975,
      "rewards/margins": 0.001252257265150547,
      "rewards/rejected": -0.00012473580136429518,
      "step": 10
    },
    {
      "epoch": 0.03222774271518732,
      "grad_norm": 4.861132978246578,
      "learning_rate": 1.5957446808510638e-07,
      "logits/chosen": -10.90822696685791,
      "logits/rejected": -10.716678619384766,
      "logps/chosen": -216.2873992919922,
      "logps/rejected": -203.54527282714844,
      "loss": 0.6932,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.0005270905676297843,
      "rewards/margins": 4.5846936700399965e-05,
      "rewards/rejected": -0.0005729374242946506,
      "step": 15
    },
    {
      "epoch": 0.042970323620249766,
      "grad_norm": 4.809482719935914,
      "learning_rate": 2.127659574468085e-07,
      "logits/chosen": -11.042614936828613,
      "logits/rejected": -10.997823715209961,
      "logps/chosen": -172.80630493164062,
      "logps/rejected": -172.10458374023438,
      "loss": 0.693,
      "rewards/accuracies": 0.48750001192092896,
      "rewards/chosen": -0.00028278256650082767,
      "rewards/margins": 2.6181707653449848e-05,
      "rewards/rejected": -0.000308964226860553,
      "step": 20
    },
    {
      "epoch": 0.05371290452531221,
      "grad_norm": 4.825836096297444,
      "learning_rate": 2.659574468085106e-07,
      "logits/chosen": -11.554967880249023,
      "logits/rejected": -11.35639762878418,
      "logps/chosen": -191.2166290283203,
      "logps/rejected": -184.11294555664062,
      "loss": 0.6928,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.0014816813636571169,
      "rewards/margins": -0.0005183071480132639,
      "rewards/rejected": 0.0019999886862933636,
      "step": 25
    },
    {
      "epoch": 0.06445548543037465,
      "grad_norm": 4.717146773464201,
      "learning_rate": 3.1914893617021275e-07,
      "logits/chosen": -11.667179107666016,
      "logits/rejected": -11.451859474182129,
      "logps/chosen": -185.59962463378906,
      "logps/rejected": -185.79769897460938,
      "loss": 0.6923,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.0023945302236825228,
      "rewards/margins": 0.0009661655058152974,
      "rewards/rejected": 0.0014283646596595645,
      "step": 30
    },
    {
      "epoch": 0.07519806633543709,
      "grad_norm": 4.359788436218923,
      "learning_rate": 3.7234042553191484e-07,
      "logits/chosen": -10.990551948547363,
      "logits/rejected": -10.911511421203613,
      "logps/chosen": -160.88067626953125,
      "logps/rejected": -162.3435821533203,
      "loss": 0.6919,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.0046579958871006966,
      "rewards/margins": 0.0034069078974425793,
      "rewards/rejected": 0.0012510876404121518,
      "step": 35
    },
    {
      "epoch": 0.08594064724049953,
      "grad_norm": 4.682184048048129,
      "learning_rate": 4.25531914893617e-07,
      "logits/chosen": -10.937858581542969,
      "logits/rejected": -10.48929500579834,
      "logps/chosen": -167.89256286621094,
      "logps/rejected": -166.53628540039062,
      "loss": 0.6905,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": 0.008901619352400303,
      "rewards/margins": 0.00515585346147418,
      "rewards/rejected": 0.0037457651924341917,
      "step": 40
    },
    {
      "epoch": 0.09668322814556198,
      "grad_norm": 4.77155478266984,
      "learning_rate": 4.787234042553192e-07,
      "logits/chosen": -11.142602920532227,
      "logits/rejected": -10.850725173950195,
      "logps/chosen": -175.64431762695312,
      "logps/rejected": -174.0598602294922,
      "loss": 0.689,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": 0.008818638511002064,
      "rewards/margins": 0.006885365583002567,
      "rewards/rejected": 0.0019332747906446457,
      "step": 45
    },
    {
      "epoch": 0.10742580905062442,
      "grad_norm": 5.101867612304127,
      "learning_rate": 4.999364550115248e-07,
      "logits/chosen": -12.041606903076172,
      "logits/rejected": -11.635477066040039,
      "logps/chosen": -182.34841918945312,
      "logps/rejected": -178.40005493164062,
      "loss": 0.6876,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.008523710072040558,
      "rewards/margins": 0.016866037622094154,
      "rewards/rejected": -0.008342329412698746,
      "step": 50
    },
    {
      "epoch": 0.11816838995568685,
      "grad_norm": 5.096625071764424,
      "learning_rate": 4.995482415049122e-07,
      "logits/chosen": -11.534649848937988,
      "logits/rejected": -11.349666595458984,
      "logps/chosen": -186.76443481445312,
      "logps/rejected": -180.79470825195312,
      "loss": 0.6831,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": 0.0006141850026324391,
      "rewards/margins": 0.01560161542147398,
      "rewards/rejected": -0.014987431466579437,
      "step": 55
    },
    {
      "epoch": 0.1289109708607493,
      "grad_norm": 5.256195598008515,
      "learning_rate": 4.98807664757556e-07,
      "logits/chosen": -12.30699634552002,
      "logits/rejected": -11.902242660522461,
      "logps/chosen": -209.91812133789062,
      "logps/rejected": -200.37374877929688,
      "loss": 0.6758,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.011014172807335854,
      "rewards/margins": 0.033601436764001846,
      "rewards/rejected": -0.04461560770869255,
      "step": 60
    },
    {
      "epoch": 0.13965355176581173,
      "grad_norm": 5.332372855512194,
      "learning_rate": 4.977157704671584e-07,
      "logits/chosen": -11.759092330932617,
      "logits/rejected": -11.736629486083984,
      "logps/chosen": -191.42971801757812,
      "logps/rejected": -190.9018096923828,
      "loss": 0.6745,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.05662290379405022,
      "rewards/margins": 0.05431179329752922,
      "rewards/rejected": -0.11093469709157944,
      "step": 65
    },
    {
      "epoch": 0.15039613267087418,
      "grad_norm": 5.384194766315848,
      "learning_rate": 4.962741003933742e-07,
      "logits/chosen": -12.996328353881836,
      "logits/rejected": -12.800023078918457,
      "logps/chosen": -205.3074188232422,
      "logps/rejected": -207.24588012695312,
      "loss": 0.6696,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.10499341785907745,
      "rewards/margins": 0.028122153133153915,
      "rewards/rejected": -0.13311555981636047,
      "step": 70
    },
    {
      "epoch": 0.16113871357593662,
      "grad_norm": 6.074413377005886,
      "learning_rate": 4.944846901808396e-07,
      "logits/chosen": -13.731378555297852,
      "logits/rejected": -13.588099479675293,
      "logps/chosen": -207.2079620361328,
      "logps/rejected": -211.04074096679688,
      "loss": 0.6575,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.15880520641803741,
      "rewards/margins": 0.08157539367675781,
      "rewards/rejected": -0.24038061499595642,
      "step": 75
    },
    {
      "epoch": 0.17188129448099906,
      "grad_norm": 6.452938390183295,
      "learning_rate": 4.923500664848326e-07,
      "logits/chosen": -13.624473571777344,
      "logits/rejected": -13.508392333984375,
      "logps/chosen": -218.89053344726562,
      "logps/rejected": -221.2880401611328,
      "loss": 0.6605,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.2407272756099701,
      "rewards/margins": 0.07505510747432709,
      "rewards/rejected": -0.3157823979854584,
      "step": 80
    },
    {
      "epoch": 0.1826238753860615,
      "grad_norm": 7.438055763179647,
      "learning_rate": 4.898732434036243e-07,
      "logits/chosen": -14.367892265319824,
      "logits/rejected": -14.180076599121094,
      "logps/chosen": -186.84841918945312,
      "logps/rejected": -204.40371704101562,
      "loss": 0.6488,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.2892145812511444,
      "rewards/margins": 0.13178381323814392,
      "rewards/rejected": -0.4209984242916107,
      "step": 85
    },
    {
      "epoch": 0.19336645629112395,
      "grad_norm": 7.432291163652407,
      "learning_rate": 4.870577182225589e-07,
      "logits/chosen": -14.431337356567383,
      "logits/rejected": -14.528360366821289,
      "logps/chosen": -235.01089477539062,
      "logps/rejected": -242.1006622314453,
      "loss": 0.6544,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.3304561972618103,
      "rewards/margins": 0.0850444883108139,
      "rewards/rejected": -0.4155007302761078,
      "step": 90
    },
    {
      "epoch": 0.2041090371961864,
      "grad_norm": 7.26568179641241,
      "learning_rate": 4.839074664758704e-07,
      "logits/chosen": -15.507716178894043,
      "logits/rejected": -15.54358196258545,
      "logps/chosen": -219.27432250976562,
      "logps/rejected": -232.99215698242188,
      "loss": 0.6392,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.425987184047699,
      "rewards/margins": 0.14367976784706116,
      "rewards/rejected": -0.5696669816970825,
      "step": 95
    },
    {
      "epoch": 0.21485161810124884,
      "grad_norm": 8.511941780068044,
      "learning_rate": 4.804269363332112e-07,
      "logits/chosen": -14.985662460327148,
      "logits/rejected": -14.705190658569336,
      "logps/chosen": -198.18209838867188,
      "logps/rejected": -206.1742401123047,
      "loss": 0.6385,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.440185546875,
      "rewards/margins": 0.12437309324741364,
      "rewards/rejected": -0.5645586252212524,
      "step": 100
    },
    {
      "epoch": 0.22559419900631128,
      "grad_norm": 8.344983653724734,
      "learning_rate": 4.7662104231881575e-07,
      "logits/chosen": -15.85981273651123,
      "logits/rejected": -15.64745807647705,
      "logps/chosen": -245.8246612548828,
      "logps/rejected": -250.5571746826172,
      "loss": 0.6295,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.6445275545120239,
      "rewards/margins": 0.055053092539310455,
      "rewards/rejected": -0.6995806694030762,
      "step": 105
    },
    {
      "epoch": 0.2363367799113737,
      "grad_norm": 8.805058588475875,
      "learning_rate": 4.724951583721707e-07,
      "logits/chosen": -16.028610229492188,
      "logits/rejected": -16.066186904907227,
      "logps/chosen": -226.5513153076172,
      "logps/rejected": -243.8558807373047,
      "loss": 0.6242,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.6525581479072571,
      "rewards/margins": 0.1965942531824112,
      "rewards/rejected": -0.8491523861885071,
      "step": 110
    },
    {
      "epoch": 0.24707936081643614,
      "grad_norm": 9.436386913158618,
      "learning_rate": 4.68055110259988e-07,
      "logits/chosen": -17.072982788085938,
      "logits/rejected": -17.03071403503418,
      "logps/chosen": -255.8236541748047,
      "logps/rejected": -276.999267578125,
      "loss": 0.6015,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7960177659988403,
      "rewards/margins": 0.268364280462265,
      "rewards/rejected": -1.0643821954727173,
      "step": 115
    },
    {
      "epoch": 0.2578219417214986,
      "grad_norm": 9.856903496627183,
      "learning_rate": 4.633071673501959e-07,
      "logits/chosen": -16.772096633911133,
      "logits/rejected": -16.724201202392578,
      "logps/chosen": -267.2545471191406,
      "logps/rejected": -286.8907470703125,
      "loss": 0.6037,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.9270988702774048,
      "rewards/margins": 0.17965544760227203,
      "rewards/rejected": -1.1067544221878052,
      "step": 120
    },
    {
      "epoch": 0.26856452262656105,
      "grad_norm": 11.574882560301482,
      "learning_rate": 4.5825803375956355e-07,
      "logits/chosen": -18.174747467041016,
      "logits/rejected": -17.954605102539062,
      "logps/chosen": -286.30426025390625,
      "logps/rejected": -311.0242919921875,
      "loss": 0.612,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -1.0375328063964844,
      "rewards/margins": 0.28692322969436646,
      "rewards/rejected": -1.324455976486206,
      "step": 125
    },
    {
      "epoch": 0.27930710353162347,
      "grad_norm": 12.175022480779115,
      "learning_rate": 4.529148388874576e-07,
      "logits/chosen": -18.297924041748047,
      "logits/rejected": -18.335847854614258,
      "logps/chosen": -281.75067138671875,
      "logps/rejected": -314.5852966308594,
      "loss": 0.6002,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -1.0217819213867188,
      "rewards/margins": 0.3027891218662262,
      "rewards/rejected": -1.324571132659912,
      "step": 130
    },
    {
      "epoch": 0.29004968443668594,
      "grad_norm": 118.26976750176874,
      "learning_rate": 4.472851273490984e-07,
      "logits/chosen": -18.722200393676758,
      "logits/rejected": -18.794912338256836,
      "logps/chosen": -339.0028991699219,
      "logps/rejected": -374.34576416015625,
      "loss": 0.5702,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.2525867223739624,
      "rewards/margins": 0.4258871078491211,
      "rewards/rejected": -1.6784738302230835,
      "step": 135
    },
    {
      "epoch": 0.30079226534174835,
      "grad_norm": 12.772135480895352,
      "learning_rate": 4.4137684832252913e-07,
      "logits/chosen": -18.832077026367188,
      "logits/rejected": -18.659208297729492,
      "logps/chosen": -345.35015869140625,
      "logps/rejected": -388.00726318359375,
      "loss": 0.5654,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -1.3465111255645752,
      "rewards/margins": 0.5241008400917053,
      "rewards/rejected": -1.8706119060516357,
      "step": 140
    },
    {
      "epoch": 0.31153484624681077,
      "grad_norm": 12.383899725959822,
      "learning_rate": 4.351983443243409e-07,
      "logits/chosen": -19.82056999206543,
      "logits/rejected": -19.986614227294922,
      "logps/chosen": -354.8511657714844,
      "logps/rejected": -392.865966796875,
      "loss": 0.571,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -1.5482107400894165,
      "rewards/margins": 0.39296191930770874,
      "rewards/rejected": -1.9411723613739014,
      "step": 145
    },
    {
      "epoch": 0.32227742715187324,
      "grad_norm": 14.735365105108821,
      "learning_rate": 4.287583394300015e-07,
      "logits/chosen": -18.688817977905273,
      "logits/rejected": -18.646465301513672,
      "logps/chosen": -351.5058288574219,
      "logps/rejected": -397.3396911621094,
      "loss": 0.5707,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.7229034900665283,
      "rewards/margins": 0.48934563994407654,
      "rewards/rejected": -2.2122488021850586,
      "step": 150
    },
    {
      "epoch": 0.33302000805693566,
      "grad_norm": 13.312063222247302,
      "learning_rate": 4.220659269554216e-07,
      "logits/chosen": -19.77194595336914,
      "logits/rejected": -19.732973098754883,
      "logps/chosen": -348.5200500488281,
      "logps/rejected": -388.44189453125,
      "loss": 0.5553,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -1.6245940923690796,
      "rewards/margins": 0.4843548834323883,
      "rewards/rejected": -2.1089487075805664,
      "step": 155
    },
    {
      "epoch": 0.34376258896199813,
      "grad_norm": 12.019324231367111,
      "learning_rate": 4.1513055661715206e-07,
      "logits/chosen": -19.84881591796875,
      "logits/rejected": -19.640766143798828,
      "logps/chosen": -377.59075927734375,
      "logps/rejected": -410.1768493652344,
      "loss": 0.5591,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -1.8603988885879517,
      "rewards/margins": 0.37856483459472656,
      "rewards/rejected": -2.2389636039733887,
      "step": 160
    },
    {
      "epoch": 0.35450516986706054,
      "grad_norm": 11.824585303309874,
      "learning_rate": 4.0796202118934096e-07,
      "logits/chosen": -19.047245025634766,
      "logits/rejected": -19.169963836669922,
      "logps/chosen": -342.62176513671875,
      "logps/rejected": -398.24237060546875,
      "loss": 0.5394,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -1.6620702743530273,
      "rewards/margins": 0.6188061237335205,
      "rewards/rejected": -2.280876636505127,
      "step": 165
    },
    {
      "epoch": 0.365247750772123,
      "grad_norm": 11.817356726675012,
      "learning_rate": 4.0057044267629303e-07,
      "logits/chosen": -18.676464080810547,
      "logits/rejected": -18.591510772705078,
      "logps/chosen": -377.8108825683594,
      "logps/rejected": -435.5389709472656,
      "loss": 0.5568,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -1.849261999130249,
      "rewards/margins": 0.5185360312461853,
      "rewards/rejected": -2.367798328399658,
      "step": 170
    },
    {
      "epoch": 0.37599033167718543,
      "grad_norm": 13.96823700531248,
      "learning_rate": 3.9296625802015354e-07,
      "logits/chosen": -19.861886978149414,
      "logits/rejected": -19.66740608215332,
      "logps/chosen": -365.8138122558594,
      "logps/rejected": -428.0379943847656,
      "loss": 0.5511,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -1.8249616622924805,
      "rewards/margins": 0.604793131351471,
      "rewards/rejected": -2.4297547340393066,
      "step": 175
    },
    {
      "epoch": 0.3867329125822479,
      "grad_norm": 13.667628227402297,
      "learning_rate": 3.851602043638994e-07,
      "logits/chosen": -20.17488670349121,
      "logits/rejected": -20.171833038330078,
      "logps/chosen": -392.9968566894531,
      "logps/rejected": -458.20660400390625,
      "loss": 0.5262,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -2.052845001220703,
      "rewards/margins": 0.6375163793563843,
      "rewards/rejected": -2.690361499786377,
      "step": 180
    },
    {
      "epoch": 0.3974754934873103,
      "grad_norm": 15.39892517008571,
      "learning_rate": 3.771633038904446e-07,
      "logits/chosen": -20.305858612060547,
      "logits/rejected": -20.228055953979492,
      "logps/chosen": -416.68560791015625,
      "logps/rejected": -482.61676025390625,
      "loss": 0.5027,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -2.168687105178833,
      "rewards/margins": 0.6908921599388123,
      "rewards/rejected": -2.859579086303711,
      "step": 185
    },
    {
      "epoch": 0.4082180743923728,
      "grad_norm": 12.128334417208155,
      "learning_rate": 3.689868482592684e-07,
      "logits/chosen": -20.484973907470703,
      "logits/rejected": -20.257816314697266,
      "logps/chosen": -466.16424560546875,
      "logps/rejected": -569.1168212890625,
      "loss": 0.5224,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -2.359057903289795,
      "rewards/margins": 1.0049209594726562,
      "rewards/rejected": -3.363978862762451,
      "step": 190
    },
    {
      "epoch": 0.4189606552974352,
      "grad_norm": 15.400085837818166,
      "learning_rate": 3.606423826625414e-07,
      "logits/chosen": -20.399295806884766,
      "logits/rejected": -20.145183563232422,
      "logps/chosen": -396.54296875,
      "logps/rejected": -456.8973693847656,
      "loss": 0.5273,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -1.9560238122940063,
      "rewards/margins": 0.6280576586723328,
      "rewards/rejected": -2.5840814113616943,
      "step": 195
    },
    {
      "epoch": 0.4297032362024977,
      "grad_norm": 14.374891311569105,
      "learning_rate": 3.5214168952326203e-07,
      "logits/chosen": -20.65167808532715,
      "logits/rejected": -20.682342529296875,
      "logps/chosen": -410.12579345703125,
      "logps/rejected": -451.20355224609375,
      "loss": 0.5324,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -2.331077814102173,
      "rewards/margins": 0.4432566165924072,
      "rewards/rejected": -2.77433443069458,
      "step": 200
    },
    {
      "epoch": 0.4404458171075601,
      "grad_norm": 14.839772397078658,
      "learning_rate": 3.4349677185842243e-07,
      "logits/chosen": -20.4150333404541,
      "logits/rejected": -20.26569938659668,
      "logps/chosen": -389.5531311035156,
      "logps/rejected": -442.98388671875,
      "loss": 0.4873,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -2.1742894649505615,
      "rewards/margins": 0.5466232895851135,
      "rewards/rejected": -2.720912456512451,
      "step": 205
    },
    {
      "epoch": 0.45118839801262256,
      "grad_norm": 73.54965957378546,
      "learning_rate": 3.347198363306941e-07,
      "logits/chosen": -20.8847713470459,
      "logits/rejected": -20.70589828491211,
      "logps/chosen": -467.5254821777344,
      "logps/rejected": -525.5687255859375,
      "loss": 0.5156,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -2.7870867252349854,
      "rewards/margins": 0.5678809881210327,
      "rewards/rejected": -3.3549678325653076,
      "step": 210
    },
    {
      "epoch": 0.461930978917685,
      "grad_norm": 13.712145057455235,
      "learning_rate": 3.2582327601256564e-07,
      "logits/chosen": -21.396650314331055,
      "logits/rejected": -21.34550666809082,
      "logps/chosen": -439.74151611328125,
      "logps/rejected": -528.05908203125,
      "loss": 0.52,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -2.560230255126953,
      "rewards/margins": 0.9471774101257324,
      "rewards/rejected": -3.5074074268341064,
      "step": 215
    },
    {
      "epoch": 0.4726735598227474,
      "grad_norm": 13.556889594752073,
      "learning_rate": 3.168196528872682e-07,
      "logits/chosen": -20.23183250427246,
      "logits/rejected": -19.94950294494629,
      "logps/chosen": -433.8387756347656,
      "logps/rejected": -496.75946044921875,
      "loss": 0.52,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.558835506439209,
      "rewards/margins": 0.7217434048652649,
      "rewards/rejected": -3.28057861328125,
      "step": 220
    },
    {
      "epoch": 0.48341614072780986,
      "grad_norm": 22.472329896519035,
      "learning_rate": 3.077216801111989e-07,
      "logits/chosen": -20.4947452545166,
      "logits/rejected": -20.315448760986328,
      "logps/chosen": -434.6400451660156,
      "logps/rejected": -493.80181884765625,
      "loss": 0.4996,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -2.2407140731811523,
      "rewards/margins": 0.6915763020515442,
      "rewards/rejected": -2.932290554046631,
      "step": 225
    },
    {
      "epoch": 0.4941587216328723,
      "grad_norm": 14.506146770106906,
      "learning_rate": 2.9854220406288666e-07,
      "logits/chosen": -20.191608428955078,
      "logits/rejected": -19.9714412689209,
      "logps/chosen": -414.80535888671875,
      "logps/rejected": -482.10015869140625,
      "loss": 0.4813,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -2.2866439819335938,
      "rewards/margins": 0.7474405169487,
      "rewards/rejected": -3.0340847969055176,
      "step": 230
    },
    {
      "epoch": 0.5049013025379347,
      "grad_norm": 13.028250670943683,
      "learning_rate": 2.892941862038475e-07,
      "logits/chosen": -19.92314910888672,
      "logits/rejected": -20.13300895690918,
      "logps/chosen": -414.1224670410156,
      "logps/rejected": -509.6715393066406,
      "loss": 0.5218,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -2.3021442890167236,
      "rewards/margins": 0.8769405484199524,
      "rewards/rejected": -3.1790850162506104,
      "step": 235
    },
    {
      "epoch": 0.5156438834429972,
      "grad_norm": 16.027232766680346,
      "learning_rate": 2.7999068477694325e-07,
      "logits/chosen": -20.204256057739258,
      "logits/rejected": -20.092927932739258,
      "logps/chosen": -400.50579833984375,
      "logps/rejected": -485.25225830078125,
      "loss": 0.4754,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -2.113093376159668,
      "rewards/margins": 0.857138454914093,
      "rewards/rejected": -2.970231771469116,
      "step": 240
    },
    {
      "epoch": 0.5263864643480596,
      "grad_norm": 14.352076936568274,
      "learning_rate": 2.706448363680831e-07,
      "logits/chosen": -20.70675277709961,
      "logits/rejected": -20.695838928222656,
      "logps/chosen": -476.7577209472656,
      "logps/rejected": -539.758544921875,
      "loss": 0.4996,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -2.558915615081787,
      "rewards/margins": 0.7049631476402283,
      "rewards/rejected": -3.2638790607452393,
      "step": 245
    },
    {
      "epoch": 0.5371290452531221,
      "grad_norm": 13.840765525316273,
      "learning_rate": 2.6126983735730555e-07,
      "logits/chosen": -20.357967376708984,
      "logits/rejected": -20.35165023803711,
      "logps/chosen": -415.8984375,
      "logps/rejected": -537.2420654296875,
      "loss": 0.4602,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -2.3948779106140137,
      "rewards/margins": 1.2330429553985596,
      "rewards/rejected": -3.6279208660125732,
      "step": 250
    },
    {
      "epoch": 0.5478716261581845,
      "grad_norm": 22.763085198341653,
      "learning_rate": 2.518789252854305e-07,
      "logits/chosen": -20.021839141845703,
      "logits/rejected": -19.900529861450195,
      "logps/chosen": -421.52142333984375,
      "logps/rejected": -513.4830322265625,
      "loss": 0.481,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -2.304274797439575,
      "rewards/margins": 0.8148962259292603,
      "rewards/rejected": -3.119170665740967,
      "step": 255
    },
    {
      "epoch": 0.5586142070632469,
      "grad_norm": 13.166723636543091,
      "learning_rate": 2.424853601625913e-07,
      "logits/chosen": -20.84389877319336,
      "logits/rejected": -20.853782653808594,
      "logps/chosen": -437.2691345214844,
      "logps/rejected": -521.557861328125,
      "loss": 0.4885,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -2.4156320095062256,
      "rewards/margins": 0.8837593197822571,
      "rewards/rejected": -3.299391269683838,
      "step": 260
    },
    {
      "epoch": 0.5693567879683094,
      "grad_norm": 15.08081211674444,
      "learning_rate": 2.3310240574504182e-07,
      "logits/chosen": -20.66632652282715,
      "logits/rejected": -20.677610397338867,
      "logps/chosen": -436.674560546875,
      "logps/rejected": -518.9525146484375,
      "loss": 0.4856,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -2.5705347061157227,
      "rewards/margins": 0.8193251490592957,
      "rewards/rejected": -3.389859676361084,
      "step": 265
    },
    {
      "epoch": 0.5800993688733719,
      "grad_norm": 14.09838406164426,
      "learning_rate": 2.2374331080667165e-07,
      "logits/chosen": -21.183074951171875,
      "logits/rejected": -21.082101821899414,
      "logps/chosen": -473.5470275878906,
      "logps/rejected": -534.5736083984375,
      "loss": 0.4661,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -2.6635870933532715,
      "rewards/margins": 0.7532528638839722,
      "rewards/rejected": -3.4168403148651123,
      "step": 270
    },
    {
      "epoch": 0.5908419497784343,
      "grad_norm": 13.797790240550018,
      "learning_rate": 2.1442129043167873e-07,
      "logits/chosen": -20.22403335571289,
      "logits/rejected": -20.4445858001709,
      "logps/chosen": -487.107666015625,
      "logps/rejected": -583.4486083984375,
      "loss": 0.4801,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -2.895622491836548,
      "rewards/margins": 1.0642651319503784,
      "rewards/rejected": -3.959887742996216,
      "step": 275
    },
    {
      "epoch": 0.6015845306834967,
      "grad_norm": 17.151814735789866,
      "learning_rate": 2.0514950735481053e-07,
      "logits/chosen": -20.70838165283203,
      "logits/rejected": -20.784690856933594,
      "logps/chosen": -465.6155700683594,
      "logps/rejected": -568.5462036132812,
      "loss": 0.483,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -2.638089418411255,
      "rewards/margins": 1.0538232326507568,
      "rewards/rejected": -3.6919121742248535,
      "step": 280
    },
    {
      "epoch": 0.6123271115885591,
      "grad_norm": 15.353425386955319,
      "learning_rate": 1.959410533755232e-07,
      "logits/chosen": -20.860107421875,
      "logits/rejected": -20.810924530029297,
      "logps/chosen": -461.963623046875,
      "logps/rejected": -551.6959228515625,
      "loss": 0.4484,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -2.7611382007598877,
      "rewards/margins": 0.9577289819717407,
      "rewards/rejected": -3.7188668251037598,
      "step": 285
    },
    {
      "epoch": 0.6230696924936215,
      "grad_norm": 14.20571732032185,
      "learning_rate": 1.8680893087230203e-07,
      "logits/chosen": -21.382736206054688,
      "logits/rejected": -21.493896484375,
      "logps/chosen": -479.07830810546875,
      "logps/rejected": -589.9129638671875,
      "loss": 0.4659,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -2.985625743865967,
      "rewards/margins": 1.1087753772735596,
      "rewards/rejected": -4.0944013595581055,
      "step": 290
    },
    {
      "epoch": 0.6338122733986841,
      "grad_norm": 14.026183532380196,
      "learning_rate": 1.7776603444324442e-07,
      "logits/chosen": -21.12714958190918,
      "logits/rejected": -20.915111541748047,
      "logps/chosen": -475.962158203125,
      "logps/rejected": -582.6109008789062,
      "loss": 0.4644,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -2.6778082847595215,
      "rewards/margins": 1.093707799911499,
      "rewards/rejected": -3.7715163230895996,
      "step": 295
    },
    {
      "epoch": 0.6445548543037465,
      "grad_norm": 15.588364098928517,
      "learning_rate": 1.6882513269882913e-07,
      "logits/chosen": -20.675750732421875,
      "logits/rejected": -20.60934829711914,
      "logps/chosen": -463.9253845214844,
      "logps/rejected": -531.9859008789062,
      "loss": 0.4895,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.831554889678955,
      "rewards/margins": 0.7405329942703247,
      "rewards/rejected": -3.572087526321411,
      "step": 300
    },
    {
      "epoch": 0.6552974352088089,
      "grad_norm": 14.269595453262387,
      "learning_rate": 1.5999885023258097e-07,
      "logits/chosen": -21.330524444580078,
      "logits/rejected": -21.496753692626953,
      "logps/chosen": -497.48052978515625,
      "logps/rejected": -608.5450439453125,
      "loss": 0.4797,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -2.899573802947998,
      "rewards/margins": 1.18546724319458,
      "rewards/rejected": -4.085041046142578,
      "step": 305
    },
    {
      "epoch": 0.6660400161138713,
      "grad_norm": 15.040839375830227,
      "learning_rate": 1.5129964979508792e-07,
      "logits/chosen": -21.34568214416504,
      "logits/rejected": -21.257770538330078,
      "logps/chosen": -451.4082946777344,
      "logps/rejected": -546.2640991210938,
      "loss": 0.4786,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -2.661827325820923,
      "rewards/margins": 1.007796049118042,
      "rewards/rejected": -3.669623613357544,
      "step": 310
    },
    {
      "epoch": 0.6767825970189338,
      "grad_norm": 15.632549985107628,
      "learning_rate": 1.4273981469654093e-07,
      "logits/chosen": -21.15200424194336,
      "logits/rejected": -21.08346176147461,
      "logps/chosen": -492.291259765625,
      "logps/rejected": -575.3035888671875,
      "loss": 0.445,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -2.991098403930664,
      "rewards/margins": 0.8909767270088196,
      "rewards/rejected": -3.882075071334839,
      "step": 315
    },
    {
      "epoch": 0.6875251779239963,
      "grad_norm": 14.077151016136973,
      "learning_rate": 1.343314314626449e-07,
      "logits/chosen": -21.24916648864746,
      "logits/rejected": -21.25559425354004,
      "logps/chosen": -479.4493103027344,
      "logps/rejected": -581.2391357421875,
      "loss": 0.4654,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -2.8373022079467773,
      "rewards/margins": 1.0110161304473877,
      "rewards/rejected": -3.848318099975586,
      "step": 320
    },
    {
      "epoch": 0.6982677588290587,
      "grad_norm": 17.9696416705504,
      "learning_rate": 1.2608637276838984e-07,
      "logits/chosen": -20.854616165161133,
      "logits/rejected": -20.866744995117188,
      "logps/chosen": -488.8270568847656,
      "logps/rejected": -580.0584716796875,
      "loss": 0.4757,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -3.009934663772583,
      "rewards/margins": 0.9066306948661804,
      "rewards/rejected": -3.9165656566619873,
      "step": 325
    },
    {
      "epoch": 0.7090103397341211,
      "grad_norm": 17.609393256344518,
      "learning_rate": 1.1801628067378031e-07,
      "logits/chosen": -21.025402069091797,
      "logits/rejected": -21.025859832763672,
      "logps/chosen": -479.85076904296875,
      "logps/rejected": -567.4064331054688,
      "loss": 0.4834,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -2.9113426208496094,
      "rewards/margins": 0.9323997497558594,
      "rewards/rejected": -3.8437423706054688,
      "step": 330
    },
    {
      "epoch": 0.7197529206391836,
      "grad_norm": 14.184878018533347,
      "learning_rate": 1.1013255018519424e-07,
      "logits/chosen": -20.682275772094727,
      "logits/rejected": -20.78445053100586,
      "logps/chosen": -497.1404724121094,
      "logps/rejected": -582.8640747070312,
      "loss": 0.492,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -3.0111889839172363,
      "rewards/margins": 0.9917904138565063,
      "rewards/rejected": -4.002979278564453,
      "step": 335
    },
    {
      "epoch": 0.730495501544246,
      "grad_norm": 19.63828439419386,
      "learning_rate": 1.0244631316558266e-07,
      "logits/chosen": -20.81680679321289,
      "logits/rejected": -20.792213439941406,
      "logps/chosen": -459.59619140625,
      "logps/rejected": -587.2064819335938,
      "loss": 0.4753,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -2.8939058780670166,
      "rewards/margins": 1.304612398147583,
      "rewards/rejected": -4.1985182762146,
      "step": 340
    },
    {
      "epoch": 0.7412380824493084,
      "grad_norm": 14.039431719587414,
      "learning_rate": 9.49684226162292e-08,
      "logits/chosen": -21.063154220581055,
      "logits/rejected": -21.0861873626709,
      "logps/chosen": -486.67181396484375,
      "logps/rejected": -604.0262451171875,
      "loss": 0.4649,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -2.923081874847412,
      "rewards/margins": 1.2030454874038696,
      "rewards/rejected": -4.12612771987915,
      "step": 345
    },
    {
      "epoch": 0.7519806633543709,
      "grad_norm": 13.629114451388812,
      "learning_rate": 8.770943735226302e-08,
      "logits/chosen": -20.562801361083984,
      "logits/rejected": -20.783218383789062,
      "logps/chosen": -468.8253479003906,
      "logps/rejected": -557.6900024414062,
      "loss": 0.476,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -2.7901599407196045,
      "rewards/margins": 0.8934658765792847,
      "rewards/rejected": -3.6836256980895996,
      "step": 350
    },
    {
      "epoch": 0.7627232442594333,
      "grad_norm": 15.10932047374051,
      "learning_rate": 8.067960709356478e-08,
      "logits/chosen": -20.748620986938477,
      "logits/rejected": -20.654150009155273,
      "logps/chosen": -437.6307067871094,
      "logps/rejected": -533.923828125,
      "loss": 0.4898,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -2.6699411869049072,
      "rewards/margins": 1.0374929904937744,
      "rewards/rejected": -3.7074341773986816,
      "step": 355
    },
    {
      "epoch": 0.7734658251644958,
      "grad_norm": 14.97714076175326,
      "learning_rate": 7.388885799211572e-08,
      "logits/chosen": -20.947423934936523,
      "logits/rejected": -20.915830612182617,
      "logps/chosen": -459.2757873535156,
      "logps/rejected": -527.4080200195312,
      "loss": 0.486,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -2.7610130310058594,
      "rewards/margins": 0.6833056211471558,
      "rewards/rejected": -3.4443182945251465,
      "step": 360
    },
    {
      "epoch": 0.7842084060695582,
      "grad_norm": 16.18171761321377,
      "learning_rate": 6.73467786162265e-08,
      "logits/chosen": -21.14028549194336,
      "logits/rejected": -21.127323150634766,
      "logps/chosen": -493.4454650878906,
      "logps/rejected": -586.5130004882812,
      "loss": 0.4738,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -3.0033657550811768,
      "rewards/margins": 0.9621575474739075,
      "rewards/rejected": -3.9655234813690186,
      "step": 365
    },
    {
      "epoch": 0.7949509869746206,
      "grad_norm": 13.115509215618868,
      "learning_rate": 6.106260641143546e-08,
      "logits/chosen": -20.768413543701172,
      "logits/rejected": -20.86481475830078,
      "logps/chosen": -466.8462829589844,
      "logps/rejected": -595.0626220703125,
      "loss": 0.4302,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -2.7288107872009277,
      "rewards/margins": 1.3605787754058838,
      "rewards/rejected": -4.089389801025391,
      "step": 370
    },
    {
      "epoch": 0.805693567879683,
      "grad_norm": 14.69508011494516,
      "learning_rate": 5.5045214657193914e-08,
      "logits/chosen": -20.987638473510742,
      "logits/rejected": -20.953441619873047,
      "logps/chosen": -478.612548828125,
      "logps/rejected": -593.9140014648438,
      "loss": 0.4562,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -2.7526895999908447,
      "rewards/margins": 1.1709507703781128,
      "rewards/rejected": -3.923640489578247,
      "step": 375
    },
    {
      "epoch": 0.8164361487847456,
      "grad_norm": 15.279291329204895,
      "learning_rate": 4.930309993775578e-08,
      "logits/chosen": -20.60492515563965,
      "logits/rejected": -20.59804916381836,
      "logps/chosen": -476.54034423828125,
      "logps/rejected": -611.9324951171875,
      "loss": 0.4789,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -2.751157760620117,
      "rewards/margins": 1.3292760848999023,
      "rewards/rejected": -4.0804338455200195,
      "step": 380
    },
    {
      "epoch": 0.827178729689808,
      "grad_norm": 13.850627883318026,
      "learning_rate": 4.384437014496214e-08,
      "logits/chosen": -20.922626495361328,
      "logits/rejected": -20.863100051879883,
      "logps/chosen": -500.1202697753906,
      "logps/rejected": -598.0996704101562,
      "loss": 0.4577,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -2.9632060527801514,
      "rewards/margins": 0.9760335087776184,
      "rewards/rejected": -3.9392402172088623,
      "step": 385
    },
    {
      "epoch": 0.8379213105948704,
      "grad_norm": 11.603127675586604,
      "learning_rate": 3.8676733029861604e-08,
      "logits/chosen": -20.93654441833496,
      "logits/rejected": -20.852642059326172,
      "logps/chosen": -520.106201171875,
      "logps/rejected": -597.531982421875,
      "loss": 0.4372,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -3.235527515411377,
      "rewards/margins": 0.8415508270263672,
      "rewards/rejected": -4.077078342437744,
      "step": 390
    },
    {
      "epoch": 0.8486638914999328,
      "grad_norm": 18.764205897014417,
      "learning_rate": 3.380748531933103e-08,
      "logits/chosen": -20.87473487854004,
      "logits/rejected": -20.917461395263672,
      "logps/chosen": -531.99365234375,
      "logps/rejected": -618.3016357421875,
      "loss": 0.4429,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -3.4170734882354736,
      "rewards/margins": 0.9161173105239868,
      "rewards/rejected": -4.33319091796875,
      "step": 395
    },
    {
      "epoch": 0.8594064724049953,
      "grad_norm": 14.247674482174348,
      "learning_rate": 2.9243502413064362e-08,
      "logits/chosen": -21.378040313720703,
      "logits/rejected": -21.21609115600586,
      "logps/chosen": -446.82244873046875,
      "logps/rejected": -543.5723876953125,
      "loss": 0.4515,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -2.90150785446167,
      "rewards/margins": 1.0038450956344604,
      "rewards/rejected": -3.905353546142578,
      "step": 400
    },
    {
      "epoch": 0.8701490533100578,
      "grad_norm": 17.314036056734587,
      "learning_rate": 2.4991228675477288e-08,
      "logits/chosen": -20.581409454345703,
      "logits/rejected": -20.422592163085938,
      "logps/chosen": -497.35101318359375,
      "logps/rejected": -586.8109130859375,
      "loss": 0.5112,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -3.250107526779175,
      "rewards/margins": 0.9303083419799805,
      "rewards/rejected": -4.180416107177734,
      "step": 405
    },
    {
      "epoch": 0.8808916342151202,
      "grad_norm": 16.039138434077582,
      "learning_rate": 2.1056668336235623e-08,
      "logits/chosen": -21.51209259033203,
      "logits/rejected": -21.335285186767578,
      "logps/chosen": -479.401611328125,
      "logps/rejected": -569.4190673828125,
      "loss": 0.4632,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -2.925386428833008,
      "rewards/margins": 0.9599825143814087,
      "rewards/rejected": -3.885369062423706,
      "step": 410
    },
    {
      "epoch": 0.8916342151201826,
      "grad_norm": 17.110617421758416,
      "learning_rate": 1.7445377012256123e-08,
      "logits/chosen": -21.36821174621582,
      "logits/rejected": -21.375337600708008,
      "logps/chosen": -483.33367919921875,
      "logps/rejected": -577.9762573242188,
      "loss": 0.465,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -3.0296521186828613,
      "rewards/margins": 0.9877541661262512,
      "rewards/rejected": -4.017406463623047,
      "step": 415
    },
    {
      "epoch": 0.9023767960252451,
      "grad_norm": 13.910969737585662,
      "learning_rate": 1.4162453863150181e-08,
      "logits/chosen": -21.081308364868164,
      "logits/rejected": -21.06283950805664,
      "logps/chosen": -493.18524169921875,
      "logps/rejected": -594.3995361328125,
      "loss": 0.4338,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -2.9149138927459717,
      "rewards/margins": 1.035150408744812,
      "rewards/rejected": -3.9500644207000732,
      "step": 420
    },
    {
      "epoch": 0.9131193769303075,
      "grad_norm": 17.52833661463194,
      "learning_rate": 1.1212534391187689e-08,
      "logits/chosen": -20.905872344970703,
      "logits/rejected": -20.837444305419922,
      "logps/chosen": -438.93048095703125,
      "logps/rejected": -538.4364013671875,
      "loss": 0.4813,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -2.69669246673584,
      "rewards/margins": 0.979594349861145,
      "rewards/rejected": -3.6762871742248535,
      "step": 425
    },
    {
      "epoch": 0.92386195783537,
      "grad_norm": 23.048838465895624,
      "learning_rate": 8.59978389594676e-09,
      "logits/chosen": -20.701757431030273,
      "logits/rejected": -20.72020149230957,
      "logps/chosen": -465.57501220703125,
      "logps/rejected": -549.2703857421875,
      "loss": 0.4546,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -2.8212685585021973,
      "rewards/margins": 0.8496788740158081,
      "rewards/rejected": -3.670947313308716,
      "step": 430
    },
    {
      "epoch": 0.9346045387404324,
      "grad_norm": 15.846431503688864,
      "learning_rate": 6.327891592892126e-09,
      "logits/chosen": -20.888439178466797,
      "logits/rejected": -20.962871551513672,
      "logps/chosen": -478.03790283203125,
      "logps/rejected": -602.384765625,
      "loss": 0.4658,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -2.8073556423187256,
      "rewards/margins": 1.2782353162765503,
      "rewards/rejected": -4.0855913162231445,
      "step": 435
    },
    {
      "epoch": 0.9453471196454948,
      "grad_norm": 14.390183815864935,
      "learning_rate": 4.400065404186276e-09,
      "logits/chosen": -20.80968475341797,
      "logits/rejected": -20.828323364257812,
      "logps/chosen": -491.7540588378906,
      "logps/rejected": -579.70703125,
      "loss": 0.4437,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -2.9315555095672607,
      "rewards/margins": 0.9275356531143188,
      "rewards/rejected": -3.8590915203094482,
      "step": 440
    },
    {
      "epoch": 0.9560897005505573,
      "grad_norm": 16.74015785431614,
      "learning_rate": 2.8190274290888215e-09,
      "logits/chosen": -21.063623428344727,
      "logits/rejected": -21.043384552001953,
      "logps/chosen": -504.7342224121094,
      "logps/rejected": -598.46044921875,
      "loss": 0.4766,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -3.1327033042907715,
      "rewards/margins": 0.9961974024772644,
      "rewards/rejected": -4.128900527954102,
      "step": 445
    },
    {
      "epoch": 0.9668322814556197,
      "grad_norm": 13.546941096732384,
      "learning_rate": 1.5870101003402081e-09,
      "logits/chosen": -20.875934600830078,
      "logits/rejected": -20.832775115966797,
      "logps/chosen": -518.4654541015625,
      "logps/rejected": -618.839111328125,
      "loss": 0.4585,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -3.1628730297088623,
      "rewards/margins": 1.0721877813339233,
      "rewards/rejected": -4.235061168670654,
      "step": 450
    },
    {
      "epoch": 0.9775748623606821,
      "grad_norm": 14.614694225953357,
      "learning_rate": 7.057530319564409e-10,
      "logits/chosen": -20.87482261657715,
      "logits/rejected": -20.740245819091797,
      "logps/chosen": -467.67559814453125,
      "logps/rejected": -538.7994384765625,
      "loss": 0.4798,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -2.8199124336242676,
      "rewards/margins": 0.7901204824447632,
      "rewards/rejected": -3.610032558441162,
      "step": 455
    },
    {
      "epoch": 0.9883174432657446,
      "grad_norm": 12.357402004120742,
      "learning_rate": 1.7650056288651127e-10,
      "logits/chosen": -20.734127044677734,
      "logits/rejected": -20.775203704833984,
      "logps/chosen": -483.3863220214844,
      "logps/rejected": -600.2500610351562,
      "loss": 0.4388,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -3.0567145347595215,
      "rewards/margins": 1.2218314409255981,
      "rewards/rejected": -4.278546333312988,
      "step": 460
    },
    {
      "epoch": 0.9990600241708071,
      "grad_norm": 15.198640734812669,
      "learning_rate": 0.0,
      "logits/chosen": -21.103975296020508,
      "logits/rejected": -21.10438346862793,
      "logps/chosen": -485.390625,
      "logps/rejected": -577.9816284179688,
      "loss": 0.4514,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -2.860701084136963,
      "rewards/margins": 0.9835926294326782,
      "rewards/rejected": -3.8442935943603516,
      "step": 465
    },
    {
      "epoch": 0.9990600241708071,
      "step": 465,
      "total_flos": 0.0,
      "train_loss": 0.5369526706716066,
      "train_runtime": 9531.2046,
      "train_samples_per_second": 6.25,
      "train_steps_per_second": 0.049
    }
  ],
  "logging_steps": 5,
  "max_steps": 465,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 1000000,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": true
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 1,
  "trial_name": null,
  "trial_params": null
}