trainer_state.json

{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 1.0,
  "eval_steps": 500,
  "global_step": 1699,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.0005885815185403178,
      "grad_norm": 32.780718251913974,
      "learning_rate": 2.942907592701589e-10,
      "logits/chosen": -2.603684186935425,
      "logits/rejected": -2.7838211059570312,
      "logps/chosen": -294.7889099121094,
      "logps/rejected": -69.72528076171875,
      "loss": 0.6931,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 1
    },
    {
      "epoch": 0.005885815185403178,
      "grad_norm": 30.421018457904477,
      "learning_rate": 2.9429075927015888e-09,
      "logits/chosen": -2.8964638710021973,
      "logits/rejected": -2.901930809020996,
      "logps/chosen": -331.2449645996094,
      "logps/rejected": -301.0341796875,
      "loss": 0.6933,
      "rewards/accuracies": 0.2222222238779068,
      "rewards/chosen": -0.0001181578409159556,
      "rewards/margins": -0.0006874248501844704,
      "rewards/rejected": 0.0005692669074051082,
      "step": 10
    },
    {
      "epoch": 0.011771630370806356,
      "grad_norm": 39.77348485571547,
      "learning_rate": 5.8858151854031776e-09,
      "logits/chosen": -2.890000581741333,
      "logits/rejected": -2.8872923851013184,
      "logps/chosen": -299.2838439941406,
      "logps/rejected": -242.45925903320312,
      "loss": 0.6928,
      "rewards/accuracies": 0.5333333611488342,
      "rewards/chosen": 0.00012406618043314666,
      "rewards/margins": 0.0003421924775466323,
      "rewards/rejected": -0.0002181262243539095,
      "step": 20
    },
    {
      "epoch": 0.017657445556209534,
      "grad_norm": 37.15226364119724,
      "learning_rate": 8.828722778104766e-09,
      "logits/chosen": -2.871821403503418,
      "logits/rejected": -2.821002721786499,
      "logps/chosen": -275.4599609375,
      "logps/rejected": -248.410888671875,
      "loss": 0.6932,
      "rewards/accuracies": 0.3999999761581421,
      "rewards/chosen": 7.882448699092492e-05,
      "rewards/margins": -0.0011783873196691275,
      "rewards/rejected": 0.0012572116684168577,
      "step": 30
    },
    {
      "epoch": 0.023543260741612712,
      "grad_norm": 28.986298484577823,
      "learning_rate": 1.1771630370806355e-08,
      "logits/chosen": -2.8257861137390137,
      "logits/rejected": -2.782503366470337,
      "logps/chosen": -278.97601318359375,
      "logps/rejected": -273.42205810546875,
      "loss": 0.6937,
      "rewards/accuracies": 0.30000001192092896,
      "rewards/chosen": -0.0018755996134132147,
      "rewards/margins": -0.0013799888547509909,
      "rewards/rejected": -0.0004956105258315802,
      "step": 40
    },
    {
      "epoch": 0.02942907592701589,
      "grad_norm": 28.82026354510354,
      "learning_rate": 1.4714537963507944e-08,
      "logits/chosen": -2.855421543121338,
      "logits/rejected": -2.8377156257629395,
      "logps/chosen": -253.5948944091797,
      "logps/rejected": -274.3114318847656,
      "loss": 0.6937,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.0006353737553581595,
      "rewards/margins": -0.0002903261920437217,
      "rewards/rejected": -0.00034504756331443787,
      "step": 50
    },
    {
      "epoch": 0.03531489111241907,
      "grad_norm": 30.822584821190027,
      "learning_rate": 1.7657445556209532e-08,
      "logits/chosen": -2.824986696243286,
      "logits/rejected": -2.8068385124206543,
      "logps/chosen": -330.98846435546875,
      "logps/rejected": -329.9060974121094,
      "loss": 0.6937,
      "rewards/accuracies": 0.4666667580604553,
      "rewards/chosen": 0.00020488860900513828,
      "rewards/margins": -0.0003379290283191949,
      "rewards/rejected": 0.000542817753739655,
      "step": 60
    },
    {
      "epoch": 0.04120070629782225,
      "grad_norm": 32.409000763227326,
      "learning_rate": 2.0600353148911124e-08,
      "logits/chosen": -2.6729750633239746,
      "logits/rejected": -2.7752492427825928,
      "logps/chosen": -288.5548400878906,
      "logps/rejected": -286.0675964355469,
      "loss": 0.6931,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": -0.0015738470247015357,
      "rewards/margins": -0.001438200124539435,
      "rewards/rejected": -0.0001356468565063551,
      "step": 70
    },
    {
      "epoch": 0.047086521483225424,
      "grad_norm": 28.66094375035891,
      "learning_rate": 2.354326074161271e-08,
      "logits/chosen": -2.8610806465148926,
      "logits/rejected": -2.809129476547241,
      "logps/chosen": -202.50711059570312,
      "logps/rejected": -256.8191833496094,
      "loss": 0.6933,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": -0.0010831685503944755,
      "rewards/margins": -0.0023852435406297445,
      "rewards/rejected": 0.001302075106650591,
      "step": 80
    },
    {
      "epoch": 0.052972336668628606,
      "grad_norm": 31.002618375475574,
      "learning_rate": 2.6486168334314303e-08,
      "logits/chosen": -2.9170703887939453,
      "logits/rejected": -2.8157236576080322,
      "logps/chosen": -280.6930236816406,
      "logps/rejected": -256.5555725097656,
      "loss": 0.6924,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.0004723949823528528,
      "rewards/margins": 0.0002984645252581686,
      "rewards/rejected": 0.00017393044254276901,
      "step": 90
    },
    {
      "epoch": 0.05885815185403178,
      "grad_norm": 35.75851601438966,
      "learning_rate": 2.942907592701589e-08,
      "logits/chosen": -2.882338285446167,
      "logits/rejected": -2.8756601810455322,
      "logps/chosen": -284.1132507324219,
      "logps/rejected": -261.82562255859375,
      "loss": 0.693,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.0010607184376567602,
      "rewards/margins": 0.0015401893761008978,
      "rewards/rejected": -0.0004794707929249853,
      "step": 100
    },
    {
      "epoch": 0.06474396703943497,
      "grad_norm": 26.830290079245987,
      "learning_rate": 3.2371983519717484e-08,
      "logits/chosen": -2.8224923610687256,
      "logits/rejected": -2.786034107208252,
      "logps/chosen": -213.18856811523438,
      "logps/rejected": -209.486572265625,
      "loss": 0.6932,
      "rewards/accuracies": 0.5000000596046448,
      "rewards/chosen": 0.0004384171334095299,
      "rewards/margins": 0.000683752354234457,
      "rewards/rejected": -0.00024533524992875755,
      "step": 110
    },
    {
      "epoch": 0.07062978222483814,
      "grad_norm": 33.59257452712036,
      "learning_rate": 3.5314891112419064e-08,
      "logits/chosen": -2.7929792404174805,
      "logits/rejected": -2.787545919418335,
      "logps/chosen": -296.0240478515625,
      "logps/rejected": -260.3634033203125,
      "loss": 0.6929,
      "rewards/accuracies": 0.4333333373069763,
      "rewards/chosen": 8.877464279066771e-05,
      "rewards/margins": -0.0005403327522799373,
      "rewards/rejected": 0.000629107526037842,
      "step": 120
    },
    {
      "epoch": 0.07651559741024132,
      "grad_norm": 31.167055741749806,
      "learning_rate": 3.8257798705120656e-08,
      "logits/chosen": -2.929863452911377,
      "logits/rejected": -2.8793587684631348,
      "logps/chosen": -192.31356811523438,
      "logps/rejected": -197.66647338867188,
      "loss": 0.6926,
      "rewards/accuracies": 0.5000000596046448,
      "rewards/chosen": 0.001261854893527925,
      "rewards/margins": 0.00017420703079551458,
      "rewards/rejected": 0.0010876477463170886,
      "step": 130
    },
    {
      "epoch": 0.0824014125956445,
      "grad_norm": 25.597516159244545,
      "learning_rate": 4.120070629782225e-08,
      "logits/chosen": -2.818716526031494,
      "logits/rejected": -2.8491830825805664,
      "logps/chosen": -267.6719970703125,
      "logps/rejected": -269.23138427734375,
      "loss": 0.6924,
      "rewards/accuracies": 0.5666667222976685,
      "rewards/chosen": 0.002183220349252224,
      "rewards/margins": 0.0013612338807433844,
      "rewards/rejected": 0.0008219865849241614,
      "step": 140
    },
    {
      "epoch": 0.08828722778104768,
      "grad_norm": 30.754492049125638,
      "learning_rate": 4.414361389052384e-08,
      "logits/chosen": -2.807731866836548,
      "logits/rejected": -2.82275652885437,
      "logps/chosen": -256.2826843261719,
      "logps/rejected": -216.8761749267578,
      "loss": 0.6922,
      "rewards/accuracies": 0.6333333253860474,
      "rewards/chosen": 0.0029171775095164776,
      "rewards/margins": 0.0019290171330794692,
      "rewards/rejected": 0.0009881604928523302,
      "step": 150
    },
    {
      "epoch": 0.09417304296645085,
      "grad_norm": 33.216197696590626,
      "learning_rate": 4.708652148322542e-08,
      "logits/chosen": -2.7577600479125977,
      "logits/rejected": -2.686497449874878,
      "logps/chosen": -271.95977783203125,
      "logps/rejected": -216.9535675048828,
      "loss": 0.6925,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.0015044666361063719,
      "rewards/margins": -0.0006265610572881997,
      "rewards/rejected": 0.0021310276351869106,
      "step": 160
    },
    {
      "epoch": 0.10005885815185403,
      "grad_norm": 31.85341360598595,
      "learning_rate": 5.002942907592701e-08,
      "logits/chosen": -2.860220432281494,
      "logits/rejected": -2.824690580368042,
      "logps/chosen": -229.2205047607422,
      "logps/rejected": -216.49691772460938,
      "loss": 0.6916,
      "rewards/accuracies": 0.7333332896232605,
      "rewards/chosen": 0.005175250582396984,
      "rewards/margins": 0.004574229009449482,
      "rewards/rejected": 0.0006010220386087894,
      "step": 170
    },
    {
      "epoch": 0.10594467333725721,
      "grad_norm": 33.35020421307099,
      "learning_rate": 5.2972336668628605e-08,
      "logits/chosen": -2.8476788997650146,
      "logits/rejected": -2.8345282077789307,
      "logps/chosen": -276.9328918457031,
      "logps/rejected": -297.92138671875,
      "loss": 0.6918,
      "rewards/accuracies": 0.6333333253860474,
      "rewards/chosen": 0.002554533304646611,
      "rewards/margins": 0.002964629326015711,
      "rewards/rejected": -0.0004100958467461169,
      "step": 180
    },
    {
      "epoch": 0.11183048852266039,
      "grad_norm": 27.907028115997573,
      "learning_rate": 5.591524426133019e-08,
      "logits/chosen": -2.8730475902557373,
      "logits/rejected": -2.839439868927002,
      "logps/chosen": -272.5675048828125,
      "logps/rejected": -264.9402770996094,
      "loss": 0.6925,
      "rewards/accuracies": 0.6333333253860474,
      "rewards/chosen": 0.002410594839602709,
      "rewards/margins": 0.000803423929028213,
      "rewards/rejected": 0.0016071706777438521,
      "step": 190
    },
    {
      "epoch": 0.11771630370806356,
      "grad_norm": 29.481219977820295,
      "learning_rate": 5.885815185403178e-08,
      "logits/chosen": -2.9408724308013916,
      "logits/rejected": -2.815699338912964,
      "logps/chosen": -310.2392883300781,
      "logps/rejected": -236.49404907226562,
      "loss": 0.6904,
      "rewards/accuracies": 0.6666666269302368,
      "rewards/chosen": 0.006228293292224407,
      "rewards/margins": 0.005451700650155544,
      "rewards/rejected": 0.0007765936315990984,
      "step": 200
    },
    {
      "epoch": 0.12360211889346674,
      "grad_norm": 35.046475552370886,
      "learning_rate": 6.180105944673337e-08,
      "logits/chosen": -2.6355643272399902,
      "logits/rejected": -2.668175458908081,
      "logps/chosen": -314.09619140625,
      "logps/rejected": -340.42645263671875,
      "loss": 0.6923,
      "rewards/accuracies": 0.5666666626930237,
      "rewards/chosen": 0.010181600227952003,
      "rewards/margins": 0.004080074839293957,
      "rewards/rejected": 0.006101525854319334,
      "step": 210
    },
    {
      "epoch": 0.12948793407886994,
      "grad_norm": 30.183227607657507,
      "learning_rate": 6.474396703943497e-08,
      "logits/chosen": -2.922337293624878,
      "logits/rejected": -2.9165425300598145,
      "logps/chosen": -311.33917236328125,
      "logps/rejected": -271.88800048828125,
      "loss": 0.6911,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": 0.008037504740059376,
      "rewards/margins": 0.006833515129983425,
      "rewards/rejected": 0.0012039883295074105,
      "step": 220
    },
    {
      "epoch": 0.1353737492642731,
      "grad_norm": 28.651320388398528,
      "learning_rate": 6.768687463213654e-08,
      "logits/chosen": -2.764054536819458,
      "logits/rejected": -2.791746139526367,
      "logps/chosen": -342.35638427734375,
      "logps/rejected": -269.53033447265625,
      "loss": 0.6902,
      "rewards/accuracies": 0.8333333730697632,
      "rewards/chosen": 0.011332266964018345,
      "rewards/margins": 0.010512511245906353,
      "rewards/rejected": 0.0008197550778277218,
      "step": 230
    },
    {
      "epoch": 0.14125956444967627,
      "grad_norm": 30.194472525654817,
      "learning_rate": 7.062978222483813e-08,
      "logits/chosen": -2.9200429916381836,
      "logits/rejected": -2.8475396633148193,
      "logps/chosen": -330.425537109375,
      "logps/rejected": -282.93597412109375,
      "loss": 0.6887,
      "rewards/accuracies": 0.7666666507720947,
      "rewards/chosen": 0.015913909301161766,
      "rewards/margins": 0.013330700807273388,
      "rewards/rejected": 0.0025832061655819416,
      "step": 240
    },
    {
      "epoch": 0.14714537963507945,
      "grad_norm": 49.29351755297091,
      "learning_rate": 7.357268981753973e-08,
      "logits/chosen": -2.809460401535034,
      "logits/rejected": -2.847788095474243,
      "logps/chosen": -228.82962036132812,
      "logps/rejected": -260.50067138671875,
      "loss": 0.6886,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.010601197369396687,
      "rewards/margins": 0.0034695893991738558,
      "rewards/rejected": 0.007131607737392187,
      "step": 250
    },
    {
      "epoch": 0.15303119482048264,
      "grad_norm": 35.22606986399358,
      "learning_rate": 7.651559741024131e-08,
      "logits/chosen": -2.829927921295166,
      "logits/rejected": -2.7731313705444336,
      "logps/chosen": -368.4117736816406,
      "logps/rejected": -278.7731018066406,
      "loss": 0.6901,
      "rewards/accuracies": 0.6333333253860474,
      "rewards/chosen": 0.014880454167723656,
      "rewards/margins": 0.008975975215435028,
      "rewards/rejected": 0.00590447848662734,
      "step": 260
    },
    {
      "epoch": 0.15891701000588582,
      "grad_norm": 31.079626356652042,
      "learning_rate": 7.945850500294291e-08,
      "logits/chosen": -2.8230886459350586,
      "logits/rejected": -2.8253014087677,
      "logps/chosen": -246.8427276611328,
      "logps/rejected": -257.07440185546875,
      "loss": 0.6887,
      "rewards/accuracies": 0.6666666269302368,
      "rewards/chosen": 0.011582859791815281,
      "rewards/margins": 0.00953072588890791,
      "rewards/rejected": 0.002052133437246084,
      "step": 270
    },
    {
      "epoch": 0.164802825191289,
      "grad_norm": 34.16689629310999,
      "learning_rate": 8.24014125956445e-08,
      "logits/chosen": -2.790585994720459,
      "logits/rejected": -2.741240978240967,
      "logps/chosen": -194.43751525878906,
      "logps/rejected": -246.21109008789062,
      "loss": 0.6856,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": 0.011640877462923527,
      "rewards/margins": 0.013520603068172932,
      "rewards/rejected": -0.0018797263037413359,
      "step": 280
    },
    {
      "epoch": 0.17068864037669218,
      "grad_norm": 26.650476040593702,
      "learning_rate": 8.534432018834608e-08,
      "logits/chosen": -2.877924680709839,
      "logits/rejected": -2.9167160987854004,
      "logps/chosen": -200.64999389648438,
      "logps/rejected": -239.98623657226562,
      "loss": 0.6864,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.015060337260365486,
      "rewards/margins": 0.004761075135320425,
      "rewards/rejected": 0.0102992607280612,
      "step": 290
    },
    {
      "epoch": 0.17657445556209536,
      "grad_norm": 37.84975273963207,
      "learning_rate": 8.828722778104768e-08,
      "logits/chosen": -2.7102274894714355,
      "logits/rejected": -2.723665475845337,
      "logps/chosen": -241.3960418701172,
      "logps/rejected": -201.9961395263672,
      "loss": 0.6829,
      "rewards/accuracies": 0.7333332896232605,
      "rewards/chosen": 0.023427072912454605,
      "rewards/margins": 0.020558174699544907,
      "rewards/rejected": 0.002868895884603262,
      "step": 300
    },
    {
      "epoch": 0.18246027074749852,
      "grad_norm": 32.582230477501014,
      "learning_rate": 9.123013537374925e-08,
      "logits/chosen": -2.771923303604126,
      "logits/rejected": -2.755645275115967,
      "logps/chosen": -236.4650421142578,
      "logps/rejected": -176.6122589111328,
      "loss": 0.6846,
      "rewards/accuracies": 0.8333331942558289,
      "rewards/chosen": 0.03548317030072212,
      "rewards/margins": 0.026407241821289062,
      "rewards/rejected": 0.00907592847943306,
      "step": 310
    },
    {
      "epoch": 0.1883460859329017,
      "grad_norm": 35.39155087276743,
      "learning_rate": 9.417304296645084e-08,
      "logits/chosen": -2.809967517852783,
      "logits/rejected": -2.7024905681610107,
      "logps/chosen": -267.1817932128906,
      "logps/rejected": -216.0163116455078,
      "loss": 0.6802,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": 0.027452057227492332,
      "rewards/margins": 0.02637614868581295,
      "rewards/rejected": 0.0010759088909253478,
      "step": 320
    },
    {
      "epoch": 0.19423190111830488,
      "grad_norm": 25.00506965748009,
      "learning_rate": 9.711595055915244e-08,
      "logits/chosen": -2.8656439781188965,
      "logits/rejected": -2.797421932220459,
      "logps/chosen": -223.9487762451172,
      "logps/rejected": -238.2415313720703,
      "loss": 0.687,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.018507182598114014,
      "rewards/margins": 0.011660409159958363,
      "rewards/rejected": 0.006846773438155651,
      "step": 330
    },
    {
      "epoch": 0.20011771630370806,
      "grad_norm": 31.48968715365344,
      "learning_rate": 1.0005885815185403e-07,
      "logits/chosen": -2.8663744926452637,
      "logits/rejected": -2.829288959503174,
      "logps/chosen": -227.17904663085938,
      "logps/rejected": -254.73300170898438,
      "loss": 0.6882,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.03473003953695297,
      "rewards/margins": -0.002185368211939931,
      "rewards/rejected": 0.036915406584739685,
      "step": 340
    },
    {
      "epoch": 0.20600353148911124,
      "grad_norm": 33.13341184823187,
      "learning_rate": 1.0300176574455561e-07,
      "logits/chosen": -2.8728482723236084,
      "logits/rejected": -2.849656581878662,
      "logps/chosen": -232.64306640625,
      "logps/rejected": -189.30641174316406,
      "loss": 0.6844,
      "rewards/accuracies": 0.5999999642372131,
      "rewards/chosen": 0.025649067014455795,
      "rewards/margins": 0.01318146288394928,
      "rewards/rejected": 0.01246760506182909,
      "step": 350
    },
    {
      "epoch": 0.21188934667451442,
      "grad_norm": 33.57559926375768,
      "learning_rate": 1.0594467333725721e-07,
      "logits/chosen": -2.94728422164917,
      "logits/rejected": -2.8343241214752197,
      "logps/chosen": -330.82861328125,
      "logps/rejected": -255.1486053466797,
      "loss": 0.6704,
      "rewards/accuracies": 0.7999999523162842,
      "rewards/chosen": 0.060690779238939285,
      "rewards/margins": 0.051527559757232666,
      "rewards/rejected": 0.009163224138319492,
      "step": 360
    },
    {
      "epoch": 0.2177751618599176,
      "grad_norm": 30.406704666801055,
      "learning_rate": 1.088875809299588e-07,
      "logits/chosen": -2.847501039505005,
      "logits/rejected": -2.859219551086426,
      "logps/chosen": -226.4930877685547,
      "logps/rejected": -411.73291015625,
      "loss": 0.6794,
      "rewards/accuracies": 0.7666666507720947,
      "rewards/chosen": 0.029315758496522903,
      "rewards/margins": 0.03848721459507942,
      "rewards/rejected": -0.009171457961201668,
      "step": 370
    },
    {
      "epoch": 0.22366097704532079,
      "grad_norm": 32.000980950184385,
      "learning_rate": 1.1183048852266038e-07,
      "logits/chosen": -2.750246047973633,
      "logits/rejected": -2.683522939682007,
      "logps/chosen": -312.06695556640625,
      "logps/rejected": -251.6304931640625,
      "loss": 0.6762,
      "rewards/accuracies": 0.6666666865348816,
      "rewards/chosen": 0.029916947707533836,
      "rewards/margins": 0.033031001687049866,
      "rewards/rejected": -0.003114058170467615,
      "step": 380
    },
    {
      "epoch": 0.22954679223072397,
      "grad_norm": 34.96067667192858,
      "learning_rate": 1.1477339611536198e-07,
      "logits/chosen": -2.855989694595337,
      "logits/rejected": -2.815114974975586,
      "logps/chosen": -262.18450927734375,
      "logps/rejected": -189.57838439941406,
      "loss": 0.6824,
      "rewards/accuracies": 0.7333333492279053,
      "rewards/chosen": 0.025654960423707962,
      "rewards/margins": 0.0397619903087616,
      "rewards/rejected": -0.01410702895373106,
      "step": 390
    },
    {
      "epoch": 0.23543260741612712,
      "grad_norm": 30.784886423492903,
      "learning_rate": 1.1771630370806355e-07,
      "logits/chosen": -2.8241119384765625,
      "logits/rejected": -2.836333751678467,
      "logps/chosen": -233.1301727294922,
      "logps/rejected": -277.805419921875,
      "loss": 0.6805,
      "rewards/accuracies": 0.7333332896232605,
      "rewards/chosen": 0.03232261911034584,
      "rewards/margins": 0.03198341652750969,
      "rewards/rejected": 0.00033920182613655925,
      "step": 400
    },
    {
      "epoch": 0.2413184226015303,
      "grad_norm": 27.065262812921755,
      "learning_rate": 1.2065921130076514e-07,
      "logits/chosen": -2.756892442703247,
      "logits/rejected": -2.6843674182891846,
      "logps/chosen": -287.8220520019531,
      "logps/rejected": -222.5053253173828,
      "loss": 0.67,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": 0.03421301394701004,
      "rewards/margins": 0.054109979420900345,
      "rewards/rejected": -0.019896967336535454,
      "step": 410
    },
    {
      "epoch": 0.24720423778693348,
      "grad_norm": 33.86568434240492,
      "learning_rate": 1.2360211889346674e-07,
      "logits/chosen": -2.8926842212677,
      "logits/rejected": -2.817612648010254,
      "logps/chosen": -268.4040222167969,
      "logps/rejected": -255.06655883789062,
      "loss": 0.6698,
      "rewards/accuracies": 0.6666666865348816,
      "rewards/chosen": 0.014465409331023693,
      "rewards/margins": 0.02410224825143814,
      "rewards/rejected": -0.009636839851737022,
      "step": 420
    },
    {
      "epoch": 0.25309005297233667,
      "grad_norm": 30.66902266053471,
      "learning_rate": 1.2654502648616834e-07,
      "logits/chosen": -2.7713985443115234,
      "logits/rejected": -2.76814341545105,
      "logps/chosen": -209.8004913330078,
      "logps/rejected": -219.0537567138672,
      "loss": 0.6823,
      "rewards/accuracies": 0.7333332896232605,
      "rewards/chosen": 0.0189804807305336,
      "rewards/margins": 0.026658236980438232,
      "rewards/rejected": -0.007677753455936909,
      "step": 430
    },
    {
      "epoch": 0.2589758681577399,
      "grad_norm": 37.33663950349735,
      "learning_rate": 1.2948793407886994e-07,
      "logits/chosen": -2.762784481048584,
      "logits/rejected": -2.707094669342041,
      "logps/chosen": -250.97177124023438,
      "logps/rejected": -232.8260040283203,
      "loss": 0.6651,
      "rewards/accuracies": 0.7000000476837158,
      "rewards/chosen": 0.03512561693787575,
      "rewards/margins": 0.09343008697032928,
      "rewards/rejected": -0.05830447003245354,
      "step": 440
    },
    {
      "epoch": 0.26486168334314303,
      "grad_norm": 38.55541735391845,
      "learning_rate": 1.324308416715715e-07,
      "logits/chosen": -2.7505548000335693,
      "logits/rejected": -2.790361166000366,
      "logps/chosen": -261.1943664550781,
      "logps/rejected": -230.3916473388672,
      "loss": 0.6643,
      "rewards/accuracies": 0.7666667103767395,
      "rewards/chosen": 0.03301350399851799,
      "rewards/margins": 0.08106649667024612,
      "rewards/rejected": -0.04805298522114754,
      "step": 450
    },
    {
      "epoch": 0.2707474985285462,
      "grad_norm": 30.70708391983573,
      "learning_rate": 1.3537374926427308e-07,
      "logits/chosen": -2.782029390335083,
      "logits/rejected": -2.710545539855957,
      "logps/chosen": -238.57211303710938,
      "logps/rejected": -251.1454315185547,
      "loss": 0.6818,
      "rewards/accuracies": 0.5666666626930237,
      "rewards/chosen": -0.005198891274631023,
      "rewards/margins": 0.015401232056319714,
      "rewards/rejected": -0.020600125193595886,
      "step": 460
    },
    {
      "epoch": 0.2766333137139494,
      "grad_norm": 33.06918545223034,
      "learning_rate": 1.3831665685697468e-07,
      "logits/chosen": -2.7734148502349854,
      "logits/rejected": -2.722306728363037,
      "logps/chosen": -236.19290161132812,
      "logps/rejected": -218.6464080810547,
      "loss": 0.6545,
      "rewards/accuracies": 0.5666666030883789,
      "rewards/chosen": 0.004786822013556957,
      "rewards/margins": 0.04201732203364372,
      "rewards/rejected": -0.03723050281405449,
      "step": 470
    },
    {
      "epoch": 0.28251912889935255,
      "grad_norm": 34.67042804510441,
      "learning_rate": 1.4125956444967625e-07,
      "logits/chosen": -2.847014904022217,
      "logits/rejected": -2.759166717529297,
      "logps/chosen": -291.9137878417969,
      "logps/rejected": -247.39566040039062,
      "loss": 0.6612,
      "rewards/accuracies": 0.46666669845581055,
      "rewards/chosen": -0.01318571250885725,
      "rewards/margins": 0.02275732532143593,
      "rewards/rejected": -0.03594304248690605,
      "step": 480
    },
    {
      "epoch": 0.28840494408475575,
      "grad_norm": 41.62165688207727,
      "learning_rate": 1.4420247204237788e-07,
      "logits/chosen": -2.8126304149627686,
      "logits/rejected": -2.8055615425109863,
      "logps/chosen": -289.50091552734375,
      "logps/rejected": -277.8962707519531,
      "loss": 0.6614,
      "rewards/accuracies": 0.6333333253860474,
      "rewards/chosen": 0.020298918709158897,
      "rewards/margins": 0.0497608408331871,
      "rewards/rejected": -0.02946191094815731,
      "step": 490
    },
    {
      "epoch": 0.2942907592701589,
      "grad_norm": 36.97324400383161,
      "learning_rate": 1.4714537963507945e-07,
      "logits/chosen": -2.885690212249756,
      "logits/rejected": -2.8348922729492188,
      "logps/chosen": -307.3095703125,
      "logps/rejected": -290.29180908203125,
      "loss": 0.6509,
      "rewards/accuracies": 0.7333332896232605,
      "rewards/chosen": 0.024543384090065956,
      "rewards/margins": 0.09910109639167786,
      "rewards/rejected": -0.07455770671367645,
      "step": 500
    },
    {
      "epoch": 0.3001765744555621,
      "grad_norm": 36.070076135435244,
      "learning_rate": 1.5008828722778105e-07,
      "logits/chosen": -2.842216968536377,
      "logits/rejected": -2.820446491241455,
      "logps/chosen": -256.1488342285156,
      "logps/rejected": -247.92672729492188,
      "loss": 0.6638,
      "rewards/accuracies": 0.6333333253860474,
      "rewards/chosen": -0.0209733247756958,
      "rewards/margins": 0.04726904630661011,
      "rewards/rejected": -0.0682423785328865,
      "step": 510
    },
    {
      "epoch": 0.30606238964096527,
      "grad_norm": 42.70438629438365,
      "learning_rate": 1.5303119482048262e-07,
      "logits/chosen": -2.6641478538513184,
      "logits/rejected": -2.66444993019104,
      "logps/chosen": -259.26348876953125,
      "logps/rejected": -269.11572265625,
      "loss": 0.6363,
      "rewards/accuracies": 0.6666666269302368,
      "rewards/chosen": -0.010639377869665623,
      "rewards/margins": 0.09347447007894516,
      "rewards/rejected": -0.10411383956670761,
      "step": 520
    },
    {
      "epoch": 0.3119482048263684,
      "grad_norm": 35.960710299811815,
      "learning_rate": 1.559741024131842e-07,
      "logits/chosen": -2.805729866027832,
      "logits/rejected": -2.7076404094696045,
      "logps/chosen": -255.88052368164062,
      "logps/rejected": -234.35415649414062,
      "loss": 0.6335,
      "rewards/accuracies": 0.6666666269302368,
      "rewards/chosen": -0.007552222814410925,
      "rewards/margins": 0.11313612759113312,
      "rewards/rejected": -0.12068834155797958,
      "step": 530
    },
    {
      "epoch": 0.31783402001177163,
      "grad_norm": 44.82672193345242,
      "learning_rate": 1.5891701000588582e-07,
      "logits/chosen": -2.8400981426239014,
      "logits/rejected": -2.8063666820526123,
      "logps/chosen": -294.94451904296875,
      "logps/rejected": -278.1987609863281,
      "loss": 0.6372,
      "rewards/accuracies": 0.6666666269302368,
      "rewards/chosen": -0.08216958492994308,
      "rewards/margins": 0.09824585914611816,
      "rewards/rejected": -0.18041543662548065,
      "step": 540
    },
    {
      "epoch": 0.3237198351971748,
      "grad_norm": 35.13933484391953,
      "learning_rate": 1.618599175985874e-07,
      "logits/chosen": -2.7300307750701904,
      "logits/rejected": -2.7713351249694824,
      "logps/chosen": -230.0304718017578,
      "logps/rejected": -262.08721923828125,
      "loss": 0.6309,
      "rewards/accuracies": 0.7333332896232605,
      "rewards/chosen": -0.08895792812108994,
      "rewards/margins": 0.13027846813201904,
      "rewards/rejected": -0.2192363739013672,
      "step": 550
    },
    {
      "epoch": 0.329605650382578,
      "grad_norm": 52.54257576269853,
      "learning_rate": 1.64802825191289e-07,
      "logits/chosen": -2.753162145614624,
      "logits/rejected": -2.768341541290283,
      "logps/chosen": -252.78878784179688,
      "logps/rejected": -274.1339111328125,
      "loss": 0.6457,
      "rewards/accuracies": 0.7333333492279053,
      "rewards/chosen": -0.1532047688961029,
      "rewards/margins": 0.1562652289867401,
      "rewards/rejected": -0.309469997882843,
      "step": 560
    },
    {
      "epoch": 0.33549146556798115,
      "grad_norm": 38.27091133745243,
      "learning_rate": 1.6774573278399057e-07,
      "logits/chosen": -2.804091215133667,
      "logits/rejected": -2.776109457015991,
      "logps/chosen": -287.103271484375,
      "logps/rejected": -222.2604217529297,
      "loss": 0.6418,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.04693541303277016,
      "rewards/margins": 0.12523198127746582,
      "rewards/rejected": -0.17216739058494568,
      "step": 570
    },
    {
      "epoch": 0.34137728075338436,
      "grad_norm": 46.61649976864576,
      "learning_rate": 1.7068864037669217e-07,
      "logits/chosen": -2.802682399749756,
      "logits/rejected": -2.7890288829803467,
      "logps/chosen": -285.7334899902344,
      "logps/rejected": -294.05181884765625,
      "loss": 0.6359,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.16137059032917023,
      "rewards/margins": 0.1200031265616417,
      "rewards/rejected": -0.2813737392425537,
      "step": 580
    },
    {
      "epoch": 0.3472630959387875,
      "grad_norm": 38.73511849094784,
      "learning_rate": 1.7363154796939374e-07,
      "logits/chosen": -2.7510268688201904,
      "logits/rejected": -2.6245083808898926,
      "logps/chosen": -265.32757568359375,
      "logps/rejected": -240.0128631591797,
      "loss": 0.6388,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.2014482468366623,
      "rewards/margins": 0.0063117193058133125,
      "rewards/rejected": -0.20775997638702393,
      "step": 590
    },
    {
      "epoch": 0.3531489111241907,
      "grad_norm": 33.48738056386425,
      "learning_rate": 1.7657445556209536e-07,
      "logits/chosen": -2.8242006301879883,
      "logits/rejected": -2.828608274459839,
      "logps/chosen": -249.8431396484375,
      "logps/rejected": -261.53948974609375,
      "loss": 0.6199,
      "rewards/accuracies": 0.6333333253860474,
      "rewards/chosen": -0.0753629207611084,
      "rewards/margins": 0.17823639512062073,
      "rewards/rejected": -0.25359928607940674,
      "step": 600
    },
    {
      "epoch": 0.3590347263095939,
      "grad_norm": 44.06827645574314,
      "learning_rate": 1.7951736315479694e-07,
      "logits/chosen": -2.652758836746216,
      "logits/rejected": -2.7120182514190674,
      "logps/chosen": -283.7317810058594,
      "logps/rejected": -316.36004638671875,
      "loss": 0.6189,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.2105875015258789,
      "rewards/margins": 0.14809870719909668,
      "rewards/rejected": -0.35868626832962036,
      "step": 610
    },
    {
      "epoch": 0.36492054149499703,
      "grad_norm": 77.88854058346173,
      "learning_rate": 1.824602707474985e-07,
      "logits/chosen": -2.792132616043091,
      "logits/rejected": -2.735396146774292,
      "logps/chosen": -308.66094970703125,
      "logps/rejected": -297.17120361328125,
      "loss": 0.615,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.38594353199005127,
      "rewards/margins": 0.13582220673561096,
      "rewards/rejected": -0.5217657685279846,
      "step": 620
    },
    {
      "epoch": 0.37080635668040024,
      "grad_norm": 51.14005626306302,
      "learning_rate": 1.854031783402001e-07,
      "logits/chosen": -2.6652121543884277,
      "logits/rejected": -2.6964430809020996,
      "logps/chosen": -246.6635284423828,
      "logps/rejected": -252.4509735107422,
      "loss": 0.6434,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.09110568463802338,
      "rewards/margins": 0.1668696105480194,
      "rewards/rejected": -0.2579752802848816,
      "step": 630
    },
    {
      "epoch": 0.3766921718658034,
      "grad_norm": 51.12058774641213,
      "learning_rate": 1.8834608593290168e-07,
      "logits/chosen": -2.837921380996704,
      "logits/rejected": -2.8364148139953613,
      "logps/chosen": -307.8941955566406,
      "logps/rejected": -311.6710205078125,
      "loss": 0.6426,
      "rewards/accuracies": 0.5333333611488342,
      "rewards/chosen": -0.19721655547618866,
      "rewards/margins": 0.042520731687545776,
      "rewards/rejected": -0.23973730206489563,
      "step": 640
    },
    {
      "epoch": 0.3825779870512066,
      "grad_norm": 55.28695481091037,
      "learning_rate": 1.9128899352560328e-07,
      "logits/chosen": -2.7750084400177,
      "logits/rejected": -2.816450595855713,
      "logps/chosen": -338.82080078125,
      "logps/rejected": -358.87921142578125,
      "loss": 0.6245,
      "rewards/accuracies": 0.6666666269302368,
      "rewards/chosen": -0.21482959389686584,
      "rewards/margins": 0.2418464869260788,
      "rewards/rejected": -0.45667609572410583,
      "step": 650
    },
    {
      "epoch": 0.38846380223660976,
      "grad_norm": 60.1584611934493,
      "learning_rate": 1.9423190111830488e-07,
      "logits/chosen": -2.808283805847168,
      "logits/rejected": -2.733508586883545,
      "logps/chosen": -268.6055908203125,
      "logps/rejected": -279.4296569824219,
      "loss": 0.6244,
      "rewards/accuracies": 0.5666666626930237,
      "rewards/chosen": -0.24825648963451385,
      "rewards/margins": 0.07473016530275345,
      "rewards/rejected": -0.3229866623878479,
      "step": 660
    },
    {
      "epoch": 0.39434961742201297,
      "grad_norm": 38.16884117276723,
      "learning_rate": 1.9717480871100648e-07,
      "logits/chosen": -2.763615131378174,
      "logits/rejected": -2.742448329925537,
      "logps/chosen": -214.5966796875,
      "logps/rejected": -293.02313232421875,
      "loss": 0.6017,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.1877942830324173,
      "rewards/margins": 0.23820248246192932,
      "rewards/rejected": -0.42599672079086304,
      "step": 670
    },
    {
      "epoch": 0.4002354326074161,
      "grad_norm": 40.282667153165065,
      "learning_rate": 2.0011771630370805e-07,
      "logits/chosen": -2.7557873725891113,
      "logits/rejected": -2.777015209197998,
      "logps/chosen": -384.47186279296875,
      "logps/rejected": -303.30877685546875,
      "loss": 0.6045,
      "rewards/accuracies": 0.7333332896232605,
      "rewards/chosen": -0.15859545767307281,
      "rewards/margins": 0.22654417157173157,
      "rewards/rejected": -0.3851396441459656,
      "step": 680
    },
    {
      "epoch": 0.40612124779281933,
      "grad_norm": 50.42522303402139,
      "learning_rate": 2.0306062389640965e-07,
      "logits/chosen": -2.7922425270080566,
      "logits/rejected": -2.707688093185425,
      "logps/chosen": -314.8849792480469,
      "logps/rejected": -246.93618774414062,
      "loss": 0.6023,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.20941126346588135,
      "rewards/margins": 0.2189422845840454,
      "rewards/rejected": -0.42835354804992676,
      "step": 690
    },
    {
      "epoch": 0.4120070629782225,
      "grad_norm": 86.59616940333294,
      "learning_rate": 2.0600353148911122e-07,
      "logits/chosen": -2.7496159076690674,
      "logits/rejected": -2.7615320682525635,
      "logps/chosen": -316.3775939941406,
      "logps/rejected": -316.27752685546875,
      "loss": 0.6312,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.4660351276397705,
      "rewards/margins": 0.19507920742034912,
      "rewards/rejected": -0.6611142754554749,
      "step": 700
    },
    {
      "epoch": 0.41789287816362564,
      "grad_norm": 113.88657488515956,
      "learning_rate": 3.61272271440356e-07,
      "logits/chosen": -2.713832139968872,
      "logits/rejected": -2.639629364013672,
      "logps/chosen": -292.7522277832031,
      "logps/rejected": -276.9837951660156,
      "loss": 0.6111,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.42747750878334045,
      "rewards/margins": 0.30797505378723145,
      "rewards/rejected": -0.7354525923728943,
      "step": 710
    },
    {
      "epoch": 0.42377869334902885,
      "grad_norm": 58.276303041068935,
      "learning_rate": 3.5664928281860625e-07,
      "logits/chosen": -2.7681069374084473,
      "logits/rejected": -2.6781115531921387,
      "logps/chosen": -293.0718078613281,
      "logps/rejected": -281.61212158203125,
      "loss": 0.5922,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.3552438020706177,
      "rewards/margins": 0.20946629345417023,
      "rewards/rejected": -0.5647100806236267,
      "step": 720
    },
    {
      "epoch": 0.429664508534432,
      "grad_norm": 34.26366533936239,
      "learning_rate": 3.5198127192199e-07,
      "logits/chosen": -2.7726988792419434,
      "logits/rejected": -2.7980246543884277,
      "logps/chosen": -358.5955810546875,
      "logps/rejected": -417.20281982421875,
      "loss": 0.5593,
      "rewards/accuracies": 0.8333333134651184,
      "rewards/chosen": -0.27696719765663147,
      "rewards/margins": 0.5634738802909851,
      "rewards/rejected": -0.8404410481452942,
      "step": 730
    },
    {
      "epoch": 0.4355503237198352,
      "grad_norm": 48.29459945401339,
      "learning_rate": 3.472702093635682e-07,
      "logits/chosen": -2.822880268096924,
      "logits/rejected": -2.754981279373169,
      "logps/chosen": -232.97921752929688,
      "logps/rejected": -225.82296752929688,
      "loss": 0.6444,
      "rewards/accuracies": 0.5333333015441895,
      "rewards/chosen": -0.568128228187561,
      "rewards/margins": -0.030178595334291458,
      "rewards/rejected": -0.5379496216773987,
      "step": 740
    },
    {
      "epoch": 0.44143613890523836,
      "grad_norm": 88.73737377845674,
      "learning_rate": 3.42518083930774e-07,
      "logits/chosen": -2.6345629692077637,
      "logits/rejected": -2.6091387271881104,
      "logps/chosen": -272.2128601074219,
      "logps/rejected": -344.5741882324219,
      "loss": 0.6375,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.46700435876846313,
      "rewards/margins": 0.42694124579429626,
      "rewards/rejected": -0.8939456939697266,
      "step": 750
    },
    {
      "epoch": 0.44732195409064157,
      "grad_norm": 74.5070307164555,
      "learning_rate": 3.377269017458409e-07,
      "logits/chosen": -2.767118453979492,
      "logits/rejected": -2.7667033672332764,
      "logps/chosen": -392.59454345703125,
      "logps/rejected": -410.0496520996094,
      "loss": 0.5401,
      "rewards/accuracies": 0.7666666507720947,
      "rewards/chosen": -0.3810432553291321,
      "rewards/margins": 0.6425568461418152,
      "rewards/rejected": -1.0236002206802368,
      "step": 760
    },
    {
      "epoch": 0.4532077692760447,
      "grad_norm": 102.79976551674226,
      "learning_rate": 3.3289868541891255e-07,
      "logits/chosen": -2.604182720184326,
      "logits/rejected": -2.527705430984497,
      "logps/chosen": -296.3623046875,
      "logps/rejected": -354.95819091796875,
      "loss": 0.6152,
      "rewards/accuracies": 0.6666666865348816,
      "rewards/chosen": -0.5251516103744507,
      "rewards/margins": 0.46194180846214294,
      "rewards/rejected": -0.9870933294296265,
      "step": 770
    },
    {
      "epoch": 0.45909358446144793,
      "grad_norm": 67.73780235887719,
      "learning_rate": 3.280354731941932e-07,
      "logits/chosen": -2.76226544380188,
      "logits/rejected": -2.6936070919036865,
      "logps/chosen": -315.97216796875,
      "logps/rejected": -342.60546875,
      "loss": 0.5639,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.4423181116580963,
      "rewards/margins": 0.47899603843688965,
      "rewards/rejected": -0.9213140606880188,
      "step": 780
    },
    {
      "epoch": 0.4649793996468511,
      "grad_norm": 38.93430396007463,
      "learning_rate": 3.231393180894973e-07,
      "logits/chosen": -2.795037031173706,
      "logits/rejected": -2.7878684997558594,
      "logps/chosen": -255.4384765625,
      "logps/rejected": -313.3453369140625,
      "loss": 0.5453,
      "rewards/accuracies": 0.7333332896232605,
      "rewards/chosen": -0.3891940712928772,
      "rewards/margins": 0.5739094614982605,
      "rewards/rejected": -0.9631035923957825,
      "step": 790
    },
    {
      "epoch": 0.47086521483225424,
      "grad_norm": 78.21647434951635,
      "learning_rate": 3.1821228702956253e-07,
      "logits/chosen": -2.5857443809509277,
      "logits/rejected": -2.5272181034088135,
      "logps/chosen": -269.7416076660156,
      "logps/rejected": -359.6760559082031,
      "loss": 0.6535,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.6391388773918152,
      "rewards/margins": 0.2941351532936096,
      "rewards/rejected": -0.9332740902900696,
      "step": 800
    },
    {
      "epoch": 0.47675103001765745,
      "grad_norm": 59.73596890978745,
      "learning_rate": 3.132564599734921e-07,
      "logits/chosen": -2.7386457920074463,
      "logits/rejected": -2.754356861114502,
      "logps/chosen": -292.2206726074219,
      "logps/rejected": -324.42816162109375,
      "loss": 0.6368,
      "rewards/accuracies": 0.6333333253860474,
      "rewards/chosen": -0.3809540867805481,
      "rewards/margins": 0.2144467830657959,
      "rewards/rejected": -0.5954009294509888,
      "step": 810
    },
    {
      "epoch": 0.4826368452030606,
      "grad_norm": 58.04453100186318,
      "learning_rate": 3.082739290366946e-07,
      "logits/chosen": -2.731048107147217,
      "logits/rejected": -2.7188940048217773,
      "logps/chosen": -367.1738586425781,
      "logps/rejected": -365.79803466796875,
      "loss": 0.5855,
      "rewards/accuracies": 0.7333333492279053,
      "rewards/chosen": -0.40680187940597534,
      "rewards/margins": 0.5839999318122864,
      "rewards/rejected": -0.9908016920089722,
      "step": 820
    },
    {
      "epoch": 0.4885226603884638,
      "grad_norm": 60.703085038442914,
      "learning_rate": 3.0326679760769226e-07,
      "logits/chosen": -2.633038282394409,
      "logits/rejected": -2.6379826068878174,
      "logps/chosen": -270.77972412109375,
      "logps/rejected": -361.93170166015625,
      "loss": 0.6076,
      "rewards/accuracies": 0.8333333730697632,
      "rewards/chosen": -0.29258260130882263,
      "rewards/margins": 0.578423798084259,
      "rewards/rejected": -0.8710063695907593,
      "step": 830
    },
    {
      "epoch": 0.49440847557386697,
      "grad_norm": 68.79606387966484,
      "learning_rate": 2.982371794601694e-07,
      "logits/chosen": -2.7220444679260254,
      "logits/rejected": -2.665432929992676,
      "logps/chosen": -456.65557861328125,
      "logps/rejected": -338.92926025390625,
      "loss": 0.6101,
      "rewards/accuracies": 0.7333333492279053,
      "rewards/chosen": -0.38200992345809937,
      "rewards/margins": 0.4571179747581482,
      "rewards/rejected": -0.8391278386116028,
      "step": 840
    },
    {
      "epoch": 0.5002942907592701,
      "grad_norm": 39.349812488010386,
      "learning_rate": 2.931871978606379e-07,
      "logits/chosen": -2.7449920177459717,
      "logits/rejected": -2.785231590270996,
      "logps/chosen": -353.2920837402344,
      "logps/rejected": -434.22979736328125,
      "loss": 0.5017,
      "rewards/accuracies": 0.7666666507720947,
      "rewards/chosen": -0.3858256936073303,
      "rewards/margins": 0.6933655738830566,
      "rewards/rejected": -1.0791910886764526,
      "step": 850
    },
    {
      "epoch": 0.5061801059446733,
      "grad_norm": 65.24381800745265,
      "learning_rate": 2.881189846720941e-07,
      "logits/chosen": -2.7466366291046143,
      "logits/rejected": -2.7046473026275635,
      "logps/chosen": -334.4920959472656,
      "logps/rejected": -363.40838623046875,
      "loss": 0.5525,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.41190671920776367,
      "rewards/margins": 0.559307873249054,
      "rewards/rejected": -0.9712145924568176,
      "step": 860
    },
    {
      "epoch": 0.5120659211300765,
      "grad_norm": 51.09800858702452,
      "learning_rate": 2.830346794540479e-07,
      "logits/chosen": -2.6742959022521973,
      "logits/rejected": -2.6640892028808594,
      "logps/chosen": -266.5992736816406,
      "logps/rejected": -313.98828125,
      "loss": 0.5778,
      "rewards/accuracies": 0.4999999403953552,
      "rewards/chosen": -0.6418452262878418,
      "rewards/margins": 0.16775313019752502,
      "rewards/rejected": -0.8095983266830444,
      "step": 870
    },
    {
      "epoch": 0.5179517363154797,
      "grad_norm": 54.983291652847775,
      "learning_rate": 2.7793642855930133e-07,
      "logits/chosen": -2.7972800731658936,
      "logits/rejected": -2.657902240753174,
      "logps/chosen": -379.5608825683594,
      "logps/rejected": -415.2431640625,
      "loss": 0.6336,
      "rewards/accuracies": 0.76666659116745,
      "rewards/chosen": -0.5884305834770203,
      "rewards/margins": 0.4619186818599701,
      "rewards/rejected": -1.0503493547439575,
      "step": 880
    },
    {
      "epoch": 0.5238375515008828,
      "grad_norm": 52.1793743861904,
      "learning_rate": 2.728263842278608e-07,
      "logits/chosen": -2.7541468143463135,
      "logits/rejected": -2.7553248405456543,
      "logps/chosen": -340.1725158691406,
      "logps/rejected": -431.4141540527344,
      "loss": 0.5083,
      "rewards/accuracies": 0.8333333134651184,
      "rewards/chosen": -0.5956707000732422,
      "rewards/margins": 0.6665983200073242,
      "rewards/rejected": -1.2622690200805664,
      "step": 890
    },
    {
      "epoch": 0.5297233666862861,
      "grad_norm": 62.43287153719138,
      "learning_rate": 2.6770670367836276e-07,
      "logits/chosen": -2.731095790863037,
      "logits/rejected": -2.734687328338623,
      "logps/chosen": -333.5868835449219,
      "logps/rejected": -405.0290222167969,
      "loss": 0.5372,
      "rewards/accuracies": 0.7666667103767395,
      "rewards/chosen": -0.7237066030502319,
      "rewards/margins": 0.5302045941352844,
      "rewards/rejected": -1.2539112567901611,
      "step": 900
    },
    {
      "epoch": 0.5356091818716893,
      "grad_norm": 82.01285085767707,
      "learning_rate": 2.625795481973985e-07,
      "logits/chosen": -2.6702892780303955,
      "logits/rejected": -2.5133378505706787,
      "logps/chosen": -338.619873046875,
      "logps/rejected": -311.3938903808594,
      "loss": 0.5897,
      "rewards/accuracies": 0.6333333253860474,
      "rewards/chosen": -0.5718550086021423,
      "rewards/margins": 0.4375828206539154,
      "rewards/rejected": -1.0094377994537354,
      "step": 910
    },
    {
      "epoch": 0.5414949970570924,
      "grad_norm": 95.54109435400643,
      "learning_rate": 2.5744708222712137e-07,
      "logits/chosen": -2.7311172485351562,
      "logits/rejected": -2.739924907684326,
      "logps/chosen": -272.01995849609375,
      "logps/rejected": -311.47833251953125,
      "loss": 0.5762,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.5505859851837158,
      "rewards/margins": 0.6296683549880981,
      "rewards/rejected": -1.180254340171814,
      "step": 920
    },
    {
      "epoch": 0.5473808122424956,
      "grad_norm": 36.48587725082776,
      "learning_rate": 2.5231147245152156e-07,
      "logits/chosen": -2.4503333568573,
      "logits/rejected": -2.468564033508301,
      "logps/chosen": -298.019775390625,
      "logps/rejected": -372.83026123046875,
      "loss": 0.5587,
      "rewards/accuracies": 0.7666666507720947,
      "rewards/chosen": -0.6832865476608276,
      "rewards/margins": 0.8449433445930481,
      "rewards/rejected": -1.5282299518585205,
      "step": 930
    },
    {
      "epoch": 0.5532666274278988,
      "grad_norm": 71.24789616161829,
      "learning_rate": 2.471748868817551e-07,
      "logits/chosen": -2.7247791290283203,
      "logits/rejected": -2.7448885440826416,
      "logps/chosen": -316.38726806640625,
      "logps/rejected": -418.75830078125,
      "loss": 0.5429,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.43857964873313904,
      "rewards/margins": 0.8119268417358398,
      "rewards/rejected": -1.2505066394805908,
      "step": 940
    },
    {
      "epoch": 0.559152442613302,
      "grad_norm": 57.37226759319327,
      "learning_rate": 2.4203949394091186e-07,
      "logits/chosen": -2.7133288383483887,
      "logits/rejected": -2.6669368743896484,
      "logps/chosen": -384.8581848144531,
      "logps/rejected": -477.08538818359375,
      "loss": 0.5152,
      "rewards/accuracies": 0.7999999523162842,
      "rewards/chosen": -0.9175896644592285,
      "rewards/margins": 1.0064847469329834,
      "rewards/rejected": -1.9240741729736328,
      "step": 950
    },
    {
      "epoch": 0.5650382577987051,
      "grad_norm": 102.83287628895852,
      "learning_rate": 2.3690746154861048e-07,
      "logits/chosen": -2.5697152614593506,
      "logits/rejected": -2.5663580894470215,
      "logps/chosen": -361.67926025390625,
      "logps/rejected": -425.91571044921875,
      "loss": 0.5645,
      "rewards/accuracies": 0.7333333492279053,
      "rewards/chosen": -0.9139325022697449,
      "rewards/margins": 0.5984528064727783,
      "rewards/rejected": -1.5123852491378784,
      "step": 960
    },
    {
      "epoch": 0.5709240729841083,
      "grad_norm": 103.97188710606014,
      "learning_rate": 2.3178095620580524e-07,
      "logits/chosen": -2.642576217651367,
      "logits/rejected": -2.576850175857544,
      "logps/chosen": -394.79290771484375,
      "logps/rejected": -357.9416809082031,
      "loss": 0.628,
      "rewards/accuracies": 0.5999999642372131,
      "rewards/chosen": -1.2030242681503296,
      "rewards/margins": 0.2695932388305664,
      "rewards/rejected": -1.4726173877716064,
      "step": 970
    },
    {
      "epoch": 0.5768098881695115,
      "grad_norm": 122.23429828784404,
      "learning_rate": 2.266621420801918e-07,
      "logits/chosen": -2.3418798446655273,
      "logits/rejected": -2.3062217235565186,
      "logps/chosen": -334.18182373046875,
      "logps/rejected": -375.04083251953125,
      "loss": 0.5985,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.9751673936843872,
      "rewards/margins": 0.36949509382247925,
      "rewards/rejected": -1.3446625471115112,
      "step": 980
    },
    {
      "epoch": 0.5826957033549146,
      "grad_norm": 53.90742131111234,
      "learning_rate": 2.2155318009259821e-07,
      "logits/chosen": -2.517277479171753,
      "logits/rejected": -2.552461862564087,
      "logps/chosen": -351.4830322265625,
      "logps/rejected": -347.33770751953125,
      "loss": 0.6017,
      "rewards/accuracies": 0.7666666507720947,
      "rewards/chosen": -0.7861822843551636,
      "rewards/margins": 0.5186395645141602,
      "rewards/rejected": -1.3048217296600342,
      "step": 990
    },
    {
      "epoch": 0.5885815185403178,
      "grad_norm": 79.56155835501555,
      "learning_rate": 2.1645622700474695e-07,
      "logits/chosen": -2.53051495552063,
      "logits/rejected": -2.50981068611145,
      "logps/chosen": -300.99261474609375,
      "logps/rejected": -307.985107421875,
      "loss": 0.6313,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.9508684873580933,
      "rewards/margins": 0.4335688054561615,
      "rewards/rejected": -1.3844374418258667,
      "step": 1000
    },
    {
      "epoch": 0.594467333725721,
      "grad_norm": 178.75847915531864,
      "learning_rate": 2.113734345087716e-07,
      "logits/chosen": -2.699451208114624,
      "logits/rejected": -2.7512571811676025,
      "logps/chosen": -213.2908172607422,
      "logps/rejected": -347.00457763671875,
      "loss": 0.4959,
      "rewards/accuracies": 0.6666666865348816,
      "rewards/chosen": -0.5839646458625793,
      "rewards/margins": 0.5955744385719299,
      "rewards/rejected": -1.1795389652252197,
      "step": 1010
    },
    {
      "epoch": 0.6003531489111242,
      "grad_norm": 144.9877689912697,
      "learning_rate": 2.0630694831887473e-07,
      "logits/chosen": -2.3605151176452637,
      "logits/rejected": -2.377347469329834,
      "logps/chosen": -302.31072998046875,
      "logps/rejected": -364.3263854980469,
      "loss": 0.5393,
      "rewards/accuracies": 0.7333333492279053,
      "rewards/chosen": -0.8714925646781921,
      "rewards/margins": 0.615392804145813,
      "rewards/rejected": -1.4868853092193604,
      "step": 1020
    },
    {
      "epoch": 0.6062389640965273,
      "grad_norm": 90.93107910277243,
      "learning_rate": 2.0125890726550896e-07,
      "logits/chosen": -2.774040937423706,
      "logits/rejected": -2.749173641204834,
      "logps/chosen": -312.368896484375,
      "logps/rejected": -329.6822814941406,
      "loss": 0.5044,
      "rewards/accuracies": 0.8333333730697632,
      "rewards/chosen": -0.3795444369316101,
      "rewards/margins": 0.5973297357559204,
      "rewards/rejected": -0.9768741726875305,
      "step": 1030
    },
    {
      "epoch": 0.6121247792819305,
      "grad_norm": 53.759522144352374,
      "learning_rate": 1.9623144239246432e-07,
      "logits/chosen": -2.555093765258789,
      "logits/rejected": -2.572145700454712,
      "logps/chosen": -320.54248046875,
      "logps/rejected": -383.291748046875,
      "loss": 0.5761,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.7120213508605957,
      "rewards/margins": 0.4912422299385071,
      "rewards/rejected": -1.203263521194458,
      "step": 1040
    },
    {
      "epoch": 0.6180105944673338,
      "grad_norm": 96.22956917547782,
      "learning_rate": 1.91226676057242e-07,
      "logits/chosen": -2.5407838821411133,
      "logits/rejected": -2.471928596496582,
      "logps/chosen": -419.9786071777344,
      "logps/rejected": -386.1612854003906,
      "loss": 0.5582,
      "rewards/accuracies": 0.7666666507720947,
      "rewards/chosen": -0.8196290135383606,
      "rewards/margins": 0.6197575330734253,
      "rewards/rejected": -1.4393866062164307,
      "step": 1050
    },
    {
      "epoch": 0.6238964096527368,
      "grad_norm": 84.05938732777776,
      "learning_rate": 1.8624672103509578e-07,
      "logits/chosen": -2.4780263900756836,
      "logits/rejected": -2.3386425971984863,
      "logps/chosen": -417.42724609375,
      "logps/rejected": -414.38262939453125,
      "loss": 0.5922,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.0109527111053467,
      "rewards/margins": 0.40876656770706177,
      "rewards/rejected": -1.4197193384170532,
      "step": 1060
    },
    {
      "epoch": 0.6297822248381401,
      "grad_norm": 125.25073964918506,
      "learning_rate": 1.8129367962711882e-07,
      "logits/chosen": -2.6754302978515625,
      "logits/rejected": -2.6922547817230225,
      "logps/chosen": -416.434326171875,
      "logps/rejected": -419.4442443847656,
      "loss": 0.5897,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.8944675326347351,
      "rewards/margins": 0.5295539498329163,
      "rewards/rejected": -1.4240214824676514,
      "step": 1070
    },
    {
      "epoch": 0.6356680400235433,
      "grad_norm": 87.45258473406294,
      "learning_rate": 1.7636964277275108e-07,
      "logits/chosen": -2.5234591960906982,
      "logits/rejected": -2.5611681938171387,
      "logps/chosen": -326.291015625,
      "logps/rejected": -389.78839111328125,
      "loss": 0.4929,
      "rewards/accuracies": 0.7999999523162842,
      "rewards/chosen": -0.7259222269058228,
      "rewards/margins": 0.8374055624008179,
      "rewards/rejected": -1.5633279085159302,
      "step": 1080
    },
    {
      "epoch": 0.6415538552089465,
      "grad_norm": 53.279206385221514,
      "learning_rate": 1.7147668916708398e-07,
      "logits/chosen": -2.480179786682129,
      "logits/rejected": -2.4370687007904053,
      "logps/chosen": -358.24444580078125,
      "logps/rejected": -384.9347229003906,
      "loss": 0.5857,
      "rewards/accuracies": 0.6333333253860474,
      "rewards/chosen": -1.065148115158081,
      "rewards/margins": 0.38636088371276855,
      "rewards/rejected": -1.4515092372894287,
      "step": 1090
    },
    {
      "epoch": 0.6474396703943496,
      "grad_norm": 52.091641562005016,
      "learning_rate": 1.666168843833339e-07,
      "logits/chosen": -2.3647563457489014,
      "logits/rejected": -2.3754029273986816,
      "logps/chosen": -263.02447509765625,
      "logps/rejected": -354.109619140625,
      "loss": 0.5574,
      "rewards/accuracies": 0.8333333134651184,
      "rewards/chosen": -0.5551760792732239,
      "rewards/margins": 0.844565749168396,
      "rewards/rejected": -1.399741768836975,
      "step": 1100
    },
    {
      "epoch": 0.6533254855797528,
      "grad_norm": 112.76354894611299,
      "learning_rate": 1.6179228000085532e-07,
      "logits/chosen": -2.617955446243286,
      "logits/rejected": -2.5767765045166016,
      "logps/chosen": -301.1888122558594,
      "logps/rejected": -372.39239501953125,
      "loss": 0.6259,
      "rewards/accuracies": 0.6666666269302368,
      "rewards/chosen": -0.8070073127746582,
      "rewards/margins": 0.5521813035011292,
      "rewards/rejected": -1.3591885566711426,
      "step": 1110
    },
    {
      "epoch": 0.659211300765156,
      "grad_norm": 72.36752468174844,
      "learning_rate": 1.570049127390604e-07,
      "logits/chosen": -2.607243299484253,
      "logits/rejected": -2.6284470558166504,
      "logps/chosen": -382.6817321777344,
      "logps/rejected": -473.088134765625,
      "loss": 0.5981,
      "rewards/accuracies": 0.5999999642372131,
      "rewards/chosen": -0.9492104649543762,
      "rewards/margins": 0.6012105941772461,
      "rewards/rejected": -1.5504209995269775,
      "step": 1120
    },
    {
      "epoch": 0.6650971159505592,
      "grad_norm": 99.38680104764003,
      "learning_rate": 1.5225680359761325e-07,
      "logits/chosen": -2.555722713470459,
      "logits/rejected": -2.5732975006103516,
      "logps/chosen": -235.4317169189453,
      "logps/rejected": -325.0386962890625,
      "loss": 0.5918,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.6323750019073486,
      "rewards/margins": 0.4906527101993561,
      "rewards/rejected": -1.1230276823043823,
      "step": 1130
    },
    {
      "epoch": 0.6709829311359623,
      "grad_norm": 113.18996714616782,
      "learning_rate": 1.4754995700325917e-07,
      "logits/chosen": -2.628831624984741,
      "logits/rejected": -2.61437726020813,
      "logps/chosen": -304.4407043457031,
      "logps/rejected": -390.73321533203125,
      "loss": 0.5161,
      "rewards/accuracies": 0.7666667103767395,
      "rewards/chosen": -0.7466219067573547,
      "rewards/margins": 0.5649534463882446,
      "rewards/rejected": -1.3115752935409546,
      "step": 1140
    },
    {
      "epoch": 0.6768687463213655,
      "grad_norm": 213.32924667526723,
      "learning_rate": 1.428863599636511e-07,
      "logits/chosen": -2.4004693031311035,
      "logits/rejected": -2.5256259441375732,
      "logps/chosen": -326.31280517578125,
      "logps/rejected": -392.92529296875,
      "loss": 0.4886,
      "rewards/accuracies": 0.73333340883255,
      "rewards/chosen": -0.8898285627365112,
      "rewards/margins": 0.6658565998077393,
      "rewards/rejected": -1.5556851625442505,
      "step": 1150
    },
    {
      "epoch": 0.6827545615067687,
      "grad_norm": 169.10265364288662,
      "learning_rate": 1.3826798122852868e-07,
      "logits/chosen": -2.6862120628356934,
      "logits/rejected": -2.5977320671081543,
      "logps/chosen": -360.7554016113281,
      "logps/rejected": -339.32513427734375,
      "loss": 0.5558,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.9063922166824341,
      "rewards/margins": 0.3310279846191406,
      "rewards/rejected": -1.2374203205108643,
      "step": 1160
    },
    {
      "epoch": 0.6886403766921718,
      "grad_norm": 173.65784239229404,
      "learning_rate": 1.3369677045860544e-07,
      "logits/chosen": -2.625609874725342,
      "logits/rejected": -2.5897438526153564,
      "logps/chosen": -319.73748779296875,
      "logps/rejected": -373.0987854003906,
      "loss": 0.4981,
      "rewards/accuracies": 0.8333333730697632,
      "rewards/chosen": -0.6690064668655396,
      "rewards/margins": 0.8604344129562378,
      "rewards/rejected": -1.5294408798217773,
      "step": 1170
    },
    {
      "epoch": 0.694526191877575,
      "grad_norm": 124.29493358693395,
      "learning_rate": 1.291746574025148e-07,
      "logits/chosen": -2.736769914627075,
      "logits/rejected": -2.7117438316345215,
      "logps/chosen": -440.48779296875,
      "logps/rejected": -379.79132080078125,
      "loss": 0.5748,
      "rewards/accuracies": 0.6333333253860474,
      "rewards/chosen": -1.1783974170684814,
      "rewards/margins": 0.2958952486515045,
      "rewards/rejected": -1.4742927551269531,
      "step": 1180
    },
    {
      "epoch": 0.7004120070629782,
      "grad_norm": 78.3006038615153,
      "learning_rate": 1.247035510821611e-07,
      "logits/chosen": -2.7176737785339355,
      "logits/rejected": -2.6454663276672363,
      "logps/chosen": -376.97381591796875,
      "logps/rejected": -335.2852783203125,
      "loss": 0.5881,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.9591771960258484,
      "rewards/margins": 0.3564377725124359,
      "rewards/rejected": -1.315614938735962,
      "step": 1190
    },
    {
      "epoch": 0.7062978222483814,
      "grad_norm": 173.0463100342363,
      "learning_rate": 1.2028533898682123e-07,
      "logits/chosen": -2.7012906074523926,
      "logits/rejected": -2.6718831062316895,
      "logps/chosen": -352.58294677734375,
      "logps/rejected": -393.17572021484375,
      "loss": 0.5845,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.8266950845718384,
      "rewards/margins": 0.6615980863571167,
      "rewards/rejected": -1.4882930517196655,
      "step": 1200
    },
    {
      "epoch": 0.7121836374337845,
      "grad_norm": 80.48548009967331,
      "learning_rate": 1.1592188627633598e-07,
      "logits/chosen": -2.583117961883545,
      "logits/rejected": -2.6228084564208984,
      "logps/chosen": -317.26708984375,
      "logps/rejected": -329.43719482421875,
      "loss": 0.632,
      "rewards/accuracies": 0.7333332896232605,
      "rewards/chosen": -0.8232676386833191,
      "rewards/margins": 0.3029004633426666,
      "rewards/rejected": -1.1261680126190186,
      "step": 1210
    },
    {
      "epoch": 0.7180694526191878,
      "grad_norm": 62.307227511632156,
      "learning_rate": 1.1161503499372835e-07,
      "logits/chosen": -2.5381112098693848,
      "logits/rejected": -2.426206350326538,
      "logps/chosen": -356.4806823730469,
      "logps/rejected": -495.2960510253906,
      "loss": 0.4735,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.9427182078361511,
      "rewards/margins": 0.6752321124076843,
      "rewards/rejected": -1.617950439453125,
      "step": 1220
    },
    {
      "epoch": 0.723955267804591,
      "grad_norm": 102.88342581327655,
      "learning_rate": 1.0736660328757954e-07,
      "logits/chosen": -2.63267183303833,
      "logits/rejected": -2.4871132373809814,
      "logps/chosen": -303.36163330078125,
      "logps/rejected": -332.2799987792969,
      "loss": 0.5103,
      "rewards/accuracies": 0.7666666507720947,
      "rewards/chosen": -0.6926143765449524,
      "rewards/margins": 0.48562532663345337,
      "rewards/rejected": -1.1782395839691162,
      "step": 1230
    },
    {
      "epoch": 0.7298410829899941,
      "grad_norm": 69.73162807423068,
      "learning_rate": 1.0317838464449318e-07,
      "logits/chosen": -2.5889840126037598,
      "logits/rejected": -2.594730854034424,
      "logps/chosen": -305.60546875,
      "logps/rejected": -347.56988525390625,
      "loss": 0.5767,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.6716678142547607,
      "rewards/margins": 0.4861713945865631,
      "rewards/rejected": -1.1578391790390015,
      "step": 1240
    },
    {
      "epoch": 0.7357268981753973,
      "grad_norm": 99.72345975810403,
      "learning_rate": 9.905214713197019e-08,
      "logits/chosen": -2.780867338180542,
      "logits/rejected": -2.7212603092193604,
      "logps/chosen": -319.85809326171875,
      "logps/rejected": -362.9429931640625,
      "loss": 0.5763,
      "rewards/accuracies": 0.7333333492279053,
      "rewards/chosen": -0.7506597638130188,
      "rewards/margins": 0.7295576333999634,
      "rewards/rejected": -1.480217456817627,
      "step": 1250
    },
    {
      "epoch": 0.7416127133608005,
      "grad_norm": 42.256039163587936,
      "learning_rate": 9.498963265201454e-08,
      "logits/chosen": -2.5502285957336426,
      "logits/rejected": -2.4693973064422607,
      "logps/chosen": -301.47845458984375,
      "logps/rejected": -348.4825439453125,
      "loss": 0.5623,
      "rewards/accuracies": 0.6333333253860474,
      "rewards/chosen": -0.7193578481674194,
      "rewards/margins": 0.40803733468055725,
      "rewards/rejected": -1.1273951530456543,
      "step": 1260
    },
    {
      "epoch": 0.7474985285462037,
      "grad_norm": 51.960392917513985,
      "learning_rate": 9.09925562057845e-08,
      "logits/chosen": -2.5758213996887207,
      "logits/rejected": -2.6676766872406006,
      "logps/chosen": -319.75018310546875,
      "logps/rejected": -392.013916015625,
      "loss": 0.5471,
      "rewards/accuracies": 0.6333333253860474,
      "rewards/chosen": -0.8297551870346069,
      "rewards/margins": 0.583861231803894,
      "rewards/rejected": -1.4136162996292114,
      "step": 1270
    },
    {
      "epoch": 0.7533843437316068,
      "grad_norm": 96.3763472702637,
      "learning_rate": 8.706260516960043e-08,
      "logits/chosen": -2.6656923294067383,
      "logits/rejected": -2.706164836883545,
      "logps/chosen": -342.38623046875,
      "logps/rejected": -424.30706787109375,
      "loss": 0.6335,
      "rewards/accuracies": 0.6666666865348816,
      "rewards/chosen": -1.1196720600128174,
      "rewards/margins": 0.4281339645385742,
      "rewards/rejected": -1.5478060245513916,
      "step": 1280
    },
    {
      "epoch": 0.75927015891701,
      "grad_norm": 82.6639251391733,
      "learning_rate": 8.320143858261486e-08,
      "logits/chosen": -2.746375560760498,
      "logits/rejected": -2.6994574069976807,
      "logps/chosen": -360.11285400390625,
      "logps/rejected": -339.80926513671875,
      "loss": 0.5403,
      "rewards/accuracies": 0.6333333253860474,
      "rewards/chosen": -0.801996111869812,
      "rewards/margins": 0.3962213695049286,
      "rewards/rejected": -1.1982173919677734,
      "step": 1290
    },
    {
      "epoch": 0.7651559741024132,
      "grad_norm": 58.83037988316151,
      "learning_rate": 7.941068644644405e-08,
      "logits/chosen": -2.7802703380584717,
      "logits/rejected": -2.643925428390503,
      "logps/chosen": -391.9327697753906,
      "logps/rejected": -413.474609375,
      "loss": 0.5632,
      "rewards/accuracies": 0.7333333492279053,
      "rewards/chosen": -1.1195310354232788,
      "rewards/margins": 0.615394115447998,
      "rewards/rejected": -1.7349250316619873,
      "step": 1300
    },
    {
      "epoch": 0.7710417892878164,
      "grad_norm": 73.64616872899245,
      "learning_rate": 7.569194903705913e-08,
      "logits/chosen": -2.7224087715148926,
      "logits/rejected": -2.751784324645996,
      "logps/chosen": -406.634521484375,
      "logps/rejected": -462.2149963378906,
      "loss": 0.5761,
      "rewards/accuracies": 0.6333333253860474,
      "rewards/chosen": -0.9177020788192749,
      "rewards/margins": 0.5259424448013306,
      "rewards/rejected": -1.4436447620391846,
      "step": 1310
    },
    {
      "epoch": 0.7769276044732195,
      "grad_norm": 226.35543077516525,
      "learning_rate": 7.204679622922502e-08,
      "logits/chosen": -2.6812875270843506,
      "logits/rejected": -2.663227081298828,
      "logps/chosen": -432.437744140625,
      "logps/rejected": -406.0752258300781,
      "loss": 0.6171,
      "rewards/accuracies": 0.7666666507720947,
      "rewards/chosen": -0.8880939483642578,
      "rewards/margins": 0.6238939166069031,
      "rewards/rejected": -1.5119879245758057,
      "step": 1320
    },
    {
      "epoch": 0.7828134196586227,
      "grad_norm": 52.032206635043295,
      "learning_rate": 6.847676683377398e-08,
      "logits/chosen": -2.6559484004974365,
      "logits/rejected": -2.5057730674743652,
      "logps/chosen": -473.78338623046875,
      "logps/rejected": -408.1139221191406,
      "loss": 0.5067,
      "rewards/accuracies": 0.7666666507720947,
      "rewards/chosen": -0.8587116003036499,
      "rewards/margins": 0.6869573593139648,
      "rewards/rejected": -1.5456688404083252,
      "step": 1330
    },
    {
      "epoch": 0.7886992348440259,
      "grad_norm": 313.43589076362645,
      "learning_rate": 6.498336794799234e-08,
      "logits/chosen": -2.5412328243255615,
      "logits/rejected": -2.4910500049591064,
      "logps/chosen": -325.08929443359375,
      "logps/rejected": -471.27276611328125,
      "loss": 0.5095,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.8783607482910156,
      "rewards/margins": 0.6811479926109314,
      "rewards/rejected": -1.5595088005065918,
      "step": 1340
    },
    {
      "epoch": 0.794585050029429,
      "grad_norm": 89.6143819430957,
      "learning_rate": 6.156807431939537e-08,
      "logits/chosen": -2.522613763809204,
      "logits/rejected": -2.5427727699279785,
      "logps/chosen": -385.08453369140625,
      "logps/rejected": -421.5887756347656,
      "loss": 0.5045,
      "rewards/accuracies": 0.8333333730697632,
      "rewards/chosen": -0.6811944246292114,
      "rewards/margins": 0.6432855725288391,
      "rewards/rejected": -1.3244801759719849,
      "step": 1350
    },
    {
      "epoch": 0.8004708652148322,
      "grad_norm": 67.09755437948724,
      "learning_rate": 5.8232327723159e-08,
      "logits/chosen": -2.4834189414978027,
      "logits/rejected": -2.527981996536255,
      "logps/chosen": -332.33831787109375,
      "logps/rejected": -436.96856689453125,
      "loss": 0.5009,
      "rewards/accuracies": 0.7666666507720947,
      "rewards/chosen": -0.9011951684951782,
      "rewards/margins": 1.019168496131897,
      "rewards/rejected": -1.920363426208496,
      "step": 1360
    },
    {
      "epoch": 0.8063566804002354,
      "grad_norm": 58.01149580333495,
      "learning_rate": 5.49775363534706e-08,
      "logits/chosen": -2.6389904022216797,
      "logits/rejected": -2.5821375846862793,
      "logps/chosen": -338.70745849609375,
      "logps/rejected": -419.2255859375,
      "loss": 0.4777,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.6776565909385681,
      "rewards/margins": 0.8060138821601868,
      "rewards/rejected": -1.4836704730987549,
      "step": 1370
    },
    {
      "epoch": 0.8122424955856387,
      "grad_norm": 102.8449817133206,
      "learning_rate": 5.180507422905583e-08,
      "logits/chosen": -2.7128119468688965,
      "logits/rejected": -2.673996925354004,
      "logps/chosen": -322.67779541015625,
      "logps/rejected": -363.71270751953125,
      "loss": 0.5261,
      "rewards/accuracies": 0.7999999523162842,
      "rewards/chosen": -0.8014975786209106,
      "rewards/margins": 0.7609502673149109,
      "rewards/rejected": -1.5624477863311768,
      "step": 1380
    },
    {
      "epoch": 0.8181283107710418,
      "grad_norm": 65.36829776160975,
      "learning_rate": 4.871628061313321e-08,
      "logits/chosen": -2.581416368484497,
      "logits/rejected": -2.682572364807129,
      "logps/chosen": -318.94915771484375,
      "logps/rejected": -439.8599548339844,
      "loss": 0.5081,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.6443731188774109,
      "rewards/margins": 1.2208740711212158,
      "rewards/rejected": -1.8652470111846924,
      "step": 1390
    },
    {
      "epoch": 0.824014125956445,
      "grad_norm": 178.13037283488754,
      "learning_rate": 4.571245944804089e-08,
      "logits/chosen": -2.489166736602783,
      "logits/rejected": -2.398707866668701,
      "logps/chosen": -375.8815612792969,
      "logps/rejected": -497.11761474609375,
      "loss": 0.5097,
      "rewards/accuracies": 0.8333333134651184,
      "rewards/chosen": -0.9333661198616028,
      "rewards/margins": 1.0528299808502197,
      "rewards/rejected": -1.9861959218978882,
      "step": 1400
    },
    {
      "epoch": 0.8298999411418482,
      "grad_norm": 172.4847999858577,
      "learning_rate": 4.279487880477342e-08,
      "logits/chosen": -2.5642523765563965,
      "logits/rejected": -2.607612133026123,
      "logps/chosen": -372.1091003417969,
      "logps/rejected": -393.6594543457031,
      "loss": 0.5695,
      "rewards/accuracies": 0.6333333253860474,
      "rewards/chosen": -0.8643810153007507,
      "rewards/margins": 0.6560724973678589,
      "rewards/rejected": -1.5204535722732544,
      "step": 1410
    },
    {
      "epoch": 0.8357857563272513,
      "grad_norm": 88.7462733661287,
      "learning_rate": 3.9964770347662715e-08,
      "logits/chosen": -2.615901470184326,
      "logits/rejected": -2.5380148887634277,
      "logps/chosen": -269.59002685546875,
      "logps/rejected": -408.44708251953125,
      "loss": 0.5431,
      "rewards/accuracies": 0.6666666865348816,
      "rewards/chosen": -0.8203733563423157,
      "rewards/margins": 1.1023374795913696,
      "rewards/rejected": -1.9227107763290405,
      "step": 1420
    },
    {
      "epoch": 0.8416715715126545,
      "grad_norm": 48.812190967043605,
      "learning_rate": 3.7223328814427414e-08,
      "logits/chosen": -2.7706921100616455,
      "logits/rejected": -2.6520023345947266,
      "logps/chosen": -346.84747314453125,
      "logps/rejected": -356.3780212402344,
      "loss": 0.4967,
      "rewards/accuracies": 0.7999999523162842,
      "rewards/chosen": -0.7521100640296936,
      "rewards/margins": 0.6091283559799194,
      "rewards/rejected": -1.3612383604049683,
      "step": 1430
    },
    {
      "epoch": 0.8475573866980577,
      "grad_norm": 81.36099469807638,
      "learning_rate": 3.457171151181154e-08,
      "logits/chosen": -2.6654701232910156,
      "logits/rejected": -2.5508956909179688,
      "logps/chosen": -465.358154296875,
      "logps/rejected": -375.4325866699219,
      "loss": 0.5797,
      "rewards/accuracies": 0.7666666507720947,
      "rewards/chosen": -0.9726330041885376,
      "rewards/margins": 0.5235468149185181,
      "rewards/rejected": -1.4961798191070557,
      "step": 1440
    },
    {
      "epoch": 0.8534432018834609,
      "grad_norm": 58.24549104706358,
      "learning_rate": 3.2011037827024054e-08,
      "logits/chosen": -2.6841177940368652,
      "logits/rejected": -2.628507137298584,
      "logps/chosen": -382.93841552734375,
      "logps/rejected": -372.9692687988281,
      "loss": 0.5755,
      "rewards/accuracies": 0.7333332896232605,
      "rewards/chosen": -0.9706902503967285,
      "rewards/margins": 0.5649546384811401,
      "rewards/rejected": -1.5356448888778687,
      "step": 1450
    },
    {
      "epoch": 0.859329017068864,
      "grad_norm": 52.79125544847847,
      "learning_rate": 2.9542388755186804e-08,
      "logits/chosen": -2.5700201988220215,
      "logits/rejected": -2.6171364784240723,
      "logps/chosen": -345.32354736328125,
      "logps/rejected": -433.79632568359375,
      "loss": 0.5057,
      "rewards/accuracies": 0.7333333492279053,
      "rewards/chosen": -0.8421486020088196,
      "rewards/margins": 0.6202816963195801,
      "rewards/rejected": -1.4624302387237549,
      "step": 1460
    },
    {
      "epoch": 0.8652148322542672,
      "grad_norm": 199.09195971504573,
      "learning_rate": 2.7166806442989743e-08,
      "logits/chosen": -2.468412399291992,
      "logits/rejected": -2.5492916107177734,
      "logps/chosen": -419.80133056640625,
      "logps/rejected": -409.2911682128906,
      "loss": 0.5485,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.1756455898284912,
      "rewards/margins": 0.4044803977012634,
      "rewards/rejected": -1.5801259279251099,
      "step": 1470
    },
    {
      "epoch": 0.8711006474396704,
      "grad_norm": 186.66911649670288,
      "learning_rate": 2.488529374874557e-08,
      "logits/chosen": -2.59171724319458,
      "logits/rejected": -2.5773372650146484,
      "logps/chosen": -398.6797790527344,
      "logps/rejected": -408.37554931640625,
      "loss": 0.4744,
      "rewards/accuracies": 0.7333332896232605,
      "rewards/chosen": -0.9879580736160278,
      "rewards/margins": 0.6637744903564453,
      "rewards/rejected": -1.6517324447631836,
      "step": 1480
    },
    {
      "epoch": 0.8769864626250736,
      "grad_norm": 121.13095753685242,
      "learning_rate": 2.26988138190308e-08,
      "logits/chosen": -2.511080265045166,
      "logits/rejected": -2.476285696029663,
      "logps/chosen": -388.0962219238281,
      "logps/rejected": -352.10211181640625,
      "loss": 0.586,
      "rewards/accuracies": 0.6666666269302368,
      "rewards/chosen": -0.7947136163711548,
      "rewards/margins": 0.43633508682250977,
      "rewards/rejected": -1.2310487031936646,
      "step": 1490
    },
    {
      "epoch": 0.8828722778104767,
      "grad_norm": 60.22203345102361,
      "learning_rate": 2.0608289682090663e-08,
      "logits/chosen": -2.488114833831787,
      "logits/rejected": -2.5074970722198486,
      "logps/chosen": -347.0746765136719,
      "logps/rejected": -471.3238220214844,
      "loss": 0.5907,
      "rewards/accuracies": 0.7666666507720947,
      "rewards/chosen": -0.9873822927474976,
      "rewards/margins": 0.9621238708496094,
      "rewards/rejected": -1.9495060443878174,
      "step": 1500
    },
    {
      "epoch": 0.8887580929958799,
      "grad_norm": 83.04601367710045,
      "learning_rate": 1.86146038581807e-08,
      "logits/chosen": -2.6847586631774902,
      "logits/rejected": -2.6667001247406006,
      "logps/chosen": -379.8851013183594,
      "logps/rejected": -402.2454528808594,
      "loss": 0.5593,
      "rewards/accuracies": 0.7666666507720947,
      "rewards/chosen": -0.6927786469459534,
      "rewards/margins": 0.8660175204277039,
      "rewards/rejected": -1.5587961673736572,
      "step": 1510
    },
    {
      "epoch": 0.8946439081812831,
      "grad_norm": 138.65953465833107,
      "learning_rate": 1.6718597987007977e-08,
      "logits/chosen": -2.6951823234558105,
      "logits/rejected": -2.6642472743988037,
      "logps/chosen": -458.5162658691406,
      "logps/rejected": -486.06890869140625,
      "loss": 0.5285,
      "rewards/accuracies": 0.7000000476837158,
      "rewards/chosen": -0.9807785153388977,
      "rewards/margins": 0.7777236700057983,
      "rewards/rejected": -1.7585022449493408,
      "step": 1520
    },
    {
      "epoch": 0.9005297233666862,
      "grad_norm": 235.95269304208253,
      "learning_rate": 1.4921072472431073e-08,
      "logits/chosen": -2.582733392715454,
      "logits/rejected": -2.54697847366333,
      "logps/chosen": -380.550048828125,
      "logps/rejected": -469.8357849121094,
      "loss": 0.465,
      "rewards/accuracies": 0.9333333969116211,
      "rewards/chosen": -0.8054933547973633,
      "rewards/margins": 1.0806586742401123,
      "rewards/rejected": -1.8861520290374756,
      "step": 1530
    },
    {
      "epoch": 0.9064155385520895,
      "grad_norm": 108.02460840596761,
      "learning_rate": 1.3222786144567317e-08,
      "logits/chosen": -2.5321972370147705,
      "logits/rejected": -2.446223735809326,
      "logps/chosen": -361.153564453125,
      "logps/rejected": -472.0574645996094,
      "loss": 0.5382,
      "rewards/accuracies": 0.6666666269302368,
      "rewards/chosen": -1.0409860610961914,
      "rewards/margins": 0.5477335453033447,
      "rewards/rejected": -1.5887196063995361,
      "step": 1540
    },
    {
      "epoch": 0.9123013537374927,
      "grad_norm": 109.47512454441822,
      "learning_rate": 1.1624455939450773e-08,
      "logits/chosen": -2.5635602474212646,
      "logits/rejected": -2.5231423377990723,
      "logps/chosen": -390.92144775390625,
      "logps/rejected": -435.39874267578125,
      "loss": 0.5468,
      "rewards/accuracies": 0.76666659116745,
      "rewards/chosen": -1.122955083847046,
      "rewards/margins": 0.48533105850219727,
      "rewards/rejected": -1.6082861423492432,
      "step": 1550
    },
    {
      "epoch": 0.9181871689228959,
      "grad_norm": 124.59873217557653,
      "learning_rate": 1.0126756596375685e-08,
      "logits/chosen": -2.5917229652404785,
      "logits/rejected": -2.533984661102295,
      "logps/chosen": -311.223388671875,
      "logps/rejected": -396.94775390625,
      "loss": 0.5317,
      "rewards/accuracies": 0.7666666507720947,
      "rewards/chosen": -1.0927371978759766,
      "rewards/margins": 0.5735793113708496,
      "rewards/rejected": -1.6663163900375366,
      "step": 1560
    },
    {
      "epoch": 0.924072984108299,
      "grad_norm": 115.76739079691396,
      "learning_rate": 8.730320373053545e-09,
      "logits/chosen": -2.56396484375,
      "logits/rejected": -2.548893451690674,
      "logps/chosen": -375.27008056640625,
      "logps/rejected": -411.82183837890625,
      "loss": 0.5228,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.8550286293029785,
      "rewards/margins": 0.5478664636611938,
      "rewards/rejected": -1.402895212173462,
      "step": 1570
    },
    {
      "epoch": 0.9299587992937022,
      "grad_norm": 92.62228298124931,
      "learning_rate": 7.4357367787039605e-09,
      "logits/chosen": -2.5038106441497803,
      "logits/rejected": -2.4670755863189697,
      "logps/chosen": -403.1292724609375,
      "logps/rejected": -438.26580810546875,
      "loss": 0.5464,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.9969646334648132,
      "rewards/margins": 0.7654547691345215,
      "rewards/rejected": -1.7624194622039795,
      "step": 1580
    },
    {
      "epoch": 0.9358446144791054,
      "grad_norm": 55.929288108030285,
      "learning_rate": 6.243552325191581e-09,
      "logits/chosen": -2.4634852409362793,
      "logits/rejected": -2.364482879638672,
      "logps/chosen": -391.3017578125,
      "logps/rejected": -450.46734619140625,
      "loss": 0.4938,
      "rewards/accuracies": 0.7666666507720947,
      "rewards/chosen": -0.8056437373161316,
      "rewards/margins": 1.2021862268447876,
      "rewards/rejected": -2.0078299045562744,
      "step": 1590
    },
    {
      "epoch": 0.9417304296645085,
      "grad_norm": 61.15914730289007,
      "learning_rate": 5.154270296314878e-09,
      "logits/chosen": -2.6435704231262207,
      "logits/rejected": -2.6495299339294434,
      "logps/chosen": -374.0923767089844,
      "logps/rejected": -469.6061096191406,
      "loss": 0.4876,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.7130493521690369,
      "rewards/margins": 0.9921830296516418,
      "rewards/rejected": -1.7052322626113892,
      "step": 1600
    },
    {
      "epoch": 0.9476162448499117,
      "grad_norm": 87.06035022289088,
      "learning_rate": 4.168350535343451e-09,
      "logits/chosen": -2.5816283226013184,
      "logits/rejected": -2.508742570877075,
      "logps/chosen": -431.2406311035156,
      "logps/rejected": -444.27606201171875,
      "loss": 0.6038,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.034767508506775,
      "rewards/margins": 0.6255626082420349,
      "rewards/rejected": -1.6603301763534546,
      "step": 1610
    },
    {
      "epoch": 0.9535020600353149,
      "grad_norm": 71.26805126819406,
      "learning_rate": 3.286209250894012e-09,
      "logits/chosen": -2.616842746734619,
      "logits/rejected": -2.5755038261413574,
      "logps/chosen": -404.6748352050781,
      "logps/rejected": -424.21905517578125,
      "loss": 0.6546,
      "rewards/accuracies": 0.7666666507720947,
      "rewards/chosen": -0.873518168926239,
      "rewards/margins": 0.5968537926673889,
      "rewards/rejected": -1.4703717231750488,
      "step": 1620
    },
    {
      "epoch": 0.9593878752207181,
      "grad_norm": 88.13236089767452,
      "learning_rate": 2.5082188412267614e-09,
      "logits/chosen": -2.623947858810425,
      "logits/rejected": -2.5589542388916016,
      "logps/chosen": -363.2535095214844,
      "logps/rejected": -465.86297607421875,
      "loss": 0.555,
      "rewards/accuracies": 0.7666667103767395,
      "rewards/chosen": -0.8597814440727234,
      "rewards/margins": 0.6761683225631714,
      "rewards/rejected": -1.53594970703125,
      "step": 1630
    },
    {
      "epoch": 0.9652736904061212,
      "grad_norm": 104.76645086528532,
      "learning_rate": 1.8347077370366726e-09,
      "logits/chosen": -2.449385404586792,
      "logits/rejected": -2.401689052581787,
      "logps/chosen": -358.8383483886719,
      "logps/rejected": -446.76092529296875,
      "loss": 0.5539,
      "rewards/accuracies": 0.8333333134651184,
      "rewards/chosen": -0.9526700973510742,
      "rewards/margins": 0.987733006477356,
      "rewards/rejected": -1.9404027462005615,
      "step": 1640
    },
    {
      "epoch": 0.9711595055915244,
      "grad_norm": 119.412074414206,
      "learning_rate": 1.2659602628053712e-09,
      "logits/chosen": -2.5362093448638916,
      "logits/rejected": -2.5611374378204346,
      "logps/chosen": -341.82537841796875,
      "logps/rejected": -383.3554382324219,
      "loss": 0.5503,
      "rewards/accuracies": 0.6333333253860474,
      "rewards/chosen": -0.9428917169570923,
      "rewards/margins": 0.6708999872207642,
      "rewards/rejected": -1.6137914657592773,
      "step": 1650
    },
    {
      "epoch": 0.9770453207769276,
      "grad_norm": 91.85897396152326,
      "learning_rate": 8.022165167730355e-10,
      "logits/chosen": -2.5891966819763184,
      "logits/rejected": -2.598982334136963,
      "logps/chosen": -303.7531433105469,
      "logps/rejected": -403.8943176269531,
      "loss": 0.5337,
      "rewards/accuracies": 0.6999999284744263,
      "rewards/chosen": -0.7517716884613037,
      "rewards/margins": 0.8414438962936401,
      "rewards/rejected": -1.5932157039642334,
      "step": 1660
    },
    {
      "epoch": 0.9829311359623308,
      "grad_norm": 49.42793849018654,
      "learning_rate": 4.4367226958005877e-10,
      "logits/chosen": -2.6086266040802,
      "logits/rejected": -2.586604356765747,
      "logps/chosen": -384.49041748046875,
      "logps/rejected": -449.49365234375,
      "loss": 0.5456,
      "rewards/accuracies": 0.6333333253860474,
      "rewards/chosen": -0.9067045450210571,
      "rewards/margins": 0.4888109266757965,
      "rewards/rejected": -1.3955154418945312,
      "step": 1670
    },
    {
      "epoch": 0.9888169511477339,
      "grad_norm": 114.79577809839017,
      "learning_rate": 1.9047888162218584e-10,
      "logits/chosen": -2.6199536323547363,
      "logits/rejected": -2.5287978649139404,
      "logps/chosen": -333.8753662109375,
      "logps/rejected": -358.57916259765625,
      "loss": 0.4755,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.7598250508308411,
      "rewards/margins": 0.8381843566894531,
      "rewards/rejected": -1.5980093479156494,
      "step": 1680
    },
    {
      "epoch": 0.9947027663331371,
      "grad_norm": 149.79667335956157,
      "learning_rate": 4.2743239153153474e-11,
      "logits/chosen": -2.4809188842773438,
      "logits/rejected": -2.3862576484680176,
      "logps/chosen": -299.9741516113281,
      "logps/rejected": -355.7412414550781,
      "loss": 0.5547,
      "rewards/accuracies": 0.6333333253860474,
      "rewards/chosen": -1.1635828018188477,
      "rewards/margins": 0.4038754403591156,
      "rewards/rejected": -1.5674583911895752,
      "step": 1690
    },
    {
      "epoch": 1.0,
      "step": 1699,
      "total_flos": 0.0,
      "train_loss": 0.3274427829874902,
      "train_runtime": 3901.2433,
      "train_samples_per_second": 3.917,
      "train_steps_per_second": 0.436
    }
  ],
  "logging_steps": 10,
  "max_steps": 1699,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 100,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": false
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 3,
  "trial_name": null,
  "trial_params": null
}