{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 0.9994242947610823,
  "eval_steps": 100,
  "global_step": 868,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.0,
      "grad_norm": 24.168190559264126,
      "learning_rate": 5.747126436781609e-09,
      "logits/chosen": -1.9734797477722168,
      "logits/rejected": -1.856537938117981,
      "logps/chosen": -206.3428497314453,
      "logps/rejected": -155.26254272460938,
      "loss": 0.6932,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 1
    },
    {
      "epoch": 0.01,
      "grad_norm": 24.91157064859394,
      "learning_rate": 5.747126436781609e-08,
      "logits/chosen": -1.9969236850738525,
      "logits/rejected": -1.9658927917480469,
      "logps/chosen": -214.8179168701172,
      "logps/rejected": -192.40969848632812,
      "loss": 0.6932,
      "rewards/accuracies": 0.4791666567325592,
      "rewards/chosen": 0.0006090968381613493,
      "rewards/margins": 0.0008069847244769335,
      "rewards/rejected": -0.00019788791541941464,
      "step": 10
    },
    {
      "epoch": 0.02,
      "grad_norm": 24.574313763724412,
      "learning_rate": 1.1494252873563217e-07,
      "logits/chosen": -2.0616955757141113,
      "logits/rejected": -1.9390573501586914,
      "logps/chosen": -256.38787841796875,
      "logps/rejected": -191.22067260742188,
      "loss": 0.6921,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": 0.0014679343439638615,
      "rewards/margins": 0.002354845404624939,
      "rewards/rejected": -0.0008869109442457557,
      "step": 20
    },
    {
      "epoch": 0.03,
      "grad_norm": 23.51349624308262,
      "learning_rate": 1.7241379310344828e-07,
      "logits/chosen": -1.9969984292984009,
      "logits/rejected": -1.9430469274520874,
      "logps/chosen": -215.03457641601562,
      "logps/rejected": -196.2289581298828,
      "loss": 0.6867,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": 0.012777608819305897,
      "rewards/margins": 0.016061924397945404,
      "rewards/rejected": -0.00328431511297822,
      "step": 30
    },
    {
      "epoch": 0.05,
      "grad_norm": 21.809703812897773,
      "learning_rate": 2.2988505747126435e-07,
      "logits/chosen": -1.9486480951309204,
      "logits/rejected": -1.9111521244049072,
      "logps/chosen": -212.9443817138672,
      "logps/rejected": -192.1443328857422,
      "loss": 0.6745,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": 0.030324190855026245,
      "rewards/margins": 0.04425760358572006,
      "rewards/rejected": -0.013933415524661541,
      "step": 40
    },
    {
      "epoch": 0.06,
      "grad_norm": 22.7618695537933,
      "learning_rate": 2.873563218390804e-07,
      "logits/chosen": -2.025139331817627,
      "logits/rejected": -1.9583957195281982,
      "logps/chosen": -201.2593231201172,
      "logps/rejected": -188.67807006835938,
      "loss": 0.6565,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": 0.029964879155158997,
      "rewards/margins": 0.10461034625768661,
      "rewards/rejected": -0.07464545965194702,
      "step": 50
    },
    {
      "epoch": 0.07,
      "grad_norm": 20.921825952553665,
      "learning_rate": 3.4482758620689656e-07,
      "logits/chosen": -2.064497709274292,
      "logits/rejected": -1.9918781518936157,
      "logps/chosen": -265.92425537109375,
      "logps/rejected": -233.8927459716797,
      "loss": 0.623,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.04738049954175949,
      "rewards/margins": 0.17422285676002502,
      "rewards/rejected": -0.22160334885120392,
      "step": 60
    },
    {
      "epoch": 0.08,
      "grad_norm": 25.686036310459873,
      "learning_rate": 4.0229885057471266e-07,
      "logits/chosen": -1.9937725067138672,
      "logits/rejected": -1.9978179931640625,
      "logps/chosen": -224.78897094726562,
      "logps/rejected": -234.25808715820312,
      "loss": 0.5987,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.28290849924087524,
      "rewards/margins": 0.266859233379364,
      "rewards/rejected": -0.549767792224884,
      "step": 70
    },
    {
      "epoch": 0.09,
      "grad_norm": 31.860669472520566,
      "learning_rate": 4.597701149425287e-07,
      "logits/chosen": -1.7500314712524414,
      "logits/rejected": -1.687111258506775,
      "logps/chosen": -242.4202117919922,
      "logps/rejected": -248.33908081054688,
      "loss": 0.5806,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.4695563316345215,
      "rewards/margins": 0.3295554518699646,
      "rewards/rejected": -0.7991117835044861,
      "step": 80
    },
    {
      "epoch": 0.1,
      "grad_norm": 37.15734802388127,
      "learning_rate": 4.999817969178237e-07,
      "logits/chosen": -1.71634042263031,
      "logits/rejected": -1.6681534051895142,
      "logps/chosen": -319.0685119628906,
      "logps/rejected": -339.21966552734375,
      "loss": 0.5175,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9969006776809692,
      "rewards/margins": 0.5603370070457458,
      "rewards/rejected": -1.5572377443313599,
      "step": 90
    },
    {
      "epoch": 0.12,
      "grad_norm": 38.96794122383384,
      "learning_rate": 4.996582603056428e-07,
      "logits/chosen": -1.7202441692352295,
      "logits/rejected": -1.6160236597061157,
      "logps/chosen": -322.7548522949219,
      "logps/rejected": -370.2288818359375,
      "loss": 0.5295,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -1.038615107536316,
      "rewards/margins": 0.6558629870414734,
      "rewards/rejected": -1.6944783926010132,
      "step": 100
    },
    {
      "epoch": 0.12,
      "eval_logits/chosen": -1.7860382795333862,
      "eval_logits/rejected": -1.69411301612854,
      "eval_logps/chosen": -378.6120910644531,
      "eval_logps/rejected": -405.63275146484375,
      "eval_loss": 0.6075600981712341,
      "eval_rewards/accuracies": 0.69921875,
      "eval_rewards/chosen": -0.43100571632385254,
      "eval_rewards/margins": 0.2942630350589752,
      "eval_rewards/rejected": -0.7252687811851501,
      "eval_runtime": 97.9953,
      "eval_samples_per_second": 20.409,
      "eval_steps_per_second": 0.327,
      "step": 100
    },
    {
      "epoch": 0.13,
      "grad_norm": 39.11820261051907,
      "learning_rate": 4.989308132738126e-07,
      "logits/chosen": -1.7690223455429077,
      "logits/rejected": -1.6496288776397705,
      "logps/chosen": -304.85015869140625,
      "logps/rejected": -325.87982177734375,
      "loss": 0.4999,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.8704218864440918,
      "rewards/margins": 0.6589836478233337,
      "rewards/rejected": -1.5294055938720703,
      "step": 110
    },
    {
      "epoch": 0.14,
      "grad_norm": 43.9866022819552,
      "learning_rate": 4.978006327248536e-07,
      "logits/chosen": -1.7066357135772705,
      "logits/rejected": -1.625906229019165,
      "logps/chosen": -341.0208740234375,
      "logps/rejected": -396.2286071777344,
      "loss": 0.48,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -1.2682462930679321,
      "rewards/margins": 0.8490262031555176,
      "rewards/rejected": -2.1172726154327393,
      "step": 120
    },
    {
      "epoch": 0.15,
      "grad_norm": 51.95435254496828,
      "learning_rate": 4.962695471250032e-07,
      "logits/chosen": -1.7163381576538086,
      "logits/rejected": -1.6467857360839844,
      "logps/chosen": -330.796142578125,
      "logps/rejected": -384.80596923828125,
      "loss": 0.4743,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -1.1125227212905884,
      "rewards/margins": 0.8911293148994446,
      "rewards/rejected": -2.0036520957946777,
      "step": 130
    },
    {
      "epoch": 0.16,
      "grad_norm": 41.76231027113118,
      "learning_rate": 4.94340033546025e-07,
      "logits/chosen": -1.768608808517456,
      "logits/rejected": -1.7792564630508423,
      "logps/chosen": -292.7682189941406,
      "logps/rejected": -356.3222351074219,
      "loss": 0.494,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -1.0807616710662842,
      "rewards/margins": 0.6589750647544861,
      "rewards/rejected": -1.739736795425415,
      "step": 140
    },
    {
      "epoch": 0.17,
      "grad_norm": 42.74539380847778,
      "learning_rate": 4.920152136576705e-07,
      "logits/chosen": -1.5584046840667725,
      "logits/rejected": -1.4847816228866577,
      "logps/chosen": -360.10626220703125,
      "logps/rejected": -438.8833923339844,
      "loss": 0.4604,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -1.4732930660247803,
      "rewards/margins": 1.046295166015625,
      "rewards/rejected": -2.519587993621826,
      "step": 150
    },
    {
      "epoch": 0.18,
      "grad_norm": 52.492556284072315,
      "learning_rate": 4.892988486772756e-07,
      "logits/chosen": -1.4800939559936523,
      "logits/rejected": -1.375672698020935,
      "logps/chosen": -331.51190185546875,
      "logps/rejected": -424.3758850097656,
      "loss": 0.4551,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -1.2153559923171997,
      "rewards/margins": 1.116850733757019,
      "rewards/rejected": -2.3322067260742188,
      "step": 160
    },
    {
      "epoch": 0.2,
      "grad_norm": 41.386384025836755,
      "learning_rate": 4.861953332846629e-07,
      "logits/chosen": -1.3303937911987305,
      "logits/rejected": -1.2361196279525757,
      "logps/chosen": -376.7776794433594,
      "logps/rejected": -447.00604248046875,
      "loss": 0.4547,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -1.4968476295471191,
      "rewards/margins": 1.0457650423049927,
      "rewards/rejected": -2.5426125526428223,
      "step": 170
    },
    {
      "epoch": 0.21,
      "grad_norm": 60.02192982418561,
      "learning_rate": 4.827096885121953e-07,
      "logits/chosen": -1.4225201606750488,
      "logits/rejected": -1.2863495349884033,
      "logps/chosen": -405.1893005371094,
      "logps/rejected": -479.203857421875,
      "loss": 0.4595,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -1.7183666229248047,
      "rewards/margins": 1.0051409006118774,
      "rewards/rejected": -2.7235074043273926,
      "step": 180
    },
    {
      "epoch": 0.22,
      "grad_norm": 43.1272858041485,
      "learning_rate": 4.788475536214821e-07,
      "logits/chosen": -1.1765668392181396,
      "logits/rejected": -1.0755670070648193,
      "logps/chosen": -343.91741943359375,
      "logps/rejected": -436.421142578125,
      "loss": 0.4334,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -1.488790512084961,
      "rewards/margins": 1.1071021556854248,
      "rewards/rejected": -2.5958926677703857,
      "step": 190
    },
    {
      "epoch": 0.23,
      "grad_norm": 46.70523415562261,
      "learning_rate": 4.746151769798818e-07,
      "logits/chosen": -1.0615050792694092,
      "logits/rejected": -0.8847019076347351,
      "logps/chosen": -420.40704345703125,
      "logps/rejected": -515.6028442382812,
      "loss": 0.436,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -1.8328346014022827,
      "rewards/margins": 1.3578665256500244,
      "rewards/rejected": -3.1907010078430176,
      "step": 200
    },
    {
      "epoch": 0.23,
      "eval_logits/chosen": -1.0563830137252808,
      "eval_logits/rejected": -0.9159815311431885,
      "eval_logps/chosen": -438.318115234375,
      "eval_logps/rejected": -487.5738525390625,
      "eval_loss": 0.5481002330780029,
      "eval_rewards/accuracies": 0.7578125,
      "eval_rewards/chosen": -1.0280659198760986,
      "eval_rewards/margins": 0.5166138410568237,
      "eval_rewards/rejected": -1.5446796417236328,
      "eval_runtime": 97.9791,
      "eval_samples_per_second": 20.413,
      "eval_steps_per_second": 0.327,
      "step": 200
    },
    {
      "epoch": 0.24,
      "grad_norm": 38.107555337824884,
      "learning_rate": 4.7001940595156055e-07,
      "logits/chosen": -1.0661436319351196,
      "logits/rejected": -0.9267956018447876,
      "logps/chosen": -323.78460693359375,
      "logps/rejected": -411.57861328125,
      "loss": 0.47,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -1.3589224815368652,
      "rewards/margins": 1.0502302646636963,
      "rewards/rejected": -2.4091525077819824,
      "step": 210
    },
    {
      "epoch": 0.25,
      "grad_norm": 40.45903949711078,
      "learning_rate": 4.650676758194623e-07,
      "logits/chosen": -0.990399181842804,
      "logits/rejected": -0.7643444538116455,
      "logps/chosen": -397.4464111328125,
      "logps/rejected": -486.4747619628906,
      "loss": 0.4237,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -1.74880051612854,
      "rewards/margins": 1.280381441116333,
      "rewards/rejected": -3.029181957244873,
      "step": 220
    },
    {
      "epoch": 0.26,
      "grad_norm": 46.05669291806829,
      "learning_rate": 4.5976799775611215e-07,
      "logits/chosen": -0.9613090753555298,
      "logits/rejected": -0.6790561676025391,
      "logps/chosen": -400.0892639160156,
      "logps/rejected": -520.5755615234375,
      "loss": 0.4331,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -1.871617078781128,
      "rewards/margins": 1.643728256225586,
      "rewards/rejected": -3.515345335006714,
      "step": 230
    },
    {
      "epoch": 0.28,
      "grad_norm": 48.193806404654026,
      "learning_rate": 4.5412894586271543e-07,
      "logits/chosen": -0.7735807299613953,
      "logits/rejected": -0.4405640959739685,
      "logps/chosen": -401.48089599609375,
      "logps/rejected": -470.61663818359375,
      "loss": 0.4189,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -1.8665469884872437,
      "rewards/margins": 1.244009256362915,
      "rewards/rejected": -3.1105563640594482,
      "step": 240
    },
    {
      "epoch": 0.29,
      "grad_norm": 42.10264842470945,
      "learning_rate": 4.481596432975201e-07,
      "logits/chosen": -0.5647836923599243,
      "logits/rejected": -0.4109571576118469,
      "logps/chosen": -371.7290954589844,
      "logps/rejected": -471.7470703125,
      "loss": 0.4267,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -1.94748055934906,
      "rewards/margins": 1.1521204710006714,
      "rewards/rejected": -3.0996010303497314,
      "step": 250
    },
    {
      "epoch": 0.3,
      "grad_norm": 47.08572063870737,
      "learning_rate": 4.41869747515886e-07,
      "logits/chosen": -0.5823490023612976,
      "logits/rejected": -0.4703378677368164,
      "logps/chosen": -381.5385437011719,
      "logps/rejected": -498.1053771972656,
      "loss": 0.427,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -1.5624696016311646,
      "rewards/margins": 1.1981754302978516,
      "rewards/rejected": -2.7606451511383057,
      "step": 260
    },
    {
      "epoch": 0.31,
      "grad_norm": 50.684416210598954,
      "learning_rate": 4.352694346459396e-07,
      "logits/chosen": -0.3632057011127472,
      "logits/rejected": -0.24974389374256134,
      "logps/chosen": -385.367431640625,
      "logps/rejected": -490.25115966796875,
      "loss": 0.4255,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -1.7891931533813477,
      "rewards/margins": 1.2302545309066772,
      "rewards/rejected": -3.0194478034973145,
      "step": 270
    },
    {
      "epoch": 0.32,
      "grad_norm": 40.676202653615285,
      "learning_rate": 4.2836938302509256e-07,
      "logits/chosen": -0.3894518315792084,
      "logits/rejected": -0.02691759541630745,
      "logps/chosen": -369.89105224609375,
      "logps/rejected": -490.12823486328125,
      "loss": 0.4298,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -1.6734466552734375,
      "rewards/margins": 1.521540641784668,
      "rewards/rejected": -3.1949872970581055,
      "step": 280
    },
    {
      "epoch": 0.33,
      "grad_norm": 44.13447631032205,
      "learning_rate": 4.2118075592405874e-07,
      "logits/chosen": -0.3429946303367615,
      "logits/rejected": -0.1553725004196167,
      "logps/chosen": -405.2006530761719,
      "logps/rejected": -525.8575439453125,
      "loss": 0.41,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -1.8301327228546143,
      "rewards/margins": 1.3930988311767578,
      "rewards/rejected": -3.223231554031372,
      "step": 290
    },
    {
      "epoch": 0.35,
      "grad_norm": 49.31357681048272,
      "learning_rate": 4.137151834863213e-07,
      "logits/chosen": 0.24964532256126404,
      "logits/rejected": 0.487697035074234,
      "logps/chosen": -384.4162902832031,
      "logps/rejected": -526.8263549804688,
      "loss": 0.4266,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -2.1807823181152344,
      "rewards/margins": 1.4443342685699463,
      "rewards/rejected": -3.6251163482666016,
      "step": 300
    },
    {
      "epoch": 0.35,
      "eval_logits/chosen": -0.2921224534511566,
      "eval_logits/rejected": -0.00792422890663147,
      "eval_logps/chosen": -501.82440185546875,
      "eval_logps/rejected": -573.9229125976562,
      "eval_loss": 0.4992181956768036,
      "eval_rewards/accuracies": 0.80859375,
      "eval_rewards/chosen": -1.66312837600708,
      "eval_rewards/margins": 0.7450418472290039,
      "eval_rewards/rejected": -2.408170223236084,
      "eval_runtime": 97.8814,
      "eval_samples_per_second": 20.433,
      "eval_steps_per_second": 0.327,
      "step": 300
    },
    {
      "epoch": 0.36,
      "grad_norm": 40.75479432871009,
      "learning_rate": 4.059847439122671e-07,
      "logits/chosen": -0.09978775680065155,
      "logits/rejected": 0.3156757950782776,
      "logps/chosen": -446.403564453125,
      "logps/rejected": -547.4361572265625,
      "loss": 0.4217,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -2.313018798828125,
      "rewards/margins": 1.2966973781585693,
      "rewards/rejected": -3.609715700149536,
      "step": 310
    },
    {
      "epoch": 0.37,
      "grad_norm": 52.15388720725064,
      "learning_rate": 3.98001943918432e-07,
      "logits/chosen": -0.26411113142967224,
      "logits/rejected": -0.02265077270567417,
      "logps/chosen": -394.48846435546875,
      "logps/rejected": -499.8885803222656,
      "loss": 0.417,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.9507391452789307,
      "rewards/margins": 1.151451826095581,
      "rewards/rejected": -3.1021907329559326,
      "step": 320
    },
    {
      "epoch": 0.38,
      "grad_norm": 51.359405875032216,
      "learning_rate": 3.8977969850346866e-07,
      "logits/chosen": -0.22085031867027283,
      "logits/rejected": 0.16305044293403625,
      "logps/chosen": -414.28179931640625,
      "logps/rejected": -521.32275390625,
      "loss": 0.4006,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -1.9662139415740967,
      "rewards/margins": 1.4125010967254639,
      "rewards/rejected": -3.3787150382995605,
      "step": 330
    },
    {
      "epoch": 0.39,
      "grad_norm": 46.94996126435224,
      "learning_rate": 3.8133131005357465e-07,
      "logits/chosen": -0.14262983202934265,
      "logits/rejected": 0.33533433079719543,
      "logps/chosen": -382.92584228515625,
      "logps/rejected": -544.5906982421875,
      "loss": 0.3957,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -1.7721102237701416,
      "rewards/margins": 1.8106200695037842,
      "rewards/rejected": -3.5827300548553467,
      "step": 340
    },
    {
      "epoch": 0.4,
      "grad_norm": 44.17747510829861,
      "learning_rate": 3.7267044682118435e-07,
      "logits/chosen": 0.15123403072357178,
      "logits/rejected": 0.505038857460022,
      "logps/chosen": -381.1158447265625,
      "logps/rejected": -513.79248046875,
      "loss": 0.3939,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -1.9833688735961914,
      "rewards/margins": 1.5414726734161377,
      "rewards/rejected": -3.524841785430908,
      "step": 350
    },
    {
      "epoch": 0.41,
      "grad_norm": 44.94408605122442,
      "learning_rate": 3.638111208117425e-07,
      "logits/chosen": -0.1444779336452484,
      "logits/rejected": 0.1327807903289795,
      "logps/chosen": -403.40716552734375,
      "logps/rejected": -509.33642578125,
      "loss": 0.4116,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -2.0103116035461426,
      "rewards/margins": 1.1474075317382812,
      "rewards/rejected": -3.157719135284424,
      "step": 360
    },
    {
      "epoch": 0.43,
      "grad_norm": 38.27635210179194,
      "learning_rate": 3.5476766511433605e-07,
      "logits/chosen": -0.3243602216243744,
      "logits/rejected": 0.23012924194335938,
      "logps/chosen": -437.81951904296875,
      "logps/rejected": -535.6306762695312,
      "loss": 0.4207,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -2.0247864723205566,
      "rewards/margins": 1.4481357336044312,
      "rewards/rejected": -3.4729220867156982,
      "step": 370
    },
    {
      "epoch": 0.44,
      "grad_norm": 45.6051115938209,
      "learning_rate": 3.455547107128602e-07,
      "logits/chosen": -0.22209982573986053,
      "logits/rejected": 0.37390798330307007,
      "logps/chosen": -420.72906494140625,
      "logps/rejected": -538.1783447265625,
      "loss": 0.3725,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -1.714195966720581,
      "rewards/margins": 1.7354304790496826,
      "rewards/rejected": -3.4496264457702637,
      "step": 380
    },
    {
      "epoch": 0.45,
      "grad_norm": 62.59138136421735,
      "learning_rate": 3.361871628152338e-07,
      "logits/chosen": 0.11889226734638214,
      "logits/rejected": 0.4943965971469879,
      "logps/chosen": -413.30584716796875,
      "logps/rejected": -576.02294921875,
      "loss": 0.4157,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -2.1105499267578125,
      "rewards/margins": 1.6212915182113647,
      "rewards/rejected": -3.7318413257598877,
      "step": 390
    },
    {
      "epoch": 0.46,
      "grad_norm": 46.94899679949689,
      "learning_rate": 3.2668017673896077e-07,
      "logits/chosen": 0.019377555698156357,
      "logits/rejected": 0.4903746545314789,
      "logps/chosen": -378.95196533203125,
      "logps/rejected": -502.42132568359375,
      "loss": 0.3779,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -1.7431423664093018,
      "rewards/margins": 1.6023037433624268,
      "rewards/rejected": -3.3454463481903076,
      "step": 400
    },
    {
      "epoch": 0.46,
      "eval_logits/chosen": -0.29974818229675293,
      "eval_logits/rejected": 0.09132258594036102,
      "eval_logps/chosen": -488.41680908203125,
      "eval_logps/rejected": -561.919921875,
      "eval_loss": 0.4760280251502991,
      "eval_rewards/accuracies": 0.81640625,
      "eval_rewards/chosen": -1.529052972793579,
      "eval_rewards/margins": 0.7590875029563904,
      "eval_rewards/rejected": -2.288140296936035,
      "eval_runtime": 97.9145,
      "eval_samples_per_second": 20.426,
      "eval_steps_per_second": 0.327,
      "step": 400
    },
    {
      "epoch": 0.47,
      "grad_norm": 42.41150092354519,
      "learning_rate": 3.1704913339205103e-07,
      "logits/chosen": 0.11346012353897095,
      "logits/rejected": 0.6633824706077576,
      "logps/chosen": -411.09710693359375,
      "logps/rejected": -563.5574951171875,
      "loss": 0.3878,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -1.9903961420059204,
      "rewards/margins": 1.649317741394043,
      "rewards/rejected": -3.639713764190674,
      "step": 410
    },
    {
      "epoch": 0.48,
      "grad_norm": 40.93253239819171,
      "learning_rate": 3.0730961438896885e-07,
      "logits/chosen": -0.09407112747430801,
      "logits/rejected": 0.41040462255477905,
      "logps/chosen": -475.8846130371094,
      "logps/rejected": -580.8755493164062,
      "loss": 0.3865,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -2.0496103763580322,
      "rewards/margins": 1.4907814264297485,
      "rewards/rejected": -3.540391445159912,
      "step": 420
    },
    {
      "epoch": 0.5,
      "grad_norm": 52.33032593793305,
      "learning_rate": 2.9747737684186795e-07,
      "logits/chosen": 0.3089054524898529,
      "logits/rejected": 0.609168529510498,
      "logps/chosen": -392.91888427734375,
      "logps/rejected": -519.9400024414062,
      "loss": 0.3899,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -1.8225826025009155,
      "rewards/margins": 1.5612199306488037,
      "rewards/rejected": -3.383802890777588,
      "step": 430
    },
    {
      "epoch": 0.51,
      "grad_norm": 40.99316990627953,
      "learning_rate": 2.8756832786789663e-07,
      "logits/chosen": 0.07930847257375717,
      "logits/rejected": 0.5284430980682373,
      "logps/chosen": -410.39459228515625,
      "logps/rejected": -527.0020751953125,
      "loss": 0.4048,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -1.807734727859497,
      "rewards/margins": 1.5688127279281616,
      "rewards/rejected": -3.3765475749969482,
      "step": 440
    },
    {
      "epoch": 0.52,
      "grad_norm": 50.98336558470055,
      "learning_rate": 2.7759849885381747e-07,
      "logits/chosen": 0.13231831789016724,
      "logits/rejected": 0.8564577102661133,
      "logps/chosen": -452.3924255371094,
      "logps/rejected": -579.27490234375,
      "loss": 0.3869,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -2.116553783416748,
      "rewards/margins": 1.8397691249847412,
      "rewards/rejected": -3.9563231468200684,
      "step": 450
    },
    {
      "epoch": 0.53,
      "grad_norm": 36.53232487094238,
      "learning_rate": 2.675840195195762e-07,
      "logits/chosen": 0.1366969347000122,
      "logits/rejected": 0.6087800860404968,
      "logps/chosen": -388.03546142578125,
      "logps/rejected": -536.0306396484375,
      "loss": 0.3903,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -1.9025928974151611,
      "rewards/margins": 1.6377290487289429,
      "rewards/rejected": -3.5403220653533936,
      "step": 460
    },
    {
      "epoch": 0.54,
      "grad_norm": 48.92609199634003,
      "learning_rate": 2.575410918227829e-07,
      "logits/chosen": 0.14545145630836487,
      "logits/rejected": 0.5727478861808777,
      "logps/chosen": -420.89129638671875,
      "logps/rejected": -546.9252319335938,
      "loss": 0.3812,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.8398689031600952,
      "rewards/margins": 1.5490201711654663,
      "rewards/rejected": -3.3888893127441406,
      "step": 470
    },
    {
      "epoch": 0.55,
      "grad_norm": 51.43432335912027,
      "learning_rate": 2.474859637463226e-07,
      "logits/chosen": 0.060339294373989105,
      "logits/rejected": 0.8116003274917603,
      "logps/chosen": -399.6466369628906,
      "logps/rejected": -525.1762084960938,
      "loss": 0.3789,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -1.7933282852172852,
      "rewards/margins": 1.7597835063934326,
      "rewards/rejected": -3.553112030029297,
      "step": 480
    },
    {
      "epoch": 0.56,
      "grad_norm": 43.984850774488535,
      "learning_rate": 2.3743490301150355e-07,
      "logits/chosen": 0.20873646438121796,
      "logits/rejected": 1.0175427198410034,
      "logps/chosen": -386.0199890136719,
      "logps/rejected": -533.820068359375,
      "loss": 0.3935,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -1.6440702676773071,
      "rewards/margins": 1.8672387599945068,
      "rewards/rejected": -3.5113091468811035,
      "step": 490
    },
    {
      "epoch": 0.58,
      "grad_norm": 47.341362361122584,
      "learning_rate": 2.274041707592724e-07,
      "logits/chosen": 0.6152507066726685,
      "logits/rejected": 1.061025619506836,
      "logps/chosen": -425.3057556152344,
      "logps/rejected": -613.043212890625,
      "loss": 0.3713,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -2.342782974243164,
      "rewards/margins": 1.9541997909545898,
      "rewards/rejected": -4.296982765197754,
      "step": 500
    },
    {
      "epoch": 0.58,
      "eval_logits/chosen": 0.0540677085518837,
      "eval_logits/rejected": 0.4885663688182831,
      "eval_logps/chosen": -471.02880859375,
      "eval_logps/rejected": -557.1675415039062,
      "eval_loss": 0.4526832401752472,
      "eval_rewards/accuracies": 0.83203125,
      "eval_rewards/chosen": -1.3551722764968872,
      "eval_rewards/margins": 0.885444164276123,
      "eval_rewards/rejected": -2.2406163215637207,
      "eval_runtime": 98.0229,
      "eval_samples_per_second": 20.403,
      "eval_steps_per_second": 0.326,
      "step": 500
    },
    {
      "epoch": 0.59,
      "grad_norm": 44.58362843206397,
      "learning_rate": 2.17409995242075e-07,
      "logits/chosen": 0.7180274128913879,
      "logits/rejected": 1.3152275085449219,
      "logps/chosen": -407.83294677734375,
      "logps/rejected": -553.6475830078125,
      "loss": 0.3947,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -2.083230972290039,
      "rewards/margins": 1.8255109786987305,
      "rewards/rejected": -3.9087421894073486,
      "step": 510
    },
    {
      "epoch": 0.6,
      "grad_norm": 39.74511163289153,
      "learning_rate": 2.0746854556892544e-07,
      "logits/chosen": 0.7251445055007935,
      "logits/rejected": 0.930740475654602,
      "logps/chosen": -376.7163391113281,
      "logps/rejected": -514.938232421875,
      "loss": 0.4067,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -1.9309794902801514,
      "rewards/margins": 1.4579864740371704,
      "rewards/rejected": -3.3889663219451904,
      "step": 520
    },
    {
      "epoch": 0.61,
      "grad_norm": 41.78948639916862,
      "learning_rate": 1.9759590554616173e-07,
      "logits/chosen": 0.3273155689239502,
      "logits/rejected": 0.7224196195602417,
      "logps/chosen": -409.301513671875,
      "logps/rejected": -529.0758056640625,
      "loss": 0.3983,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -1.8711875677108765,
      "rewards/margins": 1.4573709964752197,
      "rewards/rejected": -3.328559160232544,
      "step": 530
    },
    {
      "epoch": 0.62,
      "grad_norm": 46.45949710434537,
      "learning_rate": 1.8780804765620746e-07,
      "logits/chosen": 0.4330478608608246,
      "logits/rejected": 0.7140570878982544,
      "logps/chosen": -430.78216552734375,
      "logps/rejected": -596.1729736328125,
      "loss": 0.3842,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -1.9393028020858765,
      "rewards/margins": 1.6490939855575562,
      "rewards/rejected": -3.5883967876434326,
      "step": 540
    },
    {
      "epoch": 0.63,
      "grad_norm": 43.293723569052126,
      "learning_rate": 1.7812080721643973e-07,
      "logits/chosen": 0.6848554015159607,
      "logits/rejected": 1.3088548183441162,
      "logps/chosen": -445.448974609375,
      "logps/rejected": -563.468017578125,
      "loss": 0.3878,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -2.22265887260437,
      "rewards/margins": 1.7124555110931396,
      "rewards/rejected": -3.935114622116089,
      "step": 550
    },
    {
      "epoch": 0.64,
      "grad_norm": 45.49524925522091,
      "learning_rate": 1.6854985675997063e-07,
      "logits/chosen": 0.5498164296150208,
      "logits/rejected": 0.9551798105239868,
      "logps/chosen": -438.2308044433594,
      "logps/rejected": -576.6223754882812,
      "loss": 0.3709,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -2.190483570098877,
      "rewards/margins": 1.594870686531067,
      "rewards/rejected": -3.7853546142578125,
      "step": 560
    },
    {
      "epoch": 0.66,
      "grad_norm": 49.552774447582685,
      "learning_rate": 1.5911068067978818e-07,
      "logits/chosen": 1.0865147113800049,
      "logits/rejected": 1.311841368675232,
      "logps/chosen": -423.2745666503906,
      "logps/rejected": -610.8968505859375,
      "loss": 0.3604,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -2.3493454456329346,
      "rewards/margins": 1.8509175777435303,
      "rewards/rejected": -4.200263023376465,
      "step": 570
    },
    {
      "epoch": 0.67,
      "grad_norm": 63.52041865054028,
      "learning_rate": 1.4981855017728197e-07,
      "logits/chosen": 0.6576471328735352,
      "logits/rejected": 1.0078567266464233,
      "logps/chosen": -491.30853271484375,
      "logps/rejected": -656.1266479492188,
      "loss": 0.3887,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -2.717817783355713,
      "rewards/margins": 1.6682507991790771,
      "rewards/rejected": -4.386068820953369,
      "step": 580
    },
    {
      "epoch": 0.68,
      "grad_norm": 43.46932861137291,
      "learning_rate": 1.406884985556804e-07,
      "logits/chosen": 0.5208752155303955,
      "logits/rejected": 1.1108620166778564,
      "logps/chosen": -435.3440856933594,
      "logps/rejected": -587.5715942382812,
      "loss": 0.3827,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -2.262911081314087,
      "rewards/margins": 1.835397720336914,
      "rewards/rejected": -4.098308563232422,
      "step": 590
    },
    {
      "epoch": 0.69,
      "grad_norm": 55.685425152683656,
      "learning_rate": 1.3173529689837354e-07,
      "logits/chosen": 0.7303274869918823,
      "logits/rejected": 1.3893494606018066,
      "logps/chosen": -411.096435546875,
      "logps/rejected": -538.6531982421875,
      "loss": 0.3817,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -2.0049891471862793,
      "rewards/margins": 1.6658912897109985,
      "rewards/rejected": -3.670880079269409,
      "step": 600
    },
    {
      "epoch": 0.69,
      "eval_logits/chosen": 0.45960405468940735,
      "eval_logits/rejected": 0.9378364086151123,
      "eval_logps/chosen": -488.27398681640625,
      "eval_logps/rejected": -576.624755859375,
      "eval_loss": 0.43982604146003723,
      "eval_rewards/accuracies": 0.8515625,
      "eval_rewards/chosen": -1.5276248455047607,
      "eval_rewards/margins": 0.9075638055801392,
      "eval_rewards/rejected": -2.4351885318756104,
      "eval_runtime": 98.0553,
      "eval_samples_per_second": 20.397,
      "eval_steps_per_second": 0.326,
      "step": 600
    },
    {
      "epoch": 0.7,
      "grad_norm": 51.141776378841776,
      "learning_rate": 1.2297343017146726e-07,
      "logits/chosen": 1.2994943857192993,
      "logits/rejected": 1.7489960193634033,
      "logps/chosen": -437.8944396972656,
      "logps/rejected": -563.5843505859375,
      "loss": 0.3939,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -2.239246129989624,
      "rewards/margins": 1.5640047788619995,
      "rewards/rejected": -3.803250789642334,
      "step": 610
    },
    {
      "epoch": 0.71,
      "grad_norm": 47.72075892004601,
      "learning_rate": 1.1441707378923474e-07,
      "logits/chosen": 0.8435400128364563,
      "logits/rejected": 1.541442632675171,
      "logps/chosen": -382.5939025878906,
      "logps/rejected": -538.156005859375,
      "loss": 0.3816,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -1.7874940633773804,
      "rewards/margins": 1.7763278484344482,
      "rewards/rejected": -3.5638222694396973,
      "step": 620
    },
    {
      "epoch": 0.73,
      "grad_norm": 46.403069038802954,
      "learning_rate": 1.06080070680377e-07,
      "logits/chosen": 1.035014271736145,
      "logits/rejected": 1.5653488636016846,
      "logps/chosen": -415.8087463378906,
      "logps/rejected": -552.2741088867188,
      "loss": 0.3781,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -1.8548622131347656,
      "rewards/margins": 1.7130234241485596,
      "rewards/rejected": -3.567885637283325,
      "step": 630
    },
    {
      "epoch": 0.74,
      "grad_norm": 42.088226042573645,
      "learning_rate": 9.797590889219587e-08,
      "logits/chosen": 0.6099433302879333,
      "logits/rejected": 1.3236600160598755,
      "logps/chosen": -412.41351318359375,
      "logps/rejected": -552.0162353515625,
      "loss": 0.389,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -1.8625577688217163,
      "rewards/margins": 1.764461874961853,
      "rewards/rejected": -3.6270194053649902,
      "step": 640
    },
    {
      "epoch": 0.75,
      "grad_norm": 48.75764274598144,
      "learning_rate": 9.011769976891367e-08,
      "logits/chosen": 1.0017446279525757,
      "logits/rejected": 1.4410614967346191,
      "logps/chosen": -427.5480041503906,
      "logps/rejected": -576.4400634765625,
      "loss": 0.3789,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -2.1677966117858887,
      "rewards/margins": 1.6422433853149414,
      "rewards/rejected": -3.810039520263672,
      "step": 650
    },
    {
      "epoch": 0.76,
      "grad_norm": 48.19640920928193,
      "learning_rate": 8.251815673944218e-08,
      "logits/chosen": 1.2581113576889038,
      "logits/rejected": 1.898616075515747,
      "logps/chosen": -470.30450439453125,
      "logps/rejected": -610.0867919921875,
      "loss": 0.3836,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -2.4716596603393555,
      "rewards/margins": 1.8109443187713623,
      "rewards/rejected": -4.2826032638549805,
      "step": 660
    },
    {
      "epoch": 0.77,
      "grad_norm": 42.83800076757557,
      "learning_rate": 7.518957474892148e-08,
      "logits/chosen": 1.1020927429199219,
      "logits/rejected": 1.865277886390686,
      "logps/chosen": -445.0255432128906,
      "logps/rejected": -610.1484375,
      "loss": 0.3632,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -2.2662646770477295,
      "rewards/margins": 2.038935661315918,
      "rewards/rejected": -4.305201053619385,
      "step": 670
    },
    {
      "epoch": 0.78,
      "grad_norm": 43.94063414538923,
      "learning_rate": 6.814381036730274e-08,
      "logits/chosen": 1.0993672609329224,
      "logits/rejected": 1.476314663887024,
      "logps/chosen": -414.5472717285156,
      "logps/rejected": -555.24169921875,
      "loss": 0.3891,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -2.070984363555908,
      "rewards/margins": 1.6533327102661133,
      "rewards/rejected": -3.7243168354034424,
      "step": 680
    },
    {
      "epoch": 0.79,
      "grad_norm": 48.996886773937526,
      "learning_rate": 6.139226260715872e-08,
      "logits/chosen": 1.0745365619659424,
      "logits/rejected": 1.6123872995376587,
      "logps/chosen": -420.8257751464844,
      "logps/rejected": -591.067138671875,
      "loss": 0.3752,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -2.08198618888855,
      "rewards/margins": 1.920911431312561,
      "rewards/rejected": -4.0028977394104,
      "step": 690
    },
    {
      "epoch": 0.81,
      "grad_norm": 44.913687680486454,
      "learning_rate": 5.4945854481754734e-08,
      "logits/chosen": 1.2772502899169922,
      "logits/rejected": 1.7744108438491821,
      "logps/chosen": -427.616943359375,
      "logps/rejected": -586.7508544921875,
      "loss": 0.3613,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -2.3168463706970215,
      "rewards/margins": 1.800844430923462,
      "rewards/rejected": -4.117690563201904,
      "step": 700
    },
    {
      "epoch": 0.81,
      "eval_logits/chosen": 0.4112035036087036,
      "eval_logits/rejected": 0.9228037595748901,
      "eval_logps/chosen": -482.6694641113281,
      "eval_logps/rejected": -572.7808837890625,
      "eval_loss": 0.4307582378387451,
      "eval_rewards/accuracies": 0.87109375,
      "eval_rewards/chosen": -1.4715794324874878,
      "eval_rewards/margins": 0.9251713752746582,
      "eval_rewards/rejected": -2.3967509269714355,
      "eval_runtime": 97.8788,
      "eval_samples_per_second": 20.433,
      "eval_steps_per_second": 0.327,
      "step": 700
    },
    {
      "epoch": 0.82,
      "grad_norm": 45.470449244785705,
      "learning_rate": 4.881501533321605e-08,
      "logits/chosen": 1.6023136377334595,
      "logits/rejected": 2.1105284690856934,
      "logps/chosen": -408.48602294921875,
      "logps/rejected": -587.4254150390625,
      "loss": 0.355,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -2.2400927543640137,
      "rewards/margins": 1.9537967443466187,
      "rewards/rejected": -4.193889617919922,
      "step": 710
    },
    {
      "epoch": 0.83,
      "grad_norm": 42.625832201611274,
      "learning_rate": 4.300966395938377e-08,
      "logits/chosen": 1.143576741218567,
      "logits/rejected": 1.7894223928451538,
      "logps/chosen": -466.27520751953125,
      "logps/rejected": -622.3167724609375,
      "loss": 0.3773,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -2.4046239852905273,
      "rewards/margins": 1.8917633295059204,
      "rewards/rejected": -4.296387672424316,
      "step": 720
    },
    {
      "epoch": 0.84,
      "grad_norm": 40.15824305792834,
      "learning_rate": 3.7539192566655246e-08,
      "logits/chosen": 1.1031806468963623,
      "logits/rejected": 1.7454750537872314,
      "logps/chosen": -413.6033630371094,
      "logps/rejected": -561.4763793945312,
      "loss": 0.3708,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.902054786682129,
      "rewards/margins": 1.8310045003890991,
      "rewards/rejected": -3.7330594062805176,
      "step": 730
    },
    {
      "epoch": 0.85,
      "grad_norm": 44.28447675992636,
      "learning_rate": 3.24124515747731e-08,
      "logits/chosen": 1.2360585927963257,
      "logits/rejected": 1.7114673852920532,
      "logps/chosen": -428.1229553222656,
      "logps/rejected": -599.7882690429688,
      "loss": 0.3783,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -2.270901679992676,
      "rewards/margins": 1.863284707069397,
      "rewards/rejected": -4.134186267852783,
      "step": 740
    },
    {
      "epoch": 0.86,
      "grad_norm": 48.30307447289823,
      "learning_rate": 2.763773529814506e-08,
      "logits/chosen": 0.926419734954834,
      "logits/rejected": 1.3899872303009033,
      "logps/chosen": -468.01031494140625,
      "logps/rejected": -613.7434692382812,
      "loss": 0.3788,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -2.2661325931549072,
      "rewards/margins": 1.9401893615722656,
      "rewards/rejected": -4.206322193145752,
      "step": 750
    },
    {
      "epoch": 0.88,
      "grad_norm": 56.11779432942193,
      "learning_rate": 2.3222768526860698e-08,
      "logits/chosen": 1.0837215185165405,
      "logits/rejected": 1.9027248620986938,
      "logps/chosen": -431.9537048339844,
      "logps/rejected": -601.56787109375,
      "loss": 0.3908,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -2.2378554344177246,
      "rewards/margins": 1.9761545658111572,
      "rewards/rejected": -4.214009761810303,
      "step": 760
    },
    {
      "epoch": 0.89,
      "grad_norm": 40.169886925607514,
      "learning_rate": 1.9174694029115146e-08,
      "logits/chosen": 1.0360382795333862,
      "logits/rejected": 1.7473223209381104,
      "logps/chosen": -454.27264404296875,
      "logps/rejected": -561.0426635742188,
      "loss": 0.3798,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -2.1006946563720703,
      "rewards/margins": 1.62350332736969,
      "rewards/rejected": -3.72419810295105,
      "step": 770
    },
    {
      "epoch": 0.9,
      "grad_norm": 47.263235112753776,
      "learning_rate": 1.5500060995258134e-08,
      "logits/chosen": 1.1812760829925537,
      "logits/rejected": 1.7791898250579834,
      "logps/chosen": -428.4283142089844,
      "logps/rejected": -571.8958129882812,
      "loss": 0.349,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -2.121316909790039,
      "rewards/margins": 1.8303005695343018,
      "rewards/rejected": -3.9516170024871826,
      "step": 780
    },
    {
      "epoch": 0.91,
      "grad_norm": 54.446594272094224,
      "learning_rate": 1.2204814442165812e-08,
      "logits/chosen": 1.0776536464691162,
      "logits/rejected": 1.8896220922470093,
      "logps/chosen": -431.5889587402344,
      "logps/rejected": -589.2385864257812,
      "loss": 0.3802,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -2.211080312728882,
      "rewards/margins": 2.0271711349487305,
      "rewards/rejected": -4.238251686096191,
      "step": 790
    },
    {
      "epoch": 0.92,
      "grad_norm": 48.742336588451536,
      "learning_rate": 9.294285595075669e-09,
      "logits/chosen": 0.6350497007369995,
      "logits/rejected": 1.2895339727401733,
      "logps/chosen": -462.9306640625,
      "logps/rejected": -595.6204223632812,
      "loss": 0.4032,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -2.274238348007202,
      "rewards/margins": 1.7595179080963135,
      "rewards/rejected": -4.033755779266357,
      "step": 800
    },
    {
      "epoch": 0.92,
      "eval_logits/chosen": 0.34458938241004944,
      "eval_logits/rejected": 0.8732683658599854,
      "eval_logps/chosen": -476.1736145019531,
      "eval_logps/rejected": -567.4185180664062,
      "eval_loss": 0.42825520038604736,
      "eval_rewards/accuracies": 0.859375,
      "eval_rewards/chosen": -1.406620979309082,
      "eval_rewards/margins": 0.9365058541297913,
      "eval_rewards/rejected": -2.3431267738342285,
      "eval_runtime": 97.856,
      "eval_samples_per_second": 20.438,
      "eval_steps_per_second": 0.327,
      "step": 800
    },
    {
      "epoch": 0.93,
      "grad_norm": 49.18818695219976,
      "learning_rate": 6.773183262446914e-09,
      "logits/chosen": 1.1270101070404053,
      "logits/rejected": 1.9546995162963867,
      "logps/chosen": -432.09771728515625,
      "logps/rejected": -578.98779296875,
      "loss": 0.3869,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -2.2396090030670166,
      "rewards/margins": 1.7456976175308228,
      "rewards/rejected": -3.98530650138855,
      "step": 810
    },
    {
      "epoch": 0.94,
      "grad_norm": 48.42955108889553,
      "learning_rate": 4.645586217799452e-09,
      "logits/chosen": 1.1346285343170166,
      "logits/rejected": 1.6442772150039673,
      "logps/chosen": -434.9295349121094,
      "logps/rejected": -606.5115966796875,
      "loss": 0.4024,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -2.1286814212799072,
      "rewards/margins": 1.8881809711456299,
      "rewards/rejected": -4.016861915588379,
      "step": 820
    },
    {
      "epoch": 0.96,
      "grad_norm": 52.31327093944872,
      "learning_rate": 2.9149366008568987e-09,
      "logits/chosen": 1.01073157787323,
      "logits/rejected": 1.627968192100525,
      "logps/chosen": -430.75146484375,
      "logps/rejected": -594.0120849609375,
      "loss": 0.3907,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -2.109837055206299,
      "rewards/margins": 1.913220763206482,
      "rewards/rejected": -4.023057460784912,
      "step": 830
    },
    {
      "epoch": 0.97,
      "grad_norm": 51.70748969913889,
      "learning_rate": 1.5840343486700215e-09,
      "logits/chosen": 0.7685258984565735,
      "logits/rejected": 1.560227870941162,
      "logps/chosen": -438.5198669433594,
      "logps/rejected": -593.31591796875,
      "loss": 0.3724,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -1.9523073434829712,
      "rewards/margins": 2.0151476860046387,
      "rewards/rejected": -3.9674553871154785,
      "step": 840
    },
    {
      "epoch": 0.98,
      "grad_norm": 42.90945507861596,
      "learning_rate": 6.550326657293881e-10,
      "logits/chosen": 0.80241459608078,
      "logits/rejected": 1.472318410873413,
      "logps/chosen": -426.9049377441406,
      "logps/rejected": -606.2322387695312,
      "loss": 0.3471,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -2.0855860710144043,
      "rewards/margins": 2.160794734954834,
      "rewards/rejected": -4.24638032913208,
      "step": 850
    },
    {
      "epoch": 0.99,
      "grad_norm": 52.65613220022833,
      "learning_rate": 1.2943454039654467e-10,
      "logits/chosen": 1.3825061321258545,
      "logits/rejected": 1.9333372116088867,
      "logps/chosen": -426.72332763671875,
      "logps/rejected": -573.5838623046875,
      "loss": 0.359,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -2.260263442993164,
      "rewards/margins": 1.6737343072891235,
      "rewards/rejected": -3.9339981079101562,
      "step": 860
    },
    {
      "epoch": 1.0,
      "step": 868,
      "total_flos": 0.0,
      "train_loss": 0.42703192135156026,
      "train_runtime": 13837.373,
      "train_samples_per_second": 8.031,
      "train_steps_per_second": 0.063
    }
  ],
  "logging_steps": 10,
  "max_steps": 868,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 100,
  "total_flos": 0.0,
  "train_batch_size": 8,
  "trial_name": null,
  "trial_params": null
}