{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 1.0,
  "eval_steps": 500,
  "global_step": 625,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.0,
      "learning_rate": 7.936507936507937e-08,
      "logits/chosen": 0.36046722531318665,
      "logits/rejected": 0.23892882466316223,
      "logps/chosen": -419.6329345703125,
      "logps/rejected": -437.8076171875,
      "loss": 0.3567,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 1
    },
    {
      "epoch": 0.02,
      "learning_rate": 7.936507936507937e-07,
      "logits/chosen": 0.25086185336112976,
      "logits/rejected": 0.22883599996566772,
      "logps/chosen": -438.00592041015625,
      "logps/rejected": -430.2698059082031,
      "loss": 0.3631,
      "rewards/accuracies": 0.3333333432674408,
      "rewards/chosen": -0.00057529954938218,
      "rewards/margins": 4.002878631581552e-05,
      "rewards/rejected": -0.0006153282593004405,
      "step": 10
    },
    {
      "epoch": 0.03,
      "learning_rate": 1.5873015873015873e-06,
      "logits/chosen": 0.19798453152179718,
      "logits/rejected": 0.2455386370420456,
      "logps/chosen": -401.9042663574219,
      "logps/rejected": -393.48822021484375,
      "loss": 0.3746,
      "rewards/accuracies": 0.38749998807907104,
      "rewards/chosen": -0.0006311516626738012,
      "rewards/margins": 7.311334047699347e-05,
      "rewards/rejected": -0.0007042650249786675,
      "step": 20
    },
    {
      "epoch": 0.05,
      "learning_rate": 2.380952380952381e-06,
      "logits/chosen": 0.16958989202976227,
      "logits/rejected": 0.3218519687652588,
      "logps/chosen": -421.8275451660156,
      "logps/rejected": -406.69171142578125,
      "loss": 0.3735,
      "rewards/accuracies": 0.38749998807907104,
      "rewards/chosen": -0.0018598968163132668,
      "rewards/margins": 8.136472024489194e-05,
      "rewards/rejected": -0.0019412615802139044,
      "step": 30
    },
    {
      "epoch": 0.06,
      "learning_rate": 3.1746031746031746e-06,
      "logits/chosen": 0.2768496870994568,
      "logits/rejected": 0.2004733383655548,
      "logps/chosen": -364.9704895019531,
      "logps/rejected": -383.6319274902344,
      "loss": 0.3722,
      "rewards/accuracies": 0.41874998807907104,
      "rewards/chosen": -0.0037128510884940624,
      "rewards/margins": 0.00042822357499971986,
      "rewards/rejected": -0.004141074605286121,
      "step": 40
    },
    {
      "epoch": 0.08,
      "learning_rate": 3.968253968253968e-06,
      "logits/chosen": 0.21148820221424103,
      "logits/rejected": 0.1991022676229477,
      "logps/chosen": -363.0907897949219,
      "logps/rejected": -372.4688415527344,
      "loss": 0.3683,
      "rewards/accuracies": 0.38749998807907104,
      "rewards/chosen": -0.008444704115390778,
      "rewards/margins": 0.000421268108766526,
      "rewards/rejected": -0.00886597204953432,
      "step": 50
    },
    {
      "epoch": 0.1,
      "learning_rate": 4.761904761904762e-06,
      "logits/chosen": 0.12682433426380157,
      "logits/rejected": 0.10598279535770416,
      "logps/chosen": -406.6046447753906,
      "logps/rejected": -408.136474609375,
      "loss": 0.3698,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.013907397165894508,
      "rewards/margins": 0.0012330651516094804,
      "rewards/rejected": -0.01514046173542738,
      "step": 60
    },
    {
      "epoch": 0.11,
      "learning_rate": 4.998086282661188e-06,
      "logits/chosen": 0.15495741367340088,
      "logits/rejected": 0.2392895221710205,
      "logps/chosen": -454.59564208984375,
      "logps/rejected": -472.60150146484375,
      "loss": 0.3647,
      "rewards/accuracies": 0.4937500059604645,
      "rewards/chosen": -0.01984778419137001,
      "rewards/margins": 0.0015219683991745114,
      "rewards/rejected": -0.021369751542806625,
      "step": 70
    },
    {
      "epoch": 0.13,
      "learning_rate": 4.988720025682995e-06,
      "logits/chosen": 0.10600455850362778,
      "logits/rejected": 0.19427841901779175,
      "logps/chosen": -456.883544921875,
      "logps/rejected": -457.2091369628906,
      "loss": 0.366,
      "rewards/accuracies": 0.45625001192092896,
      "rewards/chosen": -0.027602821588516235,
      "rewards/margins": 0.0021790049504488707,
      "rewards/rejected": -0.029781833291053772,
      "step": 80
    },
    {
      "epoch": 0.14,
      "learning_rate": 4.9715789537359126e-06,
      "logits/chosen": 0.19685187935829163,
      "logits/rejected": 0.1898462027311325,
      "logps/chosen": -429.89715576171875,
      "logps/rejected": -405.14947509765625,
      "loss": 0.374,
      "rewards/accuracies": 0.5062500238418579,
      "rewards/chosen": -0.02775459550321102,
      "rewards/margins": 0.0036151439417153597,
      "rewards/rejected": -0.03136973828077316,
      "step": 90
    },
    {
      "epoch": 0.16,
      "learning_rate": 4.946716615897932e-06,
      "logits/chosen": 0.17538510262966156,
      "logits/rejected": 0.20797649025917053,
      "logps/chosen": -436.5021057128906,
      "logps/rejected": -441.6522521972656,
      "loss": 0.3647,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.030668649822473526,
      "rewards/margins": 0.0047523751854896545,
      "rewards/rejected": -0.03542102500796318,
      "step": 100
    },
    {
      "epoch": 0.18,
      "learning_rate": 4.9142106826480114e-06,
      "logits/chosen": 0.23190093040466309,
      "logits/rejected": 0.2070162296295166,
      "logps/chosen": -495.9847717285156,
      "logps/rejected": -481.325927734375,
      "loss": 0.36,
      "rewards/accuracies": 0.5062500238418579,
      "rewards/chosen": -0.030186587944626808,
      "rewards/margins": 0.006671676877886057,
      "rewards/rejected": -0.036858268082141876,
      "step": 110
    },
    {
      "epoch": 0.19,
      "learning_rate": 4.874162703221823e-06,
      "logits/chosen": 0.0589652881026268,
      "logits/rejected": 0.25573208928108215,
      "logps/chosen": -414.246826171875,
      "logps/rejected": -430.9029235839844,
      "loss": 0.3575,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.04433692619204521,
      "rewards/margins": 0.015341699123382568,
      "rewards/rejected": -0.05967862531542778,
      "step": 120
    },
    {
      "epoch": 0.21,
      "learning_rate": 4.826697788369752e-06,
      "logits/chosen": 0.12831085920333862,
      "logits/rejected": 0.1346360296010971,
      "logps/chosen": -544.7901000976562,
      "logps/rejected": -542.2559814453125,
      "loss": 0.3611,
      "rewards/accuracies": 0.48750001192092896,
      "rewards/chosen": -0.09338424354791641,
      "rewards/margins": 0.020744359120726585,
      "rewards/rejected": -0.11412860453128815,
      "step": 130
    },
    {
      "epoch": 0.22,
      "learning_rate": 4.7719642195082224e-06,
      "logits/chosen": 0.004035499412566423,
      "logits/rejected": 0.12359336763620377,
      "logps/chosen": -569.91748046875,
      "logps/rejected": -576.3137817382812,
      "loss": 0.3399,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.15238870680332184,
      "rewards/margins": 0.03546822816133499,
      "rewards/rejected": -0.18785694241523743,
      "step": 140
    },
    {
      "epoch": 0.24,
      "learning_rate": 4.710132985485355e-06,
      "logits/chosen": 0.09404455125331879,
      "logits/rejected": -0.0020357524044811726,
      "logps/chosen": -502.76776123046875,
      "logps/rejected": -537.9739990234375,
      "loss": 0.3444,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.1469426155090332,
      "rewards/margins": 0.03976910561323166,
      "rewards/rejected": -0.18671169877052307,
      "step": 150
    },
    {
      "epoch": 0.26,
      "learning_rate": 4.641397248408122e-06,
      "logits/chosen": 0.10355547815561295,
      "logits/rejected": 0.07864655554294586,
      "logps/chosen": -578.9359741210938,
      "logps/rejected": -687.6909790039062,
      "loss": 0.3059,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.16360266506671906,
      "rewards/margins": 0.07588207721710205,
      "rewards/rejected": -0.2394847422838211,
      "step": 160
    },
    {
      "epoch": 0.27,
      "learning_rate": 4.5659717401997655e-06,
      "logits/chosen": 0.032590705901384354,
      "logits/rejected": 0.08898362517356873,
      "logps/chosen": -600.9769287109375,
      "logps/rejected": -670.7233276367188,
      "loss": 0.3212,
      "rewards/accuracies": 0.48124998807907104,
      "rewards/chosen": -0.19255095720291138,
      "rewards/margins": 0.08060728758573532,
      "rewards/rejected": -0.2731582522392273,
      "step": 170
    },
    {
      "epoch": 0.29,
      "learning_rate": 4.4840920917726425e-06,
      "logits/chosen": 0.04442215710878372,
      "logits/rejected": 0.042511895298957825,
      "logps/chosen": -588.660888671875,
      "logps/rejected": -651.74853515625,
      "loss": 0.3528,
      "rewards/accuracies": 0.42500001192092896,
      "rewards/chosen": -0.20966176688671112,
      "rewards/margins": 0.05220552161335945,
      "rewards/rejected": -0.2618672847747803,
      "step": 180
    },
    {
      "epoch": 0.3,
      "learning_rate": 4.396014096912182e-06,
      "logits/chosen": 0.03010064922273159,
      "logits/rejected": 0.041020795702934265,
      "logps/chosen": -532.5487060546875,
      "logps/rejected": -654.7836303710938,
      "loss": 0.3144,
      "rewards/accuracies": 0.518750011920929,
      "rewards/chosen": -0.18499326705932617,
      "rewards/margins": 0.09925045073032379,
      "rewards/rejected": -0.28424373269081116,
      "step": 190
    },
    {
      "epoch": 0.32,
      "learning_rate": 4.302012913171584e-06,
      "logits/chosen": -0.03510774299502373,
      "logits/rejected": 0.035278841853141785,
      "logps/chosen": -640.3848876953125,
      "logps/rejected": -709.2119140625,
      "loss": 0.3341,
      "rewards/accuracies": 0.48124998807907104,
      "rewards/chosen": -0.21959897875785828,
      "rewards/margins": 0.06892947107553482,
      "rewards/rejected": -0.2885284423828125,
      "step": 200
    },
    {
      "epoch": 0.34,
      "learning_rate": 4.202382202273702e-06,
      "logits/chosen": 0.017812447622418404,
      "logits/rejected": 0.06379534304141998,
      "logps/chosen": -641.2015991210938,
      "logps/rejected": -672.9808349609375,
      "loss": 0.3513,
      "rewards/accuracies": 0.4937500059604645,
      "rewards/chosen": -0.19523058831691742,
      "rewards/margins": 0.05737825483083725,
      "rewards/rejected": -0.2526088356971741,
      "step": 210
    },
    {
      "epoch": 0.35,
      "learning_rate": 4.097433212705492e-06,
      "logits/chosen": 0.0030833049677312374,
      "logits/rejected": 0.027691733092069626,
      "logps/chosen": -645.5457763671875,
      "logps/rejected": -751.8395385742188,
      "loss": 0.3278,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.1966964602470398,
      "rewards/margins": 0.08199279010295868,
      "rewards/rejected": -0.2786892056465149,
      "step": 220
    },
    {
      "epoch": 0.37,
      "learning_rate": 3.987493807371033e-06,
      "logits/chosen": 0.06291202455759048,
      "logits/rejected": -0.08009441941976547,
      "logps/chosen": -500.90496826171875,
      "logps/rejected": -572.1060791015625,
      "loss": 0.3343,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.14460867643356323,
      "rewards/margins": 0.05721248313784599,
      "rewards/rejected": -0.20182117819786072,
      "step": 230
    },
    {
      "epoch": 0.38,
      "learning_rate": 3.872907439340758e-06,
      "logits/chosen": 0.04002127796411514,
      "logits/rejected": -0.06345082819461823,
      "logps/chosen": -543.0759887695312,
      "logps/rejected": -591.0164184570312,
      "loss": 0.3242,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": -0.1588135063648224,
      "rewards/margins": 0.05112007260322571,
      "rewards/rejected": -0.2099335640668869,
      "step": 240
    },
    {
      "epoch": 0.4,
      "learning_rate": 3.75403207889666e-06,
      "logits/chosen": 0.10061119496822357,
      "logits/rejected": 0.04563102871179581,
      "logps/chosen": -578.7415771484375,
      "logps/rejected": -615.0494384765625,
      "loss": 0.3244,
      "rewards/accuracies": 0.4437499940395355,
      "rewards/chosen": -0.1841258704662323,
      "rewards/margins": 0.055299896746873856,
      "rewards/rejected": -0.23942574858665466,
      "step": 250
    },
    {
      "epoch": 0.42,
      "learning_rate": 3.631239095225417e-06,
      "logits/chosen": 0.0002536162792239338,
      "logits/rejected": 0.06318163126707077,
      "logps/chosen": -650.6129150390625,
      "logps/rejected": -664.8394775390625,
      "loss": 0.3604,
      "rewards/accuracies": 0.518750011920929,
      "rewards/chosen": -0.1936233639717102,
      "rewards/margins": 0.039541881531476974,
      "rewards/rejected": -0.23316523432731628,
      "step": 260
    },
    {
      "epoch": 0.43,
      "learning_rate": 3.5049120962530608e-06,
      "logits/chosen": 0.04564657807350159,
      "logits/rejected": -0.02100910246372223,
      "logps/chosen": -597.6544799804688,
      "logps/rejected": -677.6250610351562,
      "loss": 0.3337,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -0.18195489048957825,
      "rewards/margins": 0.07418446242809296,
      "rewards/rejected": -0.2561393678188324,
      "step": 270
    },
    {
      "epoch": 0.45,
      "learning_rate": 3.3754457302455464e-06,
      "logits/chosen": -0.03575817123055458,
      "logits/rejected": 0.11129350960254669,
      "logps/chosen": -567.7127685546875,
      "logps/rejected": -608.2735595703125,
      "loss": 0.3427,
      "rewards/accuracies": 0.4312500059604645,
      "rewards/chosen": -0.18929126858711243,
      "rewards/margins": 0.05334322527050972,
      "rewards/rejected": -0.24263450503349304,
      "step": 280
    },
    {
      "epoch": 0.46,
      "learning_rate": 3.2432444529190714e-06,
      "logits/chosen": -0.05213651806116104,
      "logits/rejected": 0.0017232149839401245,
      "logps/chosen": -600.5894165039062,
      "logps/rejected": -662.463623046875,
      "loss": 0.3298,
      "rewards/accuracies": 0.5062500238418579,
      "rewards/chosen": -0.16451598703861237,
      "rewards/margins": 0.06247172877192497,
      "rewards/rejected": -0.22698768973350525,
      "step": 290
    },
    {
      "epoch": 0.48,
      "learning_rate": 3.1087212639117057e-06,
      "logits/chosen": 0.04510731250047684,
      "logits/rejected": 0.045666079968214035,
      "logps/chosen": -590.2169189453125,
      "logps/rejected": -643.4014892578125,
      "loss": 0.3287,
      "rewards/accuracies": 0.4937500059604645,
      "rewards/chosen": -0.18261924386024475,
      "rewards/margins": 0.05223492905497551,
      "rewards/rejected": -0.23485419154167175,
      "step": 300
    },
    {
      "epoch": 0.5,
      "learning_rate": 2.9722964165636263e-06,
      "logits/chosen": 0.006776309106498957,
      "logits/rejected": -0.025738414376974106,
      "logps/chosen": -530.9435424804688,
      "logps/rejected": -585.0616455078125,
      "loss": 0.3287,
      "rewards/accuracies": 0.46875,
      "rewards/chosen": -0.15342922508716583,
      "rewards/margins": 0.056588392704725266,
      "rewards/rejected": -0.2100176066160202,
      "step": 310
    },
    {
      "epoch": 0.51,
      "learning_rate": 2.8343961050366275e-06,
      "logits/chosen": -0.02025730162858963,
      "logits/rejected": 0.08972244709730148,
      "logps/chosen": -529.8350830078125,
      "logps/rejected": -604.9669799804688,
      "loss": 0.3274,
      "rewards/accuracies": 0.4937500059604645,
      "rewards/chosen": -0.16149136424064636,
      "rewards/margins": 0.06798367202281952,
      "rewards/rejected": -0.22947505116462708,
      "step": 320
    },
    {
      "epoch": 0.53,
      "learning_rate": 2.695451132874385e-06,
      "logits/chosen": -0.0479627326130867,
      "logits/rejected": 0.021157974377274513,
      "logps/chosen": -564.2462158203125,
      "logps/rejected": -604.8670654296875,
      "loss": 0.3409,
      "rewards/accuracies": 0.42500001192092896,
      "rewards/chosen": -0.17655444145202637,
      "rewards/margins": 0.06839374452829361,
      "rewards/rejected": -0.24494817852973938,
      "step": 330
    },
    {
      "epoch": 0.54,
      "learning_rate": 2.5558955671628964e-06,
      "logits/chosen": -0.053826987743377686,
      "logits/rejected": 0.036486681550741196,
      "logps/chosen": -616.4885864257812,
      "logps/rejected": -669.8521728515625,
      "loss": 0.3326,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.20758116245269775,
      "rewards/margins": 0.050070859491825104,
      "rewards/rejected": -0.25765201449394226,
      "step": 340
    },
    {
      "epoch": 0.56,
      "learning_rate": 2.4161653824955654e-06,
      "logits/chosen": -0.025303969159722328,
      "logits/rejected": -0.017024224624037743,
      "logps/chosen": -617.4375610351562,
      "logps/rejected": -696.9207153320312,
      "loss": 0.3223,
      "rewards/accuracies": 0.518750011920929,
      "rewards/chosen": -0.19740080833435059,
      "rewards/margins": 0.08515138924121857,
      "rewards/rejected": -0.28255218267440796,
      "step": 350
    },
    {
      "epoch": 0.58,
      "learning_rate": 2.2766970989791697e-06,
      "logits/chosen": -0.07398111373186111,
      "logits/rejected": -0.03359592705965042,
      "logps/chosen": -612.1632690429688,
      "logps/rejected": -696.7496337890625,
      "loss": 0.3282,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.20693747699260712,
      "rewards/margins": 0.08319561928510666,
      "rewards/rejected": -0.2901330888271332,
      "step": 360
    },
    {
      "epoch": 0.59,
      "learning_rate": 2.1379264185356545e-06,
      "logits/chosen": -0.06404633074998856,
      "logits/rejected": 0.025870006531476974,
      "logps/chosen": -648.0364379882812,
      "logps/rejected": -723.4362182617188,
      "loss": 0.3191,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -0.225542813539505,
      "rewards/margins": 0.09016820043325424,
      "rewards/rejected": -0.31571102142333984,
      "step": 370
    },
    {
      "epoch": 0.61,
      "learning_rate": 2.000286863759934e-06,
      "logits/chosen": 0.06071531027555466,
      "logits/rejected": 0.017028797417879105,
      "logps/chosen": -611.7649536132812,
      "logps/rejected": -710.9679565429688,
      "loss": 0.3274,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -0.19213876128196716,
      "rewards/margins": 0.08261444419622421,
      "rewards/rejected": -0.27475324273109436,
      "step": 380
    },
    {
      "epoch": 0.62,
      "learning_rate": 1.8642084235859764e-06,
      "logits/chosen": 0.0644644945859909,
      "logits/rejected": -0.021000124514102936,
      "logps/chosen": -533.0925903320312,
      "logps/rejected": -604.2486572265625,
      "loss": 0.3359,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.17501072585582733,
      "rewards/margins": 0.06491357088088989,
      "rewards/rejected": -0.23992428183555603,
      "step": 390
    },
    {
      "epoch": 0.64,
      "learning_rate": 1.7301162099921013e-06,
      "logits/chosen": -0.021198943257331848,
      "logits/rejected": 0.045659858733415604,
      "logps/chosen": -591.815185546875,
      "logps/rejected": -630.9583129882812,
      "loss": 0.328,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": -0.18797656893730164,
      "rewards/margins": 0.05078692361712456,
      "rewards/rejected": -0.23876352608203888,
      "step": 400
    },
    {
      "epoch": 0.66,
      "learning_rate": 1.5984291299420117e-06,
      "logits/chosen": -0.06265034526586533,
      "logits/rejected": -0.03586529195308685,
      "logps/chosen": -620.4393310546875,
      "logps/rejected": -664.763916015625,
      "loss": 0.3131,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.20063380897045135,
      "rewards/margins": 0.056611478328704834,
      "rewards/rejected": -0.2572453022003174,
      "step": 410
    },
    {
      "epoch": 0.67,
      "learning_rate": 1.4695585767104092e-06,
      "logits/chosen": -0.02047324925661087,
      "logits/rejected": -0.022855054587125778,
      "logps/chosen": -573.5113525390625,
      "logps/rejected": -660.650390625,
      "loss": 0.315,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.1739957630634308,
      "rewards/margins": 0.0810604989528656,
      "rewards/rejected": -0.2550562918186188,
      "step": 420
    },
    {
      "epoch": 0.69,
      "learning_rate": 1.3439071446815452e-06,
      "logits/chosen": 0.018320618197321892,
      "logits/rejected": -0.02154999040067196,
      "logps/chosen": -510.105224609375,
      "logps/rejected": -601.5553588867188,
      "loss": 0.333,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.1630430966615677,
      "rewards/margins": 0.07893416285514832,
      "rewards/rejected": -0.241977259516716,
      "step": 430
    },
    {
      "epoch": 0.7,
      "learning_rate": 1.2218673716356919e-06,
      "logits/chosen": 0.02131899632513523,
      "logits/rejected": 0.102879598736763,
      "logps/chosen": -642.7557373046875,
      "logps/rejected": -687.9267578125,
      "loss": 0.3358,
      "rewards/accuracies": 0.4937500059604645,
      "rewards/chosen": -0.2222980558872223,
      "rewards/margins": 0.061403900384902954,
      "rewards/rejected": -0.28370195627212524,
      "step": 440
    },
    {
      "epoch": 0.72,
      "learning_rate": 1.103820512452661e-06,
      "logits/chosen": -0.02241530455648899,
      "logits/rejected": -0.032763995230197906,
      "logps/chosen": -534.8072509765625,
      "logps/rejected": -587.8253173828125,
      "loss": 0.3429,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.18417826294898987,
      "rewards/margins": 0.06289814412593842,
      "rewards/rejected": -0.2470763921737671,
      "step": 450
    },
    {
      "epoch": 0.74,
      "learning_rate": 9.901353480633468e-07,
      "logits/chosen": -0.029488349333405495,
      "logits/rejected": -0.008380506187677383,
      "logps/chosen": -640.1456298828125,
      "logps/rejected": -714.1085205078125,
      "loss": 0.3348,
      "rewards/accuracies": 0.5062500238418579,
      "rewards/chosen": -0.2099982500076294,
      "rewards/margins": 0.06554147601127625,
      "rewards/rejected": -0.27553972601890564,
      "step": 460
    },
    {
      "epoch": 0.75,
      "learning_rate": 8.811670333701544e-07,
      "logits/chosen": -0.09662221372127533,
      "logits/rejected": -0.06438769400119781,
      "logps/chosen": -583.837158203125,
      "logps/rejected": -702.3072509765625,
      "loss": 0.3233,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.18826954066753387,
      "rewards/margins": 0.0807666927576065,
      "rewards/rejected": -0.2690362334251404,
      "step": 470
    },
    {
      "epoch": 0.77,
      "learning_rate": 7.772559877354341e-07,
      "logits/chosen": -0.057563841342926025,
      "logits/rejected": 0.0010903537040576339,
      "logps/chosen": -573.5906372070312,
      "logps/rejected": -646.9957885742188,
      "loss": 0.3245,
      "rewards/accuracies": 0.4937500059604645,
      "rewards/chosen": -0.17442554235458374,
      "rewards/margins": 0.08631418645381927,
      "rewards/rejected": -0.2607397139072418,
      "step": 480
    },
    {
      "epoch": 0.78,
      "learning_rate": 6.787268315040604e-07,
      "logits/chosen": -0.06349989026784897,
      "logits/rejected": 0.024532323703169823,
      "logps/chosen": -573.9180908203125,
      "logps/rejected": -613.2721557617188,
      "loss": 0.3313,
      "rewards/accuracies": 0.4312500059604645,
      "rewards/chosen": -0.18243160843849182,
      "rewards/margins": 0.051405251026153564,
      "rewards/rejected": -0.2338368445634842,
      "step": 490
    },
    {
      "epoch": 0.8,
      "learning_rate": 5.858873718824829e-07,
      "logits/chosen": -0.06668631732463837,
      "logits/rejected": -0.06943139433860779,
      "logps/chosen": -553.5023193359375,
      "logps/rejected": -593.6824951171875,
      "loss": 0.3319,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.17030836641788483,
      "rewards/margins": 0.0499136857688427,
      "rewards/rejected": -0.22022204101085663,
      "step": 500
    },
    {
      "epoch": 0.82,
      "learning_rate": 4.990276413423817e-07,
      "logits/chosen": -0.08017369359731674,
      "logits/rejected": -0.02853136695921421,
      "logps/chosen": -577.9718017578125,
      "logps/rejected": -648.7777099609375,
      "loss": 0.3326,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.19647789001464844,
      "rewards/margins": 0.058233268558979034,
      "rewards/rejected": -0.2547111511230469,
      "step": 510
    },
    {
      "epoch": 0.83,
      "learning_rate": 4.184189915529796e-07,
      "logits/chosen": -0.1332094371318817,
      "logits/rejected": 0.09619398415088654,
      "logps/chosen": -579.1356201171875,
      "logps/rejected": -609.83544921875,
      "loss": 0.3251,
      "rewards/accuracies": 0.5062500238418579,
      "rewards/chosen": -0.16035783290863037,
      "rewards/margins": 0.07735449075698853,
      "rewards/rejected": -0.2377123385667801,
      "step": 520
    },
    {
      "epoch": 0.85,
      "learning_rate": 3.4431324567258176e-07,
      "logits/chosen": -0.11564228683710098,
      "logits/rejected": -0.10405786335468292,
      "logps/chosen": -530.2095947265625,
      "logps/rejected": -617.2594604492188,
      "loss": 0.3344,
      "rewards/accuracies": 0.48750001192092896,
      "rewards/chosen": -0.17116107046604156,
      "rewards/margins": 0.08385736495256424,
      "rewards/rejected": -0.255018413066864,
      "step": 530
    },
    {
      "epoch": 0.86,
      "learning_rate": 2.769419116476052e-07,
      "logits/chosen": -0.05448366329073906,
      "logits/rejected": 0.020873898640275,
      "logps/chosen": -649.448486328125,
      "logps/rejected": -724.3923950195312,
      "loss": 0.3254,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -0.20293286442756653,
      "rewards/margins": 0.0811840295791626,
      "rewards/rejected": -0.2841168940067291,
      "step": 540
    },
    {
      "epoch": 0.88,
      "learning_rate": 2.1651545897676512e-07,
      "logits/chosen": -0.09775115549564362,
      "logits/rejected": 0.037994496524333954,
      "logps/chosen": -631.9583740234375,
      "logps/rejected": -665.892578125,
      "loss": 0.3147,
      "rewards/accuracies": 0.48750001192092896,
      "rewards/chosen": -0.192877396941185,
      "rewards/margins": 0.06704525649547577,
      "rewards/rejected": -0.25992268323898315,
      "step": 550
    },
    {
      "epoch": 0.9,
      "learning_rate": 1.6322266119983222e-07,
      "logits/chosen": -0.05889749526977539,
      "logits/rejected": 0.006100955419242382,
      "logps/chosen": -603.3699951171875,
      "logps/rejected": -652.2211303710938,
      "loss": 0.3269,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": -0.18066729605197906,
      "rewards/margins": 0.0697588175535202,
      "rewards/rejected": -0.25042611360549927,
      "step": 560
    },
    {
      "epoch": 0.91,
      "learning_rate": 1.1723000616502167e-07,
      "logits/chosen": -0.020711831748485565,
      "logits/rejected": 0.08241745829582214,
      "logps/chosen": -596.9910888671875,
      "logps/rejected": -620.6817626953125,
      "loss": 0.3278,
      "rewards/accuracies": 0.46875,
      "rewards/chosen": -0.21281632781028748,
      "rewards/margins": 0.04477155953645706,
      "rewards/rejected": -0.25758787989616394,
      "step": 570
    },
    {
      "epoch": 0.93,
      "learning_rate": 7.868117591737585e-08,
      "logits/chosen": -0.09641040116548538,
      "logits/rejected": -0.06558734178543091,
      "logps/chosen": -527.1932373046875,
      "logps/rejected": -567.8707275390625,
      "loss": 0.3403,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.1879841685295105,
      "rewards/margins": 0.048898279666900635,
      "rewards/rejected": -0.23688244819641113,
      "step": 580
    },
    {
      "epoch": 0.94,
      "learning_rate": 4.769659783295383e-08,
      "logits/chosen": -0.08810693025588989,
      "logits/rejected": -0.04142559692263603,
      "logps/chosen": -554.7185668945312,
      "logps/rejected": -658.844482421875,
      "loss": 0.3395,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -0.17377179861068726,
      "rewards/margins": 0.08222335577011108,
      "rewards/rejected": -0.25599512457847595,
      "step": 590
    },
    {
      "epoch": 0.96,
      "learning_rate": 2.4373068401120358e-08,
      "logits/chosen": -0.02737409994006157,
      "logits/rejected": -0.07764561474323273,
      "logps/chosen": -561.8909912109375,
      "logps/rejected": -653.6951904296875,
      "loss": 0.3241,
      "rewards/accuracies": 0.4937500059604645,
      "rewards/chosen": -0.18414655327796936,
      "rewards/margins": 0.0771530419588089,
      "rewards/rejected": -0.26129958033561707,
      "step": 600
    },
    {
      "epoch": 0.98,
      "learning_rate": 8.78345083022425e-09,
      "logits/chosen": -0.1056148037314415,
      "logits/rejected": 0.01646628975868225,
      "logps/chosen": -578.94970703125,
      "logps/rejected": -632.1832275390625,
      "loss": 0.3343,
      "rewards/accuracies": 0.4625000059604645,
      "rewards/chosen": -0.18656322360038757,
      "rewards/margins": 0.06646813452243805,
      "rewards/rejected": -0.25303134322166443,
      "step": 610
    },
    {
      "epoch": 0.99,
      "learning_rate": 9.764474213677654e-10,
      "logits/chosen": -0.047748979181051254,
      "logits/rejected": -0.0808296799659729,
      "logps/chosen": -601.5391235351562,
      "logps/rejected": -620.3516845703125,
      "loss": 0.3484,
      "rewards/accuracies": 0.41874998807907104,
      "rewards/chosen": -0.20741529762744904,
      "rewards/margins": 0.029957950115203857,
      "rewards/rejected": -0.2373732626438141,
      "step": 620
    },
    {
      "epoch": 1.0,
      "step": 625,
      "total_flos": 0.0,
      "train_loss": 0.3386969359874725,
      "train_runtime": 4609.7316,
      "train_samples_per_second": 4.339,
      "train_steps_per_second": 0.136
    }
  ],
  "logging_steps": 10,
  "max_steps": 625,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 100,
  "total_flos": 0.0,
  "train_batch_size": 4,
  "trial_name": null,
  "trial_params": null
}