{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 0.31405391258832765,
  "eval_steps": 400,
  "global_step": 300,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.005234231876472127,
      "grad_norm": 538.2121172028135,
      "learning_rate": 1.5706806282722512e-08,
      "loss": 712.4422,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -2.762140989303589,
      "rewards/margins": -0.44960299134254456,
      "rewards/rejected": -2.3125383853912354,
      "step": 5
    },
    {
      "epoch": 0.010468463752944255,
      "grad_norm": 769.4153649618398,
      "learning_rate": 3.1413612565445024e-08,
      "loss": 714.8706,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -3.2879066467285156,
      "rewards/margins": -0.45820775628089905,
      "rewards/rejected": -2.8296990394592285,
      "step": 10
    },
    {
      "epoch": 0.015702695629416383,
      "grad_norm": 684.4934453698337,
      "learning_rate": 4.7120418848167537e-08,
      "loss": 719.9062,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -3.12087082862854,
      "rewards/margins": -0.619767427444458,
      "rewards/rejected": -2.501103639602661,
      "step": 15
    },
    {
      "epoch": 0.02093692750588851,
      "grad_norm": 604.4540906941717,
      "learning_rate": 6.282722513089005e-08,
      "loss": 714.8108,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -2.9811909198760986,
      "rewards/margins": -0.26270246505737305,
      "rewards/rejected": -2.7184884548187256,
      "step": 20
    },
    {
      "epoch": 0.02617115938236064,
      "grad_norm": 610.5391149495696,
      "learning_rate": 7.853403141361257e-08,
      "loss": 706.6332,
      "rewards/accuracies": 0.4625000059604645,
      "rewards/chosen": -2.7390408515930176,
      "rewards/margins": -0.2960797846317291,
      "rewards/rejected": -2.4429612159729004,
      "step": 25
    },
    {
      "epoch": 0.031405391258832765,
      "grad_norm": 810.2580108311985,
      "learning_rate": 9.424083769633507e-08,
      "loss": 720.5581,
      "rewards/accuracies": 0.42500001192092896,
      "rewards/chosen": -2.9331297874450684,
      "rewards/margins": -0.5686414837837219,
      "rewards/rejected": -2.364488124847412,
      "step": 30
    },
    {
      "epoch": 0.036639623135304895,
      "grad_norm": 1306.5067007136959,
      "learning_rate": 1.0994764397905759e-07,
      "loss": 710.7288,
      "rewards/accuracies": 0.42500001192092896,
      "rewards/chosen": -2.85563325881958,
      "rewards/margins": -0.544287383556366,
      "rewards/rejected": -2.3113455772399902,
      "step": 35
    },
    {
      "epoch": 0.04187385501177702,
      "grad_norm": 599.7562132661373,
      "learning_rate": 1.256544502617801e-07,
      "loss": 714.8763,
      "rewards/accuracies": 0.36250001192092896,
      "rewards/chosen": -3.080730676651001,
      "rewards/margins": -0.7031532526016235,
      "rewards/rejected": -2.377577543258667,
      "step": 40
    },
    {
      "epoch": 0.04710808688824915,
      "grad_norm": 653.0162673671618,
      "learning_rate": 1.413612565445026e-07,
      "loss": 708.652,
      "rewards/accuracies": 0.4625000059604645,
      "rewards/chosen": -2.987445116043091,
      "rewards/margins": -0.3784741163253784,
      "rewards/rejected": -2.608970880508423,
      "step": 45
    },
    {
      "epoch": 0.05234231876472128,
      "grad_norm": 536.6525222015883,
      "learning_rate": 1.5706806282722514e-07,
      "loss": 705.5576,
      "rewards/accuracies": 0.48750001192092896,
      "rewards/chosen": -2.938387870788574,
      "rewards/margins": -0.057790376245975494,
      "rewards/rejected": -2.8805975914001465,
      "step": 50
    },
    {
      "epoch": 0.05757655064119341,
      "grad_norm": 606.538624615443,
      "learning_rate": 1.7277486910994764e-07,
      "loss": 721.314,
      "rewards/accuracies": 0.48750001192092896,
      "rewards/chosen": -3.0540528297424316,
      "rewards/margins": -0.5808142423629761,
      "rewards/rejected": -2.473238945007324,
      "step": 55
    },
    {
      "epoch": 0.06281078251766553,
      "grad_norm": 48640.63265622123,
      "learning_rate": 1.8848167539267015e-07,
      "loss": 709.0939,
      "rewards/accuracies": 0.4124999940395355,
      "rewards/chosen": -2.787937641143799,
      "rewards/margins": -0.3062622547149658,
      "rewards/rejected": -2.481675624847412,
      "step": 60
    },
    {
      "epoch": 0.06804501439413765,
      "grad_norm": 578.2635720627949,
      "learning_rate": 2.0418848167539268e-07,
      "loss": 720.4865,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -2.52620005607605,
      "rewards/margins": -0.3670107424259186,
      "rewards/rejected": -2.159188985824585,
      "step": 65
    },
    {
      "epoch": 0.07327924627060979,
      "grad_norm": 564.9512121020007,
      "learning_rate": 2.1989528795811518e-07,
      "loss": 714.9839,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -3.038364887237549,
      "rewards/margins": -0.08167771995067596,
      "rewards/rejected": -2.9566872119903564,
      "step": 70
    },
    {
      "epoch": 0.07851347814708191,
      "grad_norm": 550.0229357141382,
      "learning_rate": 2.356020942408377e-07,
      "loss": 707.9901,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -2.9208950996398926,
      "rewards/margins": 0.02573840692639351,
      "rewards/rejected": -2.9466335773468018,
      "step": 75
    },
    {
      "epoch": 0.08374771002355404,
      "grad_norm": 555.8351490149286,
      "learning_rate": 2.513089005235602e-07,
      "loss": 717.3213,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": -2.6972522735595703,
      "rewards/margins": -0.3343200981616974,
      "rewards/rejected": -2.3629322052001953,
      "step": 80
    },
    {
      "epoch": 0.08898194190002617,
      "grad_norm": 640.8944737782464,
      "learning_rate": 2.6701570680628273e-07,
      "loss": 712.8795,
      "rewards/accuracies": 0.4625000059604645,
      "rewards/chosen": -3.1766231060028076,
      "rewards/margins": -0.43173903226852417,
      "rewards/rejected": -2.7448840141296387,
      "step": 85
    },
    {
      "epoch": 0.0942161737764983,
      "grad_norm": 577.9257459351567,
      "learning_rate": 2.827225130890052e-07,
      "loss": 710.1004,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -3.379631757736206,
      "rewards/margins": -0.23687522113323212,
      "rewards/rejected": -3.142756700515747,
      "step": 90
    },
    {
      "epoch": 0.09945040565297043,
      "grad_norm": 613.8797276617438,
      "learning_rate": 2.9842931937172774e-07,
      "loss": 708.4406,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -2.8850560188293457,
      "rewards/margins": -0.02148260548710823,
      "rewards/rejected": -2.8635735511779785,
      "step": 95
    },
    {
      "epoch": 0.10468463752944256,
      "grad_norm": 596.094092017055,
      "learning_rate": 3.1413612565445027e-07,
      "loss": 701.8163,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -2.5895133018493652,
      "rewards/margins": 0.21284011006355286,
      "rewards/rejected": -2.802353620529175,
      "step": 100
    },
    {
      "epoch": 0.10991886940591468,
      "grad_norm": 575.5375774685907,
      "learning_rate": 3.2984293193717275e-07,
      "loss": 702.0941,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -2.9948737621307373,
      "rewards/margins": -0.251115620136261,
      "rewards/rejected": -2.7437584400177,
      "step": 105
    },
    {
      "epoch": 0.11515310128238682,
      "grad_norm": 533.700113274654,
      "learning_rate": 3.455497382198953e-07,
      "loss": 709.0662,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -2.807100772857666,
      "rewards/margins": 0.0010332107776775956,
      "rewards/rejected": -2.808133840560913,
      "step": 110
    },
    {
      "epoch": 0.12038733315885894,
      "grad_norm": 988.011155930704,
      "learning_rate": 3.612565445026178e-07,
      "loss": 713.1928,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": -2.714513063430786,
      "rewards/margins": 0.04552131146192551,
      "rewards/rejected": -2.7600343227386475,
      "step": 115
    },
    {
      "epoch": 0.12562156503533106,
      "grad_norm": 739.5138308539728,
      "learning_rate": 3.769633507853403e-07,
      "loss": 705.3437,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -2.5651416778564453,
      "rewards/margins": -0.2595147490501404,
      "rewards/rejected": -2.305626630783081,
      "step": 120
    },
    {
      "epoch": 0.13085579691180318,
      "grad_norm": 625.5739656998142,
      "learning_rate": 3.9267015706806277e-07,
      "loss": 706.4035,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -2.9260237216949463,
      "rewards/margins": 0.40523427724838257,
      "rewards/rejected": -3.3312580585479736,
      "step": 125
    },
    {
      "epoch": 0.1360900287882753,
      "grad_norm": 903.5093282527025,
      "learning_rate": 4.0837696335078536e-07,
      "loss": 701.3129,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -3.2097415924072266,
      "rewards/margins": 0.24607494473457336,
      "rewards/rejected": -3.4558162689208984,
      "step": 130
    },
    {
      "epoch": 0.14132426066474746,
      "grad_norm": 535.0167689463523,
      "learning_rate": 4.2408376963350784e-07,
      "loss": 703.7639,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -2.827212333679199,
      "rewards/margins": -0.04072323814034462,
      "rewards/rejected": -2.7864890098571777,
      "step": 135
    },
    {
      "epoch": 0.14655849254121958,
      "grad_norm": 1108.591468284195,
      "learning_rate": 4.3979057591623037e-07,
      "loss": 716.2931,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": -2.887146234512329,
      "rewards/margins": -0.05500054359436035,
      "rewards/rejected": -2.8321454524993896,
      "step": 140
    },
    {
      "epoch": 0.1517927244176917,
      "grad_norm": 6409.400405912346,
      "learning_rate": 4.5549738219895285e-07,
      "loss": 712.7188,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": -2.9499645233154297,
      "rewards/margins": -0.23646759986877441,
      "rewards/rejected": -2.7134969234466553,
      "step": 145
    },
    {
      "epoch": 0.15702695629416383,
      "grad_norm": 737.0739115110638,
      "learning_rate": 4.712041884816754e-07,
      "loss": 701.2093,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": -3.3118796348571777,
      "rewards/margins": -0.31808969378471375,
      "rewards/rejected": -2.9937901496887207,
      "step": 150
    },
    {
      "epoch": 0.16226118817063595,
      "grad_norm": 11958.927252863188,
      "learning_rate": 4.869109947643979e-07,
      "loss": 716.3511,
      "rewards/accuracies": 0.4124999940395355,
      "rewards/chosen": -3.256643772125244,
      "rewards/margins": -0.235758975148201,
      "rewards/rejected": -3.0208847522735596,
      "step": 155
    },
    {
      "epoch": 0.16749542004710807,
      "grad_norm": 878.9392827755225,
      "learning_rate": 5.026178010471204e-07,
      "loss": 705.1063,
      "rewards/accuracies": 0.48750001192092896,
      "rewards/chosen": -3.146069049835205,
      "rewards/margins": -0.3563377857208252,
      "rewards/rejected": -2.789731502532959,
      "step": 160
    },
    {
      "epoch": 0.17272965192358022,
      "grad_norm": 1476.135035585214,
      "learning_rate": 5.183246073298429e-07,
      "loss": 449372313.6,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -2.847057342529297,
      "rewards/margins": 0.0032722472678869963,
      "rewards/rejected": -2.8503289222717285,
      "step": 165
    },
    {
      "epoch": 0.17796388380005235,
      "grad_norm": 2784.764446013241,
      "learning_rate": 5.340314136125655e-07,
      "loss": 1420.048,
      "rewards/accuracies": 0.48750001192092896,
      "rewards/chosen": -3.036407947540283,
      "rewards/margins": -0.07195933163166046,
      "rewards/rejected": -2.9644484519958496,
      "step": 170
    },
    {
      "epoch": 0.18319811567652447,
      "grad_norm": 815.3571941273965,
      "learning_rate": 5.497382198952879e-07,
      "loss": 721.8058,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -3.4207043647766113,
      "rewards/margins": -0.5545729398727417,
      "rewards/rejected": -2.86613130569458,
      "step": 175
    },
    {
      "epoch": 0.1884323475529966,
      "grad_norm": 2978.504981388167,
      "learning_rate": 5.654450261780104e-07,
      "loss": 706.5416,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -3.202298641204834,
      "rewards/margins": -0.1947997510433197,
      "rewards/rejected": -3.0074992179870605,
      "step": 180
    },
    {
      "epoch": 0.19366657942946872,
      "grad_norm": 4490.800709074202,
      "learning_rate": 5.811518324607329e-07,
      "loss": 1582.3522,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -2.7045609951019287,
      "rewards/margins": 0.35880425572395325,
      "rewards/rejected": -3.0633649826049805,
      "step": 185
    },
    {
      "epoch": 0.19890081130594087,
      "grad_norm": 1493.9599531264716,
      "learning_rate": 5.968586387434555e-07,
      "loss": 710.4521,
      "rewards/accuracies": 0.42500001192092896,
      "rewards/chosen": -3.22709584236145,
      "rewards/margins": -0.09556224942207336,
      "rewards/rejected": -3.131533145904541,
      "step": 190
    },
    {
      "epoch": 0.204135043182413,
      "grad_norm": 1648.907933222567,
      "learning_rate": 5.999919840075465e-07,
      "loss": 720.1209,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -2.912754535675049,
      "rewards/margins": -0.15186476707458496,
      "rewards/rejected": -2.7608895301818848,
      "step": 195
    },
    {
      "epoch": 0.2093692750588851,
      "grad_norm": 121118.77200816409,
      "learning_rate": 5.999594197723797e-07,
      "loss": 1838.7574,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -3.3992724418640137,
      "rewards/margins": -0.38026368618011475,
      "rewards/rejected": -3.0190091133117676,
      "step": 200
    },
    {
      "epoch": 0.21460350693535724,
      "grad_norm": 1762.1952155814568,
      "learning_rate": 5.999018090119838e-07,
      "loss": 677.926,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -2.8025829792022705,
      "rewards/margins": 0.26382964849472046,
      "rewards/rejected": -3.0664122104644775,
      "step": 205
    },
    {
      "epoch": 0.21983773881182936,
      "grad_norm": 2775.2012036512388,
      "learning_rate": 5.998191565368407e-07,
      "loss": 926.7622,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -3.3869636058807373,
      "rewards/margins": -0.10393693298101425,
      "rewards/rejected": -3.283027172088623,
      "step": 210
    },
    {
      "epoch": 0.22507197068830148,
      "grad_norm": 2094.318880812516,
      "learning_rate": 5.99711469248408e-07,
      "loss": 705.6884,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": -3.8220913410186768,
      "rewards/margins": -0.42126813530921936,
      "rewards/rejected": -3.4008231163024902,
      "step": 215
    },
    {
      "epoch": 0.23030620256477363,
      "grad_norm": 3420.7148646185233,
      "learning_rate": 5.995787561385437e-07,
      "loss": 40136.0437,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -3.344470500946045,
      "rewards/margins": -0.2966476082801819,
      "rewards/rejected": -3.047822952270508,
      "step": 220
    },
    {
      "epoch": 0.23554043444124576,
      "grad_norm": 3528.8612883563264,
      "learning_rate": 5.994210282887542e-07,
      "loss": 1070.1403,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -3.552372694015503,
      "rewards/margins": 0.3352223038673401,
      "rewards/rejected": -3.8875949382781982,
      "step": 225
    },
    {
      "epoch": 0.24077466631771788,
      "grad_norm": 2653.3569938495953,
      "learning_rate": 5.992382988692695e-07,
      "loss": 702.4689,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": -3.4368958473205566,
      "rewards/margins": 0.05198148638010025,
      "rewards/rejected": -3.488877058029175,
      "step": 230
    },
    {
      "epoch": 0.24600889819419,
      "grad_norm": 2460.6404326177712,
      "learning_rate": 5.990305831379434e-07,
      "loss": 716.1542,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -3.244385242462158,
      "rewards/margins": -0.12405101209878922,
      "rewards/rejected": -3.1203341484069824,
      "step": 235
    },
    {
      "epoch": 0.2512431300706621,
      "grad_norm": 19277.06656363596,
      "learning_rate": 5.9879789843898e-07,
      "loss": 699.4835,
      "rewards/accuracies": 0.48750001192092896,
      "rewards/chosen": -3.8388168811798096,
      "rewards/margins": -0.275703489780426,
      "rewards/rejected": -3.56311297416687,
      "step": 240
    },
    {
      "epoch": 0.2564773619471343,
      "grad_norm": 2404.72693437907,
      "learning_rate": 5.985402642014844e-07,
      "loss": 687.0264,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -3.710552215576172,
      "rewards/margins": 0.02756020799279213,
      "rewards/rejected": -3.738111972808838,
      "step": 245
    },
    {
      "epoch": 0.26171159382360637,
      "grad_norm": 3549.8837869207086,
      "learning_rate": 5.982577019378414e-07,
      "loss": 706.0577,
      "rewards/accuracies": 0.42500001192092896,
      "rewards/chosen": -4.365290641784668,
      "rewards/margins": -0.21943306922912598,
      "rewards/rejected": -4.145857334136963,
      "step": 250
    },
    {
      "epoch": 0.2669458257000785,
      "grad_norm": 5445.226361964557,
      "learning_rate": 5.979502352419188e-07,
      "loss": 693.9541,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -4.030621528625488,
      "rewards/margins": 0.30792179703712463,
      "rewards/rejected": -4.338542461395264,
      "step": 255
    },
    {
      "epoch": 0.2721800575765506,
      "grad_norm": 13503.534518998504,
      "learning_rate": 5.97617889787097e-07,
      "loss": 686.5193,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -3.8500771522521973,
      "rewards/margins": 0.2836206555366516,
      "rewards/rejected": -4.133697986602783,
      "step": 260
    },
    {
      "epoch": 0.27741428945302277,
      "grad_norm": 3617.057176411029,
      "learning_rate": 5.972606933241262e-07,
      "loss": 684.1671,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -4.0012664794921875,
      "rewards/margins": 0.2939813733100891,
      "rewards/rejected": -4.295248031616211,
      "step": 265
    },
    {
      "epoch": 0.2826485213294949,
      "grad_norm": 2953.5437133367595,
      "learning_rate": 5.968786756788079e-07,
      "loss": 693.8761,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -4.83554220199585,
      "rewards/margins": 0.40575289726257324,
      "rewards/rejected": -5.241294860839844,
      "step": 270
    },
    {
      "epoch": 0.287882753205967,
      "grad_norm": 4568.606142512149,
      "learning_rate": 5.964718687495056e-07,
      "loss": 691.1848,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -4.564460277557373,
      "rewards/margins": 0.23287418484687805,
      "rewards/rejected": -4.79733419418335,
      "step": 275
    },
    {
      "epoch": 0.29311698508243916,
      "grad_norm": 5298.708166238084,
      "learning_rate": 5.96040306504481e-07,
      "loss": 688.4041,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -4.424349784851074,
      "rewards/margins": 0.0478137843310833,
      "rewards/rejected": -4.472163200378418,
      "step": 280
    },
    {
      "epoch": 0.29835121695891126,
      "grad_norm": 5381.311535255818,
      "learning_rate": 5.955840249790574e-07,
      "loss": 695.5199,
      "rewards/accuracies": 0.42500001192092896,
      "rewards/chosen": -4.609699726104736,
      "rewards/margins": -0.1945555955171585,
      "rewards/rejected": -4.415144443511963,
      "step": 285
    },
    {
      "epoch": 0.3035854488353834,
      "grad_norm": 9454.121207295755,
      "learning_rate": 5.951030622726106e-07,
      "loss": 695.824,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -4.399235725402832,
      "rewards/margins": 0.22916913032531738,
      "rewards/rejected": -4.62840461730957,
      "step": 290
    },
    {
      "epoch": 0.30881968071185556,
      "grad_norm": 4620.12357538617,
      "learning_rate": 5.945974585453883e-07,
      "loss": 686.6351,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -4.199953079223633,
      "rewards/margins": 0.389130175113678,
      "rewards/rejected": -4.589083671569824,
      "step": 295
    },
    {
      "epoch": 0.31405391258832765,
      "grad_norm": 4965.233081176048,
      "learning_rate": 5.940672560151562e-07,
      "loss": 676.3912,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -4.602032661437988,
      "rewards/margins": 0.5422828793525696,
      "rewards/rejected": -5.144315242767334,
      "step": 300
    }
  ],
  "logging_steps": 5,
  "max_steps": 1910,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 2,
  "save_steps": 100,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": false
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 1,
  "trial_name": null,
  "trial_params": null
}