{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 1.0,
  "eval_steps": 500,
  "global_step": 1250,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.0,
      "learning_rate": 4e-08,
      "logits/chosen": -0.21237623691558838,
      "logits/rejected": -0.020425911992788315,
      "logps/chosen": -339.1651306152344,
      "logps/rejected": -403.4797058105469,
      "loss": 2.2013,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 1
    },
    {
      "epoch": 0.01,
      "learning_rate": 4.0000000000000003e-07,
      "logits/chosen": -0.1229037418961525,
      "logits/rejected": -0.09083450585603714,
      "logps/chosen": -409.6603698730469,
      "logps/rejected": -385.8681640625,
      "loss": 2.2536,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": 0.0001750251540215686,
      "rewards/margins": -0.0002073741634376347,
      "rewards/rejected": 0.0003823993029072881,
      "step": 10
    },
    {
      "epoch": 0.02,
      "learning_rate": 8.000000000000001e-07,
      "logits/chosen": -0.2033734768629074,
      "logits/rejected": -0.14241939783096313,
      "logps/chosen": -430.78778076171875,
      "logps/rejected": -420.1714782714844,
      "loss": 2.0797,
      "rewards/accuracies": 0.46875,
      "rewards/chosen": -7.134475890779868e-05,
      "rewards/margins": 0.0001290472282562405,
      "rewards/rejected": -0.00020039197988808155,
      "step": 20
    },
    {
      "epoch": 0.02,
      "learning_rate": 1.2000000000000002e-06,
      "logits/chosen": -0.1662522554397583,
      "logits/rejected": -0.07332392036914825,
      "logps/chosen": -389.4374084472656,
      "logps/rejected": -392.1519775390625,
      "loss": 2.1227,
      "rewards/accuracies": 0.5062500238418579,
      "rewards/chosen": -0.0009240208892151713,
      "rewards/margins": 0.00010561713861534372,
      "rewards/rejected": -0.001029637991450727,
      "step": 30
    },
    {
      "epoch": 0.03,
      "learning_rate": 1.6000000000000001e-06,
      "logits/chosen": -0.18387895822525024,
      "logits/rejected": -0.09537571668624878,
      "logps/chosen": -394.1351013183594,
      "logps/rejected": -378.9622497558594,
      "loss": 2.1895,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.0023726867511868477,
      "rewards/margins": 0.000768904690630734,
      "rewards/rejected": -0.003141591791063547,
      "step": 40
    },
    {
      "epoch": 0.04,
      "learning_rate": 2.0000000000000003e-06,
      "logits/chosen": -0.18213973939418793,
      "logits/rejected": -0.09330956637859344,
      "logps/chosen": -374.6993713378906,
      "logps/rejected": -371.1645812988281,
      "loss": 2.1547,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.0057793245650827885,
      "rewards/margins": 0.0022816271521151066,
      "rewards/rejected": -0.00806095264852047,
      "step": 50
    },
    {
      "epoch": 0.05,
      "learning_rate": 2.4000000000000003e-06,
      "logits/chosen": -0.1841249316930771,
      "logits/rejected": -0.1609618067741394,
      "logps/chosen": -420.17218017578125,
      "logps/rejected": -388.83990478515625,
      "loss": 2.1266,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -0.013447234407067299,
      "rewards/margins": 0.004250804893672466,
      "rewards/rejected": -0.01769803836941719,
      "step": 60
    },
    {
      "epoch": 0.06,
      "learning_rate": 2.8000000000000003e-06,
      "logits/chosen": -0.1828843504190445,
      "logits/rejected": -0.12139382213354111,
      "logps/chosen": -453.1728515625,
      "logps/rejected": -439.40167236328125,
      "loss": 2.1854,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -0.03316130489110947,
      "rewards/margins": 0.01242602989077568,
      "rewards/rejected": -0.045587338507175446,
      "step": 70
    },
    {
      "epoch": 0.06,
      "learning_rate": 3.2000000000000003e-06,
      "logits/chosen": -0.18017946183681488,
      "logits/rejected": -0.16199861466884613,
      "logps/chosen": -492.6966857910156,
      "logps/rejected": -502.58282470703125,
      "loss": 2.0852,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.06695900857448578,
      "rewards/margins": 0.03570399060845375,
      "rewards/rejected": -0.10266300290822983,
      "step": 80
    },
    {
      "epoch": 0.07,
      "learning_rate": 3.6000000000000003e-06,
      "logits/chosen": -0.17906096577644348,
      "logits/rejected": -0.16469471156597137,
      "logps/chosen": -588.4837036132812,
      "logps/rejected": -648.6444702148438,
      "loss": 1.9145,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.16310617327690125,
      "rewards/margins": 0.08765674382448196,
      "rewards/rejected": -0.2507628798484802,
      "step": 90
    },
    {
      "epoch": 0.08,
      "learning_rate": 4.000000000000001e-06,
      "logits/chosen": -0.20857279002666473,
      "logits/rejected": -0.19442644715309143,
      "logps/chosen": -678.5848388671875,
      "logps/rejected": -815.7537231445312,
      "loss": 1.9158,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.2612169682979584,
      "rewards/margins": 0.18096813559532166,
      "rewards/rejected": -0.44218510389328003,
      "step": 100
    },
    {
      "epoch": 0.09,
      "learning_rate": 4.4e-06,
      "logits/chosen": -0.23685956001281738,
      "logits/rejected": -0.1955154836177826,
      "logps/chosen": -766.2359619140625,
      "logps/rejected": -969.7020263671875,
      "loss": 1.8551,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.3886342942714691,
      "rewards/margins": 0.19053591787815094,
      "rewards/rejected": -0.5791702270507812,
      "step": 110
    },
    {
      "epoch": 0.1,
      "learning_rate": 4.800000000000001e-06,
      "logits/chosen": -0.23603197932243347,
      "logits/rejected": -0.17843177914619446,
      "logps/chosen": -816.71826171875,
      "logps/rejected": -1092.5975341796875,
      "loss": 1.8855,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.38946443796157837,
      "rewards/margins": 0.2521887421607971,
      "rewards/rejected": -0.6416531205177307,
      "step": 120
    },
    {
      "epoch": 0.1,
      "learning_rate": 4.999756310023261e-06,
      "logits/chosen": -0.2602929472923279,
      "logits/rejected": -0.20586004853248596,
      "logps/chosen": -778.2180786132812,
      "logps/rejected": -966.8118286132812,
      "loss": 1.8652,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.38287556171417236,
      "rewards/margins": 0.1863778531551361,
      "rewards/rejected": -0.5692533850669861,
      "step": 130
    },
    {
      "epoch": 0.11,
      "learning_rate": 4.997807075247147e-06,
      "logits/chosen": -0.2347811758518219,
      "logits/rejected": -0.1346631944179535,
      "logps/chosen": -744.4739379882812,
      "logps/rejected": -884.68994140625,
      "loss": 1.9158,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.3381068706512451,
      "rewards/margins": 0.1455555260181427,
      "rewards/rejected": -0.4836624264717102,
      "step": 140
    },
    {
      "epoch": 0.12,
      "learning_rate": 4.993910125649561e-06,
      "logits/chosen": -0.24987225234508514,
      "logits/rejected": -0.2068624198436737,
      "logps/chosen": -865.6614379882812,
      "logps/rejected": -1111.9520263671875,
      "loss": 1.9346,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.43612995743751526,
      "rewards/margins": 0.25621694326400757,
      "rewards/rejected": -0.6923469305038452,
      "step": 150
    },
    {
      "epoch": 0.13,
      "learning_rate": 4.988068499954578e-06,
      "logits/chosen": -0.2418244630098343,
      "logits/rejected": -0.1966630518436432,
      "logps/chosen": -785.2879028320312,
      "logps/rejected": -991.7936401367188,
      "loss": 1.8176,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.39799419045448303,
      "rewards/margins": 0.18280699849128723,
      "rewards/rejected": -0.580801248550415,
      "step": 160
    },
    {
      "epoch": 0.14,
      "learning_rate": 4.980286753286196e-06,
      "logits/chosen": -0.18229472637176514,
      "logits/rejected": -0.09505169093608856,
      "logps/chosen": -779.7567138671875,
      "logps/rejected": -905.3895263671875,
      "loss": 1.9224,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.34025901556015015,
      "rewards/margins": 0.165119469165802,
      "rewards/rejected": -0.5053785443305969,
      "step": 170
    },
    {
      "epoch": 0.14,
      "learning_rate": 4.970570953616383e-06,
      "logits/chosen": -0.24447326362133026,
      "logits/rejected": -0.18509261310100555,
      "logps/chosen": -806.5858154296875,
      "logps/rejected": -1070.8280029296875,
      "loss": 1.8121,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.3903842568397522,
      "rewards/margins": 0.23679156601428986,
      "rewards/rejected": -0.6271758079528809,
      "step": 180
    },
    {
      "epoch": 0.15,
      "learning_rate": 4.958928677033465e-06,
      "logits/chosen": -0.22912044823169708,
      "logits/rejected": -0.22532768547534943,
      "logps/chosen": -948.6514892578125,
      "logps/rejected": -1230.310302734375,
      "loss": 1.7871,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.5143545866012573,
      "rewards/margins": 0.28488418459892273,
      "rewards/rejected": -0.7992388010025024,
      "step": 190
    },
    {
      "epoch": 0.16,
      "learning_rate": 4.9453690018345144e-06,
      "logits/chosen": -0.2154952734708786,
      "logits/rejected": -0.17434343695640564,
      "logps/chosen": -969.3416748046875,
      "logps/rejected": -1133.4462890625,
      "loss": 1.9032,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.5185384750366211,
      "rewards/margins": 0.21671628952026367,
      "rewards/rejected": -0.7352547645568848,
      "step": 200
    },
    {
      "epoch": 0.17,
      "learning_rate": 4.9299025014463665e-06,
      "logits/chosen": -0.20597633719444275,
      "logits/rejected": -0.13240107893943787,
      "logps/chosen": -952.5860595703125,
      "logps/rejected": -1088.127685546875,
      "loss": 1.774,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.4815760552883148,
      "rewards/margins": 0.22714009881019592,
      "rewards/rejected": -0.708716094493866,
      "step": 210
    },
    {
      "epoch": 0.18,
      "learning_rate": 4.912541236180779e-06,
      "logits/chosen": -0.16825485229492188,
      "logits/rejected": -0.1814446598291397,
      "logps/chosen": -859.4645385742188,
      "logps/rejected": -1155.5140380859375,
      "loss": 1.6983,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.46208739280700684,
      "rewards/margins": 0.26932135224342346,
      "rewards/rejected": -0.7314087152481079,
      "step": 220
    },
    {
      "epoch": 0.18,
      "learning_rate": 4.893298743830168e-06,
      "logits/chosen": -0.1780141294002533,
      "logits/rejected": -0.1537729650735855,
      "logps/chosen": -854.94189453125,
      "logps/rejected": -1050.1890869140625,
      "loss": 1.8135,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.4414963722229004,
      "rewards/margins": 0.22997328639030457,
      "rewards/rejected": -0.6714696884155273,
      "step": 230
    },
    {
      "epoch": 0.19,
      "learning_rate": 4.8721900291112415e-06,
      "logits/chosen": -0.15549209713935852,
      "logits/rejected": -0.1134282797574997,
      "logps/chosen": -820.55224609375,
      "logps/rejected": -1063.4244384765625,
      "loss": 1.7236,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.39099445939064026,
      "rewards/margins": 0.22765907645225525,
      "rewards/rejected": -0.6186535954475403,
      "step": 240
    },
    {
      "epoch": 0.2,
      "learning_rate": 4.849231551964771e-06,
      "logits/chosen": -0.12339262664318085,
      "logits/rejected": -0.13772717118263245,
      "logps/chosen": -767.2117919921875,
      "logps/rejected": -1062.240966796875,
      "loss": 1.6988,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.41696324944496155,
      "rewards/margins": 0.25669869780540466,
      "rewards/rejected": -0.6736618876457214,
      "step": 250
    },
    {
      "epoch": 0.21,
      "learning_rate": 4.824441214720629e-06,
      "logits/chosen": -0.12506547570228577,
      "logits/rejected": -0.1212812289595604,
      "logps/chosen": -927.6405029296875,
      "logps/rejected": -1118.3206787109375,
      "loss": 1.6725,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.45764121413230896,
      "rewards/margins": 0.23505349457263947,
      "rewards/rejected": -0.6926947832107544,
      "step": 260
    },
    {
      "epoch": 0.22,
      "learning_rate": 4.7978383481380865e-06,
      "logits/chosen": -0.17072610557079315,
      "logits/rejected": -0.15457463264465332,
      "logps/chosen": -907.0338745117188,
      "logps/rejected": -1103.2227783203125,
      "loss": 1.9004,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.5299123525619507,
      "rewards/margins": 0.2055232971906662,
      "rewards/rejected": -0.7354356646537781,
      "step": 270
    },
    {
      "epoch": 0.22,
      "learning_rate": 4.769443696332272e-06,
      "logits/chosen": -0.17946860194206238,
      "logits/rejected": -0.12909087538719177,
      "logps/chosen": -944.4387817382812,
      "logps/rejected": -1121.226806640625,
      "loss": 1.8032,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.5064274668693542,
      "rewards/margins": 0.1990656554698944,
      "rewards/rejected": -0.705493152141571,
      "step": 280
    },
    {
      "epoch": 0.23,
      "learning_rate": 4.7392794005985324e-06,
      "logits/chosen": -0.19116660952568054,
      "logits/rejected": -0.19017921388149261,
      "logps/chosen": -936.3536987304688,
      "logps/rejected": -1282.28125,
      "loss": 1.5663,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.5539158582687378,
      "rewards/margins": 0.33316585421562195,
      "rewards/rejected": -0.8870818018913269,
      "step": 290
    },
    {
      "epoch": 0.24,
      "learning_rate": 4.707368982147318e-06,
      "logits/chosen": -0.1529879868030548,
      "logits/rejected": -0.12170819938182831,
      "logps/chosen": -992.0021362304688,
      "logps/rejected": -1266.134521484375,
      "loss": 1.6594,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5842385292053223,
      "rewards/margins": 0.28491389751434326,
      "rewards/rejected": -0.8691525459289551,
      "step": 300
    },
    {
      "epoch": 0.25,
      "learning_rate": 4.673737323763048e-06,
      "logits/chosen": -0.13799923658370972,
      "logits/rejected": -0.11788105964660645,
      "logps/chosen": -874.9791259765625,
      "logps/rejected": -1244.6304931640625,
      "loss": 1.6174,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.48227429389953613,
      "rewards/margins": 0.33959290385246277,
      "rewards/rejected": -0.8218671679496765,
      "step": 310
    },
    {
      "epoch": 0.26,
      "learning_rate": 4.638410650401267e-06,
      "logits/chosen": -0.09933747351169586,
      "logits/rejected": -0.0572042241692543,
      "logps/chosen": -885.7242431640625,
      "logps/rejected": -1079.048583984375,
      "loss": 1.684,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.47158098220825195,
      "rewards/margins": 0.20864149928092957,
      "rewards/rejected": -0.6802224516868591,
      "step": 320
    },
    {
      "epoch": 0.26,
      "learning_rate": 4.601416508739211e-06,
      "logits/chosen": -0.10739854723215103,
      "logits/rejected": -0.08502750098705292,
      "logps/chosen": -963.4494018554688,
      "logps/rejected": -1153.433349609375,
      "loss": 1.6356,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.5099435448646545,
      "rewards/margins": 0.2381589412689209,
      "rewards/rejected": -0.7481025457382202,
      "step": 330
    },
    {
      "epoch": 0.27,
      "learning_rate": 4.562783745695738e-06,
      "logits/chosen": -0.16697099804878235,
      "logits/rejected": -0.14366139471530914,
      "logps/chosen": -860.5040283203125,
      "logps/rejected": -1141.27685546875,
      "loss": 1.6149,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.4745675027370453,
      "rewards/margins": 0.276507169008255,
      "rewards/rejected": -0.7510746717453003,
      "step": 340
    },
    {
      "epoch": 0.28,
      "learning_rate": 4.522542485937369e-06,
      "logits/chosen": -0.11317078024148941,
      "logits/rejected": -0.07480302453041077,
      "logps/chosen": -983.2384033203125,
      "logps/rejected": -1304.386962890625,
      "loss": 1.7718,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.5476148724555969,
      "rewards/margins": 0.34910523891448975,
      "rewards/rejected": -0.8967201113700867,
      "step": 350
    },
    {
      "epoch": 0.29,
      "learning_rate": 4.4807241083879774e-06,
      "logits/chosen": -0.12416689097881317,
      "logits/rejected": -0.1058887392282486,
      "logps/chosen": -879.2247924804688,
      "logps/rejected": -1108.842529296875,
      "loss": 1.6448,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.4638194441795349,
      "rewards/margins": 0.2646525502204895,
      "rewards/rejected": -0.7284719944000244,
      "step": 360
    },
    {
      "epoch": 0.3,
      "learning_rate": 4.437361221760449e-06,
      "logits/chosen": -0.15833915770053864,
      "logits/rejected": -0.10952518880367279,
      "logps/chosen": -856.3928833007812,
      "logps/rejected": -1185.8890380859375,
      "loss": 1.6059,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.47408080101013184,
      "rewards/margins": 0.3529198169708252,
      "rewards/rejected": -0.827000617980957,
      "step": 370
    },
    {
      "epoch": 0.3,
      "learning_rate": 4.3924876391293915e-06,
      "logits/chosen": -0.1157221645116806,
      "logits/rejected": -0.11532659828662872,
      "logps/chosen": -977.6126708984375,
      "logps/rejected": -1276.198486328125,
      "loss": 1.6257,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.558844804763794,
      "rewards/margins": 0.3456457555294037,
      "rewards/rejected": -0.9044904708862305,
      "step": 380
    },
    {
      "epoch": 0.31,
      "learning_rate": 4.346138351564711e-06,
      "logits/chosen": -0.08272431790828705,
      "logits/rejected": -0.06370075047016144,
      "logps/chosen": -987.0380859375,
      "logps/rejected": -1292.915771484375,
      "loss": 1.704,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.5780712366104126,
      "rewards/margins": 0.3437100052833557,
      "rewards/rejected": -0.9217812418937683,
      "step": 390
    },
    {
      "epoch": 0.32,
      "learning_rate": 4.2983495008466285e-06,
      "logits/chosen": -0.12420694530010223,
      "logits/rejected": -0.08448062092065811,
      "logps/chosen": -891.9703369140625,
      "logps/rejected": -1206.5775146484375,
      "loss": 1.6398,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.49519339203834534,
      "rewards/margins": 0.30879420042037964,
      "rewards/rejected": -0.8039876222610474,
      "step": 400
    },
    {
      "epoch": 0.33,
      "learning_rate": 4.249158351283414e-06,
      "logits/chosen": -0.05765647441148758,
      "logits/rejected": -0.07125648856163025,
      "logps/chosen": -906.8709106445312,
      "logps/rejected": -1206.505126953125,
      "loss": 1.6809,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.5138260126113892,
      "rewards/margins": 0.2949921488761902,
      "rewards/rejected": -0.8088181614875793,
      "step": 410
    },
    {
      "epoch": 0.34,
      "learning_rate": 4.198603260653792e-06,
      "logits/chosen": -0.05834274739027023,
      "logits/rejected": -0.06940310448408127,
      "logps/chosen": -890.9304809570312,
      "logps/rejected": -1269.909423828125,
      "loss": 1.6515,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.48780736327171326,
      "rewards/margins": 0.38115113973617554,
      "rewards/rejected": -0.8689583539962769,
      "step": 420
    },
    {
      "epoch": 0.34,
      "learning_rate": 4.146723650296701e-06,
      "logits/chosen": -0.05108857899904251,
      "logits/rejected": -0.03374727442860603,
      "logps/chosen": -964.2681884765625,
      "logps/rejected": -1255.4764404296875,
      "loss": 1.5672,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.5296367406845093,
      "rewards/margins": 0.31283584237098694,
      "rewards/rejected": -0.8424724340438843,
      "step": 430
    },
    {
      "epoch": 0.35,
      "learning_rate": 4.093559974371725e-06,
      "logits/chosen": 0.002035899553447962,
      "logits/rejected": -0.05042330548167229,
      "logps/chosen": -1061.304443359375,
      "logps/rejected": -1401.3017578125,
      "loss": 1.7551,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.6395093202590942,
      "rewards/margins": 0.3655704855918884,
      "rewards/rejected": -1.005079746246338,
      "step": 440
    },
    {
      "epoch": 0.36,
      "learning_rate": 4.039153688314146e-06,
      "logits/chosen": -0.039297740906476974,
      "logits/rejected": -0.07409851998090744,
      "logps/chosen": -919.8971557617188,
      "logps/rejected": -1238.085693359375,
      "loss": 1.7409,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.54268878698349,
      "rewards/margins": 0.30365562438964844,
      "rewards/rejected": -0.8463443517684937,
      "step": 450
    },
    {
      "epoch": 0.37,
      "learning_rate": 3.983547216509254e-06,
      "logits/chosen": -0.05830153077840805,
      "logits/rejected": -0.010070801712572575,
      "logps/chosen": -911.67529296875,
      "logps/rejected": -1207.1807861328125,
      "loss": 1.5601,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.4990604519844055,
      "rewards/margins": 0.32518327236175537,
      "rewards/rejected": -0.8242437243461609,
      "step": 460
    },
    {
      "epoch": 0.38,
      "learning_rate": 3.92678391921108e-06,
      "logits/chosen": -0.03084222413599491,
      "logits/rejected": -0.033790141344070435,
      "logps/chosen": -880.0311279296875,
      "logps/rejected": -1202.804443359375,
      "loss": 1.5018,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.4940199851989746,
      "rewards/margins": 0.34093010425567627,
      "rewards/rejected": -0.8349500894546509,
      "step": 470
    },
    {
      "epoch": 0.38,
      "learning_rate": 3.868908058731376e-06,
      "logits/chosen": -0.04235939309000969,
      "logits/rejected": -0.03030604124069214,
      "logps/chosen": -948.1820068359375,
      "logps/rejected": -1385.6903076171875,
      "loss": 1.5846,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.5281907916069031,
      "rewards/margins": 0.44982367753982544,
      "rewards/rejected": -0.9780144691467285,
      "step": 480
    },
    {
      "epoch": 0.39,
      "learning_rate": 3.8099647649251984e-06,
      "logits/chosen": -0.016060033813118935,
      "logits/rejected": 0.0315425805747509,
      "logps/chosen": -872.0232543945312,
      "logps/rejected": -1230.6595458984375,
      "loss": 1.5608,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.49027585983276367,
      "rewards/margins": 0.3540174663066864,
      "rewards/rejected": -0.8442932963371277,
      "step": 490
    },
    {
      "epoch": 0.4,
      "learning_rate": 3.7500000000000005e-06,
      "logits/chosen": 0.008252490311861038,
      "logits/rejected": -0.006942494306713343,
      "logps/chosen": -965.0009765625,
      "logps/rejected": -1298.984375,
      "loss": 1.5949,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.5271399617195129,
      "rewards/margins": 0.34466665983200073,
      "rewards/rejected": -0.8718066215515137,
      "step": 500
    },
    {
      "epoch": 0.41,
      "learning_rate": 3.689060522675689e-06,
      "logits/chosen": 0.05910163372755051,
      "logits/rejected": 0.038642317056655884,
      "logps/chosen": -1022.37109375,
      "logps/rejected": -1327.8564453125,
      "loss": 1.71,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.5376920700073242,
      "rewards/margins": 0.3471304476261139,
      "rewards/rejected": -0.8848224878311157,
      "step": 510
    },
    {
      "epoch": 0.42,
      "learning_rate": 3.627193851723577e-06,
      "logits/chosen": 0.04024091362953186,
      "logits/rejected": 0.05306211858987808,
      "logps/chosen": -803.6287231445312,
      "logps/rejected": -1148.608154296875,
      "loss": 1.6553,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.42975902557373047,
      "rewards/margins": 0.3103187680244446,
      "rewards/rejected": -0.7400776743888855,
      "step": 520
    },
    {
      "epoch": 0.42,
      "learning_rate": 3.564448228912682e-06,
      "logits/chosen": 0.0793144553899765,
      "logits/rejected": 0.07511784881353378,
      "logps/chosen": -946.8004760742188,
      "logps/rejected": -1296.5035400390625,
      "loss": 1.6789,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.511221706867218,
      "rewards/margins": 0.3727003335952759,
      "rewards/rejected": -0.8839219808578491,
      "step": 530
    },
    {
      "epoch": 0.43,
      "learning_rate": 3.5008725813922383e-06,
      "logits/chosen": 0.04899642989039421,
      "logits/rejected": 0.07576090097427368,
      "logps/chosen": -968.6793823242188,
      "logps/rejected": -1360.913330078125,
      "loss": 1.5873,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.5624467134475708,
      "rewards/margins": 0.3971159756183624,
      "rewards/rejected": -0.9595626592636108,
      "step": 540
    },
    {
      "epoch": 0.44,
      "learning_rate": 3.436516483539781e-06,
      "logits/chosen": 0.07961883395910263,
      "logits/rejected": 0.07691974937915802,
      "logps/chosen": -1067.054931640625,
      "logps/rejected": -1277.112548828125,
      "loss": 1.7135,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.5898305773735046,
      "rewards/margins": 0.2757055163383484,
      "rewards/rejected": -0.8655362129211426,
      "step": 550
    },
    {
      "epoch": 0.45,
      "learning_rate": 3.3714301183045382e-06,
      "logits/chosen": 0.05680055543780327,
      "logits/rejected": 0.030003631487488747,
      "logps/chosen": -970.2757568359375,
      "logps/rejected": -1333.9105224609375,
      "loss": 1.6507,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.5233026742935181,
      "rewards/margins": 0.37386059761047363,
      "rewards/rejected": -0.8971632719039917,
      "step": 560
    },
    {
      "epoch": 0.46,
      "learning_rate": 3.3056642380762783e-06,
      "logits/chosen": 0.03873239830136299,
      "logits/rejected": 0.08405943214893341,
      "logps/chosen": -984.0048828125,
      "logps/rejected": -1360.584228515625,
      "loss": 1.6165,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.5443267226219177,
      "rewards/margins": 0.3884132504463196,
      "rewards/rejected": -0.9327400922775269,
      "step": 570
    },
    {
      "epoch": 0.46,
      "learning_rate": 3.2392701251101172e-06,
      "logits/chosen": 0.037966854870319366,
      "logits/rejected": 0.02861429750919342,
      "logps/chosen": -917.8508911132812,
      "logps/rejected": -1293.6751708984375,
      "loss": 1.5682,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.5233009457588196,
      "rewards/margins": 0.3688820004463196,
      "rewards/rejected": -0.8921829462051392,
      "step": 580
    },
    {
      "epoch": 0.47,
      "learning_rate": 3.1722995515381644e-06,
      "logits/chosen": 0.047953229397535324,
      "logits/rejected": 0.08454461395740509,
      "logps/chosen": -1099.385009765625,
      "logps/rejected": -1388.120849609375,
      "loss": 1.6362,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.6879197955131531,
      "rewards/margins": 0.29069215059280396,
      "rewards/rejected": -0.978611946105957,
      "step": 590
    },
    {
      "epoch": 0.48,
      "learning_rate": 3.1048047389991693e-06,
      "logits/chosen": 0.091717928647995,
      "logits/rejected": 0.09912924468517303,
      "logps/chosen": -909.0963745117188,
      "logps/rejected": -1410.45751953125,
      "loss": 1.5942,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.4968685209751129,
      "rewards/margins": 0.4905162453651428,
      "rewards/rejected": -0.9873846769332886,
      "step": 600
    },
    {
      "epoch": 0.49,
      "learning_rate": 3.0368383179176584e-06,
      "logits/chosen": 0.07560348510742188,
      "logits/rejected": 0.06965458393096924,
      "logps/chosen": -914.50634765625,
      "logps/rejected": -1271.376220703125,
      "loss": 1.6215,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.5211359262466431,
      "rewards/margins": 0.3135939836502075,
      "rewards/rejected": -0.8347299695014954,
      "step": 610
    },
    {
      "epoch": 0.5,
      "learning_rate": 2.9684532864643123e-06,
      "logits/chosen": 0.11191339790821075,
      "logits/rejected": 0.12147488445043564,
      "logps/chosen": -939.2978515625,
      "logps/rejected": -1317.5257568359375,
      "loss": 1.71,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.5019097328186035,
      "rewards/margins": 0.36823293566703796,
      "rewards/rejected": -0.8701427578926086,
      "step": 620
    },
    {
      "epoch": 0.5,
      "learning_rate": 2.8997029692295875e-06,
      "logits/chosen": 0.08082602918148041,
      "logits/rejected": 0.11734504997730255,
      "logps/chosen": -875.3718872070312,
      "logps/rejected": -1250.5029296875,
      "loss": 1.5481,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.4535561203956604,
      "rewards/margins": 0.38246890902519226,
      "rewards/rejected": -0.8360249400138855,
      "step": 630
    },
    {
      "epoch": 0.51,
      "learning_rate": 2.8306409756428067e-06,
      "logits/chosen": 0.08714647591114044,
      "logits/rejected": 0.09734050184488297,
      "logps/chosen": -969.7335815429688,
      "logps/rejected": -1301.7027587890625,
      "loss": 1.5968,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.5595206022262573,
      "rewards/margins": 0.35599860548973083,
      "rewards/rejected": -0.9155190587043762,
      "step": 640
    },
    {
      "epoch": 0.52,
      "learning_rate": 2.761321158169134e-06,
      "logits/chosen": 0.006733921356499195,
      "logits/rejected": 0.03773912042379379,
      "logps/chosen": -1084.8623046875,
      "logps/rejected": -1479.010498046875,
      "loss": 1.5928,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.6584538221359253,
      "rewards/margins": 0.4003337025642395,
      "rewards/rejected": -1.0587873458862305,
      "step": 650
    },
    {
      "epoch": 0.53,
      "learning_rate": 2.6917975703170466e-06,
      "logits/chosen": 0.10130703449249268,
      "logits/rejected": 0.09567786753177643,
      "logps/chosen": -978.0599365234375,
      "logps/rejected": -1393.5048828125,
      "loss": 1.5646,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.5595641136169434,
      "rewards/margins": 0.4086734652519226,
      "rewards/rejected": -0.9682376980781555,
      "step": 660
    },
    {
      "epoch": 0.54,
      "learning_rate": 2.6221244244890336e-06,
      "logits/chosen": 0.10669932514429092,
      "logits/rejected": 0.08464895188808441,
      "logps/chosen": -792.4083251953125,
      "logps/rejected": -1029.5029296875,
      "loss": 1.63,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.4270346760749817,
      "rewards/margins": 0.26256269216537476,
      "rewards/rejected": -0.6895973682403564,
      "step": 670
    },
    {
      "epoch": 0.54,
      "learning_rate": 2.5523560497083927e-06,
      "logits/chosen": 0.11974094063043594,
      "logits/rejected": 0.13801291584968567,
      "logps/chosen": -934.68310546875,
      "logps/rejected": -1160.298828125,
      "loss": 1.6015,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.4680783152580261,
      "rewards/margins": 0.2951990067958832,
      "rewards/rejected": -0.7632774114608765,
      "step": 680
    },
    {
      "epoch": 0.55,
      "learning_rate": 2.482546849255096e-06,
      "logits/chosen": 0.11049097776412964,
      "logits/rejected": 0.18580587208271027,
      "logps/chosen": -908.0875244140625,
      "logps/rejected": -1278.4056396484375,
      "loss": 1.5788,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.46382030844688416,
      "rewards/margins": 0.3706962466239929,
      "rewards/rejected": -0.8345165252685547,
      "step": 690
    },
    {
      "epoch": 0.56,
      "learning_rate": 2.4127512582437486e-06,
      "logits/chosen": 0.1090225949883461,
      "logits/rejected": 0.11428926140069962,
      "logps/chosen": -887.1666259765625,
      "logps/rejected": -1290.92236328125,
      "loss": 1.6283,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.47191792726516724,
      "rewards/margins": 0.4161871075630188,
      "rewards/rejected": -0.8881050944328308,
      "step": 700
    },
    {
      "epoch": 0.57,
      "learning_rate": 2.3430237011767166e-06,
      "logits/chosen": 0.08914731442928314,
      "logits/rejected": 0.09316228330135345,
      "logps/chosen": -951.3321533203125,
      "logps/rejected": -1253.3704833984375,
      "loss": 1.5467,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.5280056595802307,
      "rewards/margins": 0.3312267065048218,
      "rewards/rejected": -0.8592323064804077,
      "step": 710
    },
    {
      "epoch": 0.58,
      "learning_rate": 2.2734185495055503e-06,
      "logits/chosen": 0.10006880760192871,
      "logits/rejected": 0.10086695849895477,
      "logps/chosen": -909.28271484375,
      "logps/rejected": -1240.0157470703125,
      "loss": 1.532,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.4997311234474182,
      "rewards/margins": 0.3538741171360016,
      "rewards/rejected": -0.8536051511764526,
      "step": 720
    },
    {
      "epoch": 0.58,
      "learning_rate": 2.2039900792337477e-06,
      "logits/chosen": 0.08773048967123032,
      "logits/rejected": 0.10859861224889755,
      "logps/chosen": -968.7990112304688,
      "logps/rejected": -1337.583740234375,
      "loss": 1.6446,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.5440322160720825,
      "rewards/margins": 0.3773999512195587,
      "rewards/rejected": -0.9214321374893188,
      "step": 730
    },
    {
      "epoch": 0.59,
      "learning_rate": 2.134792428593971e-06,
      "logits/chosen": 0.10109611600637436,
      "logits/rejected": 0.1010131984949112,
      "logps/chosen": -934.5894775390625,
      "logps/rejected": -1251.2464599609375,
      "loss": 1.647,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.5000175833702087,
      "rewards/margins": 0.3368659019470215,
      "rewards/rejected": -0.836883544921875,
      "step": 740
    },
    {
      "epoch": 0.6,
      "learning_rate": 2.0658795558326745e-06,
      "logits/chosen": 0.07538788765668869,
      "logits/rejected": 0.03233175352215767,
      "logps/chosen": -919.93994140625,
      "logps/rejected": -1380.2998046875,
      "loss": 1.5396,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.5493655800819397,
      "rewards/margins": 0.42367228865623474,
      "rewards/rejected": -0.9730378985404968,
      "step": 750
    },
    {
      "epoch": 0.61,
      "learning_rate": 1.997305197135089e-06,
      "logits/chosen": 0.016186673194169998,
      "logits/rejected": 0.035506926476955414,
      "logps/chosen": -1090.354736328125,
      "logps/rejected": -1475.09912109375,
      "loss": 1.5146,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6771215200424194,
      "rewards/margins": 0.3887900412082672,
      "rewards/rejected": -1.0659115314483643,
      "step": 760
    },
    {
      "epoch": 0.62,
      "learning_rate": 1.9291228247233607e-06,
      "logits/chosen": 0.0642293244600296,
      "logits/rejected": 0.07756002992391586,
      "logps/chosen": -1072.716064453125,
      "logps/rejected": -1393.173583984375,
      "loss": 1.6319,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.648370623588562,
      "rewards/margins": 0.35922181606292725,
      "rewards/rejected": -1.0075924396514893,
      "step": 770
    },
    {
      "epoch": 0.62,
      "learning_rate": 1.8613856051605242e-06,
      "logits/chosen": 0.09995916485786438,
      "logits/rejected": 0.059368181973695755,
      "logps/chosen": -991.5946044921875,
      "logps/rejected": -1411.4442138671875,
      "loss": 1.4733,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.5653289556503296,
      "rewards/margins": 0.4559949040412903,
      "rewards/rejected": -1.0213239192962646,
      "step": 780
    },
    {
      "epoch": 0.63,
      "learning_rate": 1.7941463578928088e-06,
      "logits/chosen": 0.07716236263513565,
      "logits/rejected": 0.08072565495967865,
      "logps/chosen": -1078.119384765625,
      "logps/rejected": -1377.1890869140625,
      "loss": 1.5073,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.629800021648407,
      "rewards/margins": 0.32973819971084595,
      "rewards/rejected": -0.9595381617546082,
      "step": 790
    },
    {
      "epoch": 0.64,
      "learning_rate": 1.7274575140626318e-06,
      "logits/chosen": 0.08353432267904282,
      "logits/rejected": 0.11776747554540634,
      "logps/chosen": -1066.1820068359375,
      "logps/rejected": -1459.1993408203125,
      "loss": 1.472,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.587757408618927,
      "rewards/margins": 0.4555942416191101,
      "rewards/rejected": -1.0433517694473267,
      "step": 800
    },
    {
      "epoch": 0.65,
      "learning_rate": 1.661371075624363e-06,
      "logits/chosen": 0.07389640808105469,
      "logits/rejected": 0.02921133115887642,
      "logps/chosen": -1045.249755859375,
      "logps/rejected": -1426.687744140625,
      "loss": 1.5891,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.6339725255966187,
      "rewards/margins": 0.38414081931114197,
      "rewards/rejected": -1.018113374710083,
      "step": 810
    },
    {
      "epoch": 0.66,
      "learning_rate": 1.5959385747947697e-06,
      "logits/chosen": 0.08418832719326019,
      "logits/rejected": 0.059553779661655426,
      "logps/chosen": -903.3878173828125,
      "logps/rejected": -1283.9080810546875,
      "loss": 1.4831,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.5188735127449036,
      "rewards/margins": 0.38101640343666077,
      "rewards/rejected": -0.8998898267745972,
      "step": 820
    },
    {
      "epoch": 0.66,
      "learning_rate": 1.5312110338697427e-06,
      "logits/chosen": 0.07461550831794739,
      "logits/rejected": 0.09411834180355072,
      "logps/chosen": -978.8884887695312,
      "logps/rejected": -1332.2344970703125,
      "loss": 1.6651,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.5540666580200195,
      "rewards/margins": 0.3479211926460266,
      "rewards/rejected": -0.9019878506660461,
      "step": 830
    },
    {
      "epoch": 0.67,
      "learning_rate": 1.467238925438646e-06,
      "logits/chosen": 0.126978799700737,
      "logits/rejected": 0.15114851295948029,
      "logps/chosen": -988.6935424804688,
      "logps/rejected": -1252.6251220703125,
      "loss": 1.4992,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.5259130597114563,
      "rewards/margins": 0.3344604969024658,
      "rewards/rejected": -0.8603734970092773,
      "step": 840
    },
    {
      "epoch": 0.68,
      "learning_rate": 1.4040721330273063e-06,
      "logits/chosen": 0.13616199791431427,
      "logits/rejected": 0.10154382139444351,
      "logps/chosen": -930.3357543945312,
      "logps/rejected": -1316.8492431640625,
      "loss": 1.5081,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.538154125213623,
      "rewards/margins": 0.3978419005870819,
      "rewards/rejected": -0.9359959363937378,
      "step": 850
    },
    {
      "epoch": 0.69,
      "learning_rate": 1.3417599122003464e-06,
      "logits/chosen": 0.10873869806528091,
      "logits/rejected": 0.08275686204433441,
      "logps/chosen": -944.9284057617188,
      "logps/rejected": -1242.4952392578125,
      "loss": 1.4874,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.5550948977470398,
      "rewards/margins": 0.30027762055397034,
      "rewards/rejected": -0.8553726077079773,
      "step": 860
    },
    {
      "epoch": 0.7,
      "learning_rate": 1.280350852153168e-06,
      "logits/chosen": 0.13027873635292053,
      "logits/rejected": 0.12578675150871277,
      "logps/chosen": -937.29345703125,
      "logps/rejected": -1311.0787353515625,
      "loss": 1.5071,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.5160200595855713,
      "rewards/margins": 0.4040035307407379,
      "rewards/rejected": -0.9200236201286316,
      "step": 870
    },
    {
      "epoch": 0.7,
      "learning_rate": 1.2198928378235717e-06,
      "logits/chosen": 0.08686872571706772,
      "logits/rejected": 0.118919238448143,
      "logps/chosen": -991.64208984375,
      "logps/rejected": -1404.045166015625,
      "loss": 1.4595,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.5468842387199402,
      "rewards/margins": 0.4538387656211853,
      "rewards/rejected": -1.0007230043411255,
      "step": 880
    },
    {
      "epoch": 0.71,
      "learning_rate": 1.160433012552508e-06,
      "logits/chosen": 0.054034192115068436,
      "logits/rejected": 0.07009950280189514,
      "logps/chosen": -1012.7716674804688,
      "logps/rejected": -1450.068359375,
      "loss": 1.5362,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.5747925043106079,
      "rewards/margins": 0.4582456648349762,
      "rewards/rejected": -1.0330379009246826,
      "step": 890
    },
    {
      "epoch": 0.72,
      "learning_rate": 1.1020177413231334e-06,
      "logits/chosen": 0.06726670265197754,
      "logits/rejected": 0.07763611525297165,
      "logps/chosen": -1003.4699096679688,
      "logps/rejected": -1429.7281494140625,
      "loss": 1.6021,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.5936052799224854,
      "rewards/margins": 0.4537973403930664,
      "rewards/rejected": -1.0474026203155518,
      "step": 900
    },
    {
      "epoch": 0.73,
      "learning_rate": 1.0446925746067768e-06,
      "logits/chosen": 0.08046019077301025,
      "logits/rejected": 0.09484287351369858,
      "logps/chosen": -1176.3572998046875,
      "logps/rejected": -1445.795166015625,
      "loss": 1.6634,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.7165821194648743,
      "rewards/margins": 0.31557074189186096,
      "rewards/rejected": -1.0321528911590576,
      "step": 910
    },
    {
      "epoch": 0.74,
      "learning_rate": 9.88502212844063e-07,
      "logits/chosen": 0.09446735680103302,
      "logits/rejected": 0.1036868467926979,
      "logps/chosen": -1062.044677734375,
      "logps/rejected": -1472.5697021484375,
      "loss": 1.6091,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.6323626637458801,
      "rewards/margins": 0.4094136357307434,
      "rewards/rejected": -1.041776180267334,
      "step": 920
    },
    {
      "epoch": 0.74,
      "learning_rate": 9.334904715888496e-07,
      "logits/chosen": 0.07529421150684357,
      "logits/rejected": 0.08289067447185516,
      "logps/chosen": -970.4074096679688,
      "logps/rejected": -1391.994140625,
      "loss": 1.4436,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.546066403388977,
      "rewards/margins": 0.4431384205818176,
      "rewards/rejected": -0.9892047643661499,
      "step": 930
    },
    {
      "epoch": 0.75,
      "learning_rate": 8.797002473421729e-07,
      "logits/chosen": 0.11385681480169296,
      "logits/rejected": 0.07344277948141098,
      "logps/chosen": -1011.7611083984375,
      "logps/rejected": -1272.4300537109375,
      "loss": 1.5694,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.5955858826637268,
      "rewards/margins": 0.2856913208961487,
      "rewards/rejected": -0.8812772035598755,
      "step": 940
    },
    {
      "epoch": 0.76,
      "learning_rate": 8.271734841028553e-07,
      "logits/chosen": 0.14825263619422913,
      "logits/rejected": 0.12270005792379379,
      "logps/chosen": -1037.7218017578125,
      "logps/rejected": -1463.168212890625,
      "loss": 1.4799,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.5987383723258972,
      "rewards/margins": 0.4431166648864746,
      "rewards/rejected": -1.0418550968170166,
      "step": 950
    },
    {
      "epoch": 0.77,
      "learning_rate": 7.759511406608255e-07,
      "logits/chosen": 0.1073087602853775,
      "logits/rejected": 0.09663400799036026,
      "logps/chosen": -948.8695068359375,
      "logps/rejected": -1318.7125244140625,
      "loss": 1.6156,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.5697445869445801,
      "rewards/margins": 0.38244208693504333,
      "rewards/rejected": -0.9521867036819458,
      "step": 960
    },
    {
      "epoch": 0.78,
      "learning_rate": 7.260731586586983e-07,
      "logits/chosen": 0.13227181136608124,
      "logits/rejected": 0.1124822273850441,
      "logps/chosen": -1044.2213134765625,
      "logps/rejected": -1377.6756591796875,
      "loss": 1.486,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.6263742446899414,
      "rewards/margins": 0.3383250832557678,
      "rewards/rejected": -0.9646994471549988,
      "step": 970
    },
    {
      "epoch": 0.78,
      "learning_rate": 6.775784314464717e-07,
      "logits/chosen": 0.15785950422286987,
      "logits/rejected": 0.13977470993995667,
      "logps/chosen": -1023.9119262695312,
      "logps/rejected": -1362.277587890625,
      "loss": 1.4655,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.579637885093689,
      "rewards/margins": 0.39033398032188416,
      "rewards/rejected": -0.9699718356132507,
      "step": 980
    },
    {
      "epoch": 0.79,
      "learning_rate": 6.305047737536707e-07,
      "logits/chosen": 0.08758912980556488,
      "logits/rejected": 0.12277774512767792,
      "logps/chosen": -999.2574462890625,
      "logps/rejected": -1349.63720703125,
      "loss": 1.5246,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.5825763940811157,
      "rewards/margins": 0.368095338344574,
      "rewards/rejected": -0.9506716728210449,
      "step": 990
    },
    {
      "epoch": 0.8,
      "learning_rate": 5.848888922025553e-07,
      "logits/chosen": 0.11750923097133636,
      "logits/rejected": 0.12221167236566544,
      "logps/chosen": -1051.95654296875,
      "logps/rejected": -1356.2454833984375,
      "loss": 1.5642,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.6182184219360352,
      "rewards/margins": 0.3450503945350647,
      "rewards/rejected": -0.9632687568664551,
      "step": 1000
    },
    {
      "epoch": 0.81,
      "learning_rate": 5.407663566854008e-07,
      "logits/chosen": 0.06591792404651642,
      "logits/rejected": 0.09111978113651276,
      "logps/chosen": -1000.8797607421875,
      "logps/rejected": -1322.9415283203125,
      "loss": 1.5276,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.5938758254051208,
      "rewards/margins": 0.35901254415512085,
      "rewards/rejected": -0.9528884887695312,
      "step": 1010
    },
    {
      "epoch": 0.82,
      "learning_rate": 4.981715726281666e-07,
      "logits/chosen": 0.11376409232616425,
      "logits/rejected": 0.09700876474380493,
      "logps/chosen": -1020.0313720703125,
      "logps/rejected": -1457.311767578125,
      "loss": 1.3381,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.6390109062194824,
      "rewards/margins": 0.4118656516075134,
      "rewards/rejected": -1.0508766174316406,
      "step": 1020
    },
    {
      "epoch": 0.82,
      "learning_rate": 4.5713775416217884e-07,
      "logits/chosen": 0.10929641872644424,
      "logits/rejected": 0.10328598320484161,
      "logps/chosen": -1016.06591796875,
      "logps/rejected": -1392.699462890625,
      "loss": 1.5685,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.6151790022850037,
      "rewards/margins": 0.3843514025211334,
      "rewards/rejected": -0.9995304942131042,
      "step": 1030
    },
    {
      "epoch": 0.83,
      "learning_rate": 4.1769689822475147e-07,
      "logits/chosen": 0.12536247074604034,
      "logits/rejected": 0.10010683536529541,
      "logps/chosen": -1026.385986328125,
      "logps/rejected": -1466.6982421875,
      "loss": 1.5254,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.6345069408416748,
      "rewards/margins": 0.4376339018344879,
      "rewards/rejected": -1.0721409320831299,
      "step": 1040
    },
    {
      "epoch": 0.84,
      "learning_rate": 3.798797596089351e-07,
      "logits/chosen": 0.11945215612649918,
      "logits/rejected": 0.1324438452720642,
      "logps/chosen": -1003.3302612304688,
      "logps/rejected": -1490.3330078125,
      "loss": 1.4181,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.604376494884491,
      "rewards/margins": 0.43948906660079956,
      "rewards/rejected": -1.0438655614852905,
      "step": 1050
    },
    {
      "epoch": 0.85,
      "learning_rate": 3.4371582698185636e-07,
      "logits/chosen": 0.11982687562704086,
      "logits/rejected": 0.11596560478210449,
      "logps/chosen": -965.4796752929688,
      "logps/rejected": -1305.4368896484375,
      "loss": 1.4891,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.5759687423706055,
      "rewards/margins": 0.33464914560317993,
      "rewards/rejected": -0.9106179475784302,
      "step": 1060
    },
    {
      "epoch": 0.86,
      "learning_rate": 3.092332998903416e-07,
      "logits/chosen": 0.11303956806659698,
      "logits/rejected": 0.12296398729085922,
      "logps/chosen": -1185.8704833984375,
      "logps/rejected": -1574.617919921875,
      "loss": 1.4492,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.7027804255485535,
      "rewards/margins": 0.45428234338760376,
      "rewards/rejected": -1.1570627689361572,
      "step": 1070
    },
    {
      "epoch": 0.86,
      "learning_rate": 2.764590667717562e-07,
      "logits/chosen": 0.11120734363794327,
      "logits/rejected": 0.07134538888931274,
      "logps/chosen": -1128.311279296875,
      "logps/rejected": -1455.035888671875,
      "loss": 1.5757,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.6751740574836731,
      "rewards/margins": 0.3886902928352356,
      "rewards/rejected": -1.0638644695281982,
      "step": 1080
    },
    {
      "epoch": 0.87,
      "learning_rate": 2.454186839872158e-07,
      "logits/chosen": 0.13448989391326904,
      "logits/rejected": 0.13337567448616028,
      "logps/chosen": -1010.8844604492188,
      "logps/rejected": -1454.37841796875,
      "loss": 1.5715,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6244133710861206,
      "rewards/margins": 0.4344162344932556,
      "rewards/rejected": -1.058829665184021,
      "step": 1090
    },
    {
      "epoch": 0.88,
      "learning_rate": 2.1613635589349756e-07,
      "logits/chosen": 0.13915595412254333,
      "logits/rejected": 0.1342434138059616,
      "logps/chosen": -1146.306396484375,
      "logps/rejected": -1514.5679931640625,
      "loss": 1.5751,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.6373551487922668,
      "rewards/margins": 0.4376014173030853,
      "rewards/rejected": -1.0749565362930298,
      "step": 1100
    },
    {
      "epoch": 0.89,
      "learning_rate": 1.8863491596921745e-07,
      "logits/chosen": 0.15741156041622162,
      "logits/rejected": 0.13911879062652588,
      "logps/chosen": -1010.8033447265625,
      "logps/rejected": -1388.541015625,
      "loss": 1.5382,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.5765799283981323,
      "rewards/margins": 0.40866199135780334,
      "rewards/rejected": -0.9852418899536133,
      "step": 1110
    },
    {
      "epoch": 0.9,
      "learning_rate": 1.629358090099639e-07,
      "logits/chosen": 0.13894052803516388,
      "logits/rejected": 0.13587644696235657,
      "logps/chosen": -981.1856689453125,
      "logps/rejected": -1380.902587890625,
      "loss": 1.5256,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.5833259224891663,
      "rewards/margins": 0.4001302719116211,
      "rewards/rejected": -0.9834562540054321,
      "step": 1120
    },
    {
      "epoch": 0.9,
      "learning_rate": 1.3905907440629752e-07,
      "logits/chosen": 0.1367521584033966,
      "logits/rejected": 0.10814319550991058,
      "logps/chosen": -1037.4566650390625,
      "logps/rejected": -1406.663818359375,
      "loss": 1.5083,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.5931031703948975,
      "rewards/margins": 0.40140143036842346,
      "rewards/rejected": -0.9945046305656433,
      "step": 1130
    },
    {
      "epoch": 0.91,
      "learning_rate": 1.1702333051763271e-07,
      "logits/chosen": 0.13864858448505402,
      "logits/rejected": 0.10779663175344467,
      "logps/chosen": -968.2605590820312,
      "logps/rejected": -1421.97900390625,
      "loss": 1.4325,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.553028404712677,
      "rewards/margins": 0.45522522926330566,
      "rewards/rejected": -1.0082536935806274,
      "step": 1140
    },
    {
      "epoch": 0.92,
      "learning_rate": 9.684576015420277e-08,
      "logits/chosen": 0.1990373134613037,
      "logits/rejected": 0.15076985955238342,
      "logps/chosen": -959.9166259765625,
      "logps/rejected": -1385.095703125,
      "loss": 1.4883,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5342432856559753,
      "rewards/margins": 0.45701155066490173,
      "rewards/rejected": -0.9912548065185547,
      "step": 1150
    },
    {
      "epoch": 0.93,
      "learning_rate": 7.854209717842231e-08,
      "logits/chosen": 0.167617067694664,
      "logits/rejected": 0.16784724593162537,
      "logps/chosen": -999.2423095703125,
      "logps/rejected": -1370.2662353515625,
      "loss": 1.4697,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.5615602135658264,
      "rewards/margins": 0.4337054193019867,
      "rewards/rejected": -0.9952656030654907,
      "step": 1160
    },
    {
      "epoch": 0.94,
      "learning_rate": 6.212661423609184e-08,
      "logits/chosen": 0.1487255096435547,
      "logits/rejected": 0.17393285036087036,
      "logps/chosen": -991.5260009765625,
      "logps/rejected": -1270.6396484375,
      "loss": 1.7102,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.5859721899032593,
      "rewards/margins": 0.29911959171295166,
      "rewards/rejected": -0.8850919008255005,
      "step": 1170
    },
    {
      "epoch": 0.94,
      "learning_rate": 4.761211162702117e-08,
      "logits/chosen": 0.12677620351314545,
      "logits/rejected": 0.1536310613155365,
      "logps/chosen": -1026.4810791015625,
      "logps/rejected": -1375.422119140625,
      "loss": 1.5775,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.6072763800621033,
      "rewards/margins": 0.3752721846103668,
      "rewards/rejected": -0.982548713684082,
      "step": 1180
    },
    {
      "epoch": 0.95,
      "learning_rate": 3.5009907323737826e-08,
      "logits/chosen": 0.13358378410339355,
      "logits/rejected": 0.13474227488040924,
      "logps/chosen": -1048.68994140625,
      "logps/rejected": -1420.0101318359375,
      "loss": 1.5444,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.6096453070640564,
      "rewards/margins": 0.3957948684692383,
      "rewards/rejected": -1.00544011592865,
      "step": 1190
    },
    {
      "epoch": 0.96,
      "learning_rate": 2.4329828146074096e-08,
      "logits/chosen": 0.10293509066104889,
      "logits/rejected": 0.08836082369089127,
      "logps/chosen": -981.24755859375,
      "logps/rejected": -1432.976318359375,
      "loss": 1.4505,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.5690222382545471,
      "rewards/margins": 0.4183219075202942,
      "rewards/rejected": -0.9873440861701965,
      "step": 1200
    },
    {
      "epoch": 0.97,
      "learning_rate": 1.5580202098509078e-08,
      "logits/chosen": 0.09086032956838608,
      "logits/rejected": 0.0959467664361,
      "logps/chosen": -972.2115478515625,
      "logps/rejected": -1355.9515380859375,
      "loss": 1.4382,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.5868245959281921,
      "rewards/margins": 0.3957366347312927,
      "rewards/rejected": -0.9825612306594849,
      "step": 1210
    },
    {
      "epoch": 0.98,
      "learning_rate": 8.767851876239075e-09,
      "logits/chosen": 0.12658867239952087,
      "logits/rejected": 0.1394813060760498,
      "logps/chosen": -984.2681884765625,
      "logps/rejected": -1267.7216796875,
      "loss": 1.5614,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.5679751634597778,
      "rewards/margins": 0.30647456645965576,
      "rewards/rejected": -0.8744497299194336,
      "step": 1220
    },
    {
      "epoch": 0.98,
      "learning_rate": 3.8980895450474455e-09,
      "logits/chosen": 0.11635242402553558,
      "logits/rejected": 0.08196248859167099,
      "logps/chosen": -1042.649169921875,
      "logps/rejected": -1407.2928466796875,
      "loss": 1.603,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.5844318270683289,
      "rewards/margins": 0.4090692400932312,
      "rewards/rejected": -0.9935010671615601,
      "step": 1230
    },
    {
      "epoch": 0.99,
      "learning_rate": 9.747123991141193e-10,
      "logits/chosen": 0.1424560546875,
      "logits/rejected": 0.1417275071144104,
      "logps/chosen": -1018.15234375,
      "logps/rejected": -1289.4718017578125,
      "loss": 1.4991,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.5752192735671997,
      "rewards/margins": 0.3309435248374939,
      "rewards/rejected": -0.9061626195907593,
      "step": 1240
    },
    {
      "epoch": 1.0,
      "learning_rate": 0.0,
      "logits/chosen": 0.11531805992126465,
      "logits/rejected": 0.14260968565940857,
      "logps/chosen": -945.6492919921875,
      "logps/rejected": -1332.210693359375,
      "loss": 1.5185,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5279589891433716,
      "rewards/margins": 0.39913317561149597,
      "rewards/rejected": -0.9270921945571899,
      "step": 1250
    },
    {
      "epoch": 1.0,
      "step": 1250,
      "total_flos": 0.0,
      "train_loss": 1.6514112674713135,
      "train_runtime": 16819.2653,
      "train_samples_per_second": 3.567,
      "train_steps_per_second": 0.074
    }
  ],
  "logging_steps": 10,
  "max_steps": 1250,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 100,
  "total_flos": 0.0,
  "train_batch_size": 4,
  "trial_name": null,
  "trial_params": null
}