{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 0.6379585326953748,
  "global_step": 100,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.01,
      "learning_rate": 4e-07,
      "logits/chosen": -2.9071130752563477,
      "logits/rejected": -2.8750061988830566,
      "loss": 0.6931,
      "policy_logps/chosen": -127.82667541503906,
      "policy_logps/rejected": -130.1011505126953,
      "referece_logps/chosen": -127.82667541503906,
      "referece_logps/rejected": -130.1011505126953,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 1
    },
    {
      "epoch": 0.01,
      "learning_rate": 8e-07,
      "logits/chosen": -2.8690595626831055,
      "logits/rejected": -2.921374797821045,
      "loss": 0.6931,
      "policy_logps/chosen": -127.44483947753906,
      "policy_logps/rejected": -118.97954559326172,
      "referece_logps/chosen": -127.44483947753906,
      "referece_logps/rejected": -118.97954559326172,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 2
    },
    {
      "epoch": 0.02,
      "learning_rate": 1.2e-06,
      "logits/chosen": -2.876248359680176,
      "logits/rejected": -2.9442975521087646,
      "loss": 0.6931,
      "policy_logps/chosen": -151.38050842285156,
      "policy_logps/rejected": -104.58517456054688,
      "referece_logps/chosen": -151.3734893798828,
      "referece_logps/rejected": -104.58135986328125,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.0007032513385638595,
      "rewards/margins": -0.00032021405058912933,
      "rewards/rejected": -0.0003830373170785606,
      "step": 3
    },
    {
      "epoch": 0.03,
      "learning_rate": 1.6e-06,
      "logits/chosen": -2.923973560333252,
      "logits/rejected": -2.9425337314605713,
      "loss": 0.6924,
      "policy_logps/chosen": -121.4244613647461,
      "policy_logps/rejected": -117.06787109375,
      "referece_logps/chosen": -121.42953491210938,
      "referece_logps/rejected": -117.06060028076172,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.0005075454828329384,
      "rewards/margins": 0.0012342334957793355,
      "rewards/rejected": -0.0007266878965310752,
      "step": 4
    },
    {
      "epoch": 0.03,
      "learning_rate": 2e-06,
      "logits/chosen": -2.896630048751831,
      "logits/rejected": -2.874107837677002,
      "loss": 0.6925,
      "policy_logps/chosen": -112.3115234375,
      "policy_logps/rejected": -117.28302764892578,
      "referece_logps/chosen": -112.33646392822266,
      "referece_logps/rejected": -117.27299499511719,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.0024944781325757504,
      "rewards/margins": 0.003496956778690219,
      "rewards/rejected": -0.0010024786461144686,
      "step": 5
    },
    {
      "epoch": 0.04,
      "learning_rate": 1.999783578606323e-06,
      "logits/chosen": -2.8770463466644287,
      "logits/rejected": -2.893326759338379,
      "loss": 0.6908,
      "policy_logps/chosen": -111.89071655273438,
      "policy_logps/rejected": -110.08009338378906,
      "referece_logps/chosen": -111.92901611328125,
      "referece_logps/rejected": -110.06503295898438,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.00383090996183455,
      "rewards/margins": 0.005336493253707886,
      "rewards/rejected": -0.0015055835247039795,
      "step": 6
    },
    {
      "epoch": 0.04,
      "learning_rate": 1.999134408101731e-06,
      "logits/chosen": -2.8742122650146484,
      "logits/rejected": -2.9248428344726562,
      "loss": 0.6872,
      "policy_logps/chosen": -160.53836059570312,
      "policy_logps/rejected": -137.42295837402344,
      "referece_logps/chosen": -160.57460021972656,
      "referece_logps/rejected": -137.33425903320312,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.003623390104621649,
      "rewards/margins": 0.01249313447624445,
      "rewards/rejected": -0.008869742974638939,
      "step": 7
    },
    {
      "epoch": 0.05,
      "learning_rate": 1.998052769474995e-06,
      "logits/chosen": -2.9328360557556152,
      "logits/rejected": -2.975874185562134,
      "loss": 0.6898,
      "policy_logps/chosen": -75.984619140625,
      "policy_logps/rejected": -74.07447814941406,
      "referece_logps/chosen": -76.07889556884766,
      "referece_logps/rejected": -74.01802825927734,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.009427506476640701,
      "rewards/margins": 0.015072083100676537,
      "rewards/rejected": -0.0056445784866809845,
      "step": 8
    },
    {
      "epoch": 0.06,
      "learning_rate": 1.9965391309055927e-06,
      "logits/chosen": -2.882821798324585,
      "logits/rejected": -2.901142120361328,
      "loss": 0.6773,
      "policy_logps/chosen": -107.12052917480469,
      "policy_logps/rejected": -93.86226654052734,
      "referece_logps/chosen": -107.38616943359375,
      "referece_logps/rejected": -93.67011260986328,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.026563584804534912,
      "rewards/margins": 0.04577912390232086,
      "rewards/rejected": -0.019215542823076248,
      "step": 9
    },
    {
      "epoch": 0.06,
      "learning_rate": 1.9945941475610623e-06,
      "logits/chosen": -2.8708529472351074,
      "logits/rejected": -2.9397594928741455,
      "loss": 0.6735,
      "policy_logps/chosen": -123.71336364746094,
      "policy_logps/rejected": -100.81864166259766,
      "referece_logps/chosen": -124.01136779785156,
      "referece_logps/rejected": -100.63360595703125,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.029799818992614746,
      "rewards/margins": 0.048302434384822845,
      "rewards/rejected": -0.0185026116669178,
      "step": 10
    },
    {
      "epoch": 0.07,
      "learning_rate": 1.992218661313415e-06,
      "logits/chosen": -2.8874049186706543,
      "logits/rejected": -2.9123711585998535,
      "loss": 0.6729,
      "policy_logps/chosen": -104.46468353271484,
      "policy_logps/rejected": -98.05982971191406,
      "referece_logps/chosen": -104.58216857910156,
      "referece_logps/rejected": -97.82801055908203,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.011748719029128551,
      "rewards/margins": 0.03493000194430351,
      "rewards/rejected": -0.023181283846497536,
      "step": 11
    },
    {
      "epoch": 0.08,
      "learning_rate": 1.98941370037474e-06,
      "logits/chosen": -2.9089269638061523,
      "logits/rejected": -2.887115478515625,
      "loss": 0.67,
      "policy_logps/chosen": -125.26738739013672,
      "policy_logps/rejected": -118.79678344726562,
      "referece_logps/chosen": -125.5167007446289,
      "referece_logps/rejected": -118.42857360839844,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.024931641295552254,
      "rewards/margins": 0.06175263226032257,
      "rewards/rejected": -0.036820992827415466,
      "step": 12
    },
    {
      "epoch": 0.08,
      "learning_rate": 1.986180478852149e-06,
      "logits/chosen": -2.8961265087127686,
      "logits/rejected": -2.85491943359375,
      "loss": 0.6635,
      "policy_logps/chosen": -152.51437377929688,
      "policy_logps/rejected": -152.42971801757812,
      "referece_logps/chosen": -152.56588745117188,
      "referece_logps/rejected": -152.2041015625,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.005150413140654564,
      "rewards/margins": 0.027712417766451836,
      "rewards/rejected": -0.02256200462579727,
      "step": 13
    },
    {
      "epoch": 0.09,
      "learning_rate": 1.982520396222257e-06,
      "logits/chosen": -2.8753738403320312,
      "logits/rejected": -2.912999153137207,
      "loss": 0.6472,
      "policy_logps/chosen": -101.14571380615234,
      "policy_logps/rejected": -99.60147094726562,
      "referece_logps/chosen": -101.51478576660156,
      "referece_logps/rejected": -98.9403076171875,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.03690744936466217,
      "rewards/margins": 0.10302485525608063,
      "rewards/rejected": -0.06611741334199905,
      "step": 14
    },
    {
      "epoch": 0.1,
      "learning_rate": 1.978435036725432e-06,
      "logits/chosen": -2.9012084007263184,
      "logits/rejected": -2.8953423500061035,
      "loss": 0.6372,
      "policy_logps/chosen": -126.27330017089844,
      "policy_logps/rejected": -128.28578186035156,
      "referece_logps/chosen": -126.68486785888672,
      "referece_logps/rejected": -127.51297760009766,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.04115738719701767,
      "rewards/margins": 0.11843809485435486,
      "rewards/rejected": -0.07728070020675659,
      "step": 15
    },
    {
      "epoch": 0.1,
      "learning_rate": 1.9739261686800657e-06,
      "logits/chosen": -2.9024605751037598,
      "logits/rejected": -2.8853814601898193,
      "loss": 0.6414,
      "policy_logps/chosen": -114.59141540527344,
      "policy_logps/rejected": -128.96331787109375,
      "referece_logps/chosen": -114.98219299316406,
      "referece_logps/rejected": -128.28781127929688,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.03907782956957817,
      "rewards/margins": 0.1066286489367485,
      "rewards/rejected": -0.06755081564188004,
      "step": 16
    },
    {
      "epoch": 0.11,
      "learning_rate": 1.968995743717171e-06,
      "logits/chosen": -2.912461757659912,
      "logits/rejected": -2.9493064880371094,
      "loss": 0.6429,
      "policy_logps/chosen": -111.6058349609375,
      "policy_logps/rejected": -109.18556213378906,
      "referece_logps/chosen": -111.89552307128906,
      "referece_logps/rejected": -108.55181884765625,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.028969965875148773,
      "rewards/margins": 0.09234414994716644,
      "rewards/rejected": -0.06337418407201767,
      "step": 17
    },
    {
      "epoch": 0.11,
      "learning_rate": 1.9636458959356316e-06,
      "logits/chosen": -2.8974556922912598,
      "logits/rejected": -2.89353084564209,
      "loss": 0.6528,
      "policy_logps/chosen": -138.28262329101562,
      "policy_logps/rejected": -128.17782592773438,
      "referece_logps/chosen": -138.3295440673828,
      "referece_logps/rejected": -127.26476287841797,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.004691671580076218,
      "rewards/margins": 0.09599801898002625,
      "rewards/rejected": -0.09130635112524033,
      "step": 18
    },
    {
      "epoch": 0.12,
      "learning_rate": 1.9578789409784727e-06,
      "logits/chosen": -2.8959522247314453,
      "logits/rejected": -2.9369704723358154,
      "loss": 0.6446,
      "policy_logps/chosen": -111.41603088378906,
      "policy_logps/rejected": -104.68195343017578,
      "referece_logps/chosen": -111.60549926757812,
      "referece_logps/rejected": -104.04905700683594,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.018947793170809746,
      "rewards/margins": 0.08223824948072433,
      "rewards/rejected": -0.06329045444726944,
      "step": 19
    },
    {
      "epoch": 0.13,
      "learning_rate": 1.951697375030553e-06,
      "logits/chosen": -2.853641986846924,
      "logits/rejected": -2.8619699478149414,
      "loss": 0.6466,
      "policy_logps/chosen": -145.3450927734375,
      "policy_logps/rejected": -134.5753631591797,
      "referece_logps/chosen": -145.5601348876953,
      "referece_logps/rejected": -133.66200256347656,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.021503955125808716,
      "rewards/margins": 0.11283906549215317,
      "rewards/rejected": -0.09133510291576385,
      "step": 20
    },
    {
      "epoch": 0.13,
      "learning_rate": 1.9451038737381077e-06,
      "logits/chosen": -2.93542218208313,
      "logits/rejected": -2.9347047805786133,
      "loss": 0.6346,
      "policy_logps/chosen": -97.80740356445312,
      "policy_logps/rejected": -92.36180877685547,
      "referece_logps/chosen": -97.90306091308594,
      "referece_logps/rejected": -91.67369079589844,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.009566396474838257,
      "rewards/margins": 0.07837802916765213,
      "rewards/rejected": -0.06881163269281387,
      "step": 21
    },
    {
      "epoch": 0.14,
      "learning_rate": 1.9381012910506143e-06,
      "logits/chosen": -2.8574209213256836,
      "logits/rejected": -2.860379219055176,
      "loss": 0.6102,
      "policy_logps/chosen": -152.8387908935547,
      "policy_logps/rejected": -156.60391235351562,
      "referece_logps/chosen": -152.914306640625,
      "referece_logps/rejected": -155.2355499267578,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.007550956681370735,
      "rewards/margins": 0.14438626170158386,
      "rewards/rejected": -0.13683530688285828,
      "step": 22
    },
    {
      "epoch": 0.15,
      "learning_rate": 1.9306926579854817e-06,
      "logits/chosen": -2.8623623847961426,
      "logits/rejected": -2.8780465126037598,
      "loss": 0.6306,
      "policy_logps/chosen": -143.302490234375,
      "policy_logps/rejected": -127.38278198242188,
      "referece_logps/chosen": -143.29205322265625,
      "referece_logps/rejected": -126.24747467041016,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.0010449867695569992,
      "rewards/margins": 0.11248550564050674,
      "rewards/rejected": -0.11353050917387009,
      "step": 23
    },
    {
      "epoch": 0.15,
      "learning_rate": 1.922881181316097e-06,
      "logits/chosen": -2.949070930480957,
      "logits/rejected": -2.954594135284424,
      "loss": 0.6002,
      "policy_logps/chosen": -74.232177734375,
      "policy_logps/rejected": -73.76258850097656,
      "referece_logps/chosen": -75.20740509033203,
      "referece_logps/rejected": -72.94302368164062,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.09752248972654343,
      "rewards/margins": 0.17947959899902344,
      "rewards/rejected": -0.08195710927248001,
      "step": 24
    },
    {
      "epoch": 0.16,
      "learning_rate": 1.9146702421837946e-06,
      "logits/chosen": -2.8681116104125977,
      "logits/rejected": -2.8957810401916504,
      "loss": 0.612,
      "policy_logps/chosen": -127.76103210449219,
      "policy_logps/rejected": -125.47801208496094,
      "referece_logps/chosen": -128.02197265625,
      "referece_logps/rejected": -124.08172607421875,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.026093529537320137,
      "rewards/margins": 0.1657221019268036,
      "rewards/rejected": -0.1396285593509674,
      "step": 25
    },
    {
      "epoch": 0.17,
      "learning_rate": 1.906063394634356e-06,
      "logits/chosen": -2.862689256668091,
      "logits/rejected": -2.877995252609253,
      "loss": 0.6398,
      "policy_logps/chosen": -120.70744323730469,
      "policy_logps/rejected": -113.76991271972656,
      "referece_logps/chosen": -120.96971130371094,
      "referece_logps/rejected": -112.71368408203125,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.026226602494716644,
      "rewards/margins": 0.13184988498687744,
      "rewards/rejected": -0.1056232899427414,
      "step": 26
    },
    {
      "epoch": 0.17,
      "learning_rate": 1.897064364079664e-06,
      "logits/chosen": -2.9357306957244873,
      "logits/rejected": -2.976917028427124,
      "loss": 0.5904,
      "policy_logps/chosen": -105.41529846191406,
      "policy_logps/rejected": -96.86430358886719,
      "referece_logps/chosen": -106.42874145507812,
      "referece_logps/rejected": -95.49075317382812,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.10134478658437729,
      "rewards/margins": 0.23869961500167847,
      "rewards/rejected": -0.13735483586788177,
      "step": 27
    },
    {
      "epoch": 0.18,
      "learning_rate": 1.8876770456851876e-06,
      "logits/chosen": -2.8511600494384766,
      "logits/rejected": -2.888685941696167,
      "loss": 0.5817,
      "policy_logps/chosen": -135.38853454589844,
      "policy_logps/rejected": -136.4590301513672,
      "referece_logps/chosen": -135.95562744140625,
      "referece_logps/rejected": -134.50482177734375,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.05670913681387901,
      "rewards/margins": 0.2521297335624695,
      "rewards/rejected": -0.19542059302330017,
      "step": 28
    },
    {
      "epoch": 0.19,
      "learning_rate": 1.8779055026839868e-06,
      "logits/chosen": -2.9084866046905518,
      "logits/rejected": -2.9267258644104004,
      "loss": 0.6042,
      "policy_logps/chosen": -134.9838409423828,
      "policy_logps/rejected": -114.98860931396484,
      "referece_logps/chosen": -135.53085327148438,
      "referece_logps/rejected": -113.39382934570312,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.05469997972249985,
      "rewards/margins": 0.21417750418186188,
      "rewards/rejected": -0.15947751700878143,
      "step": 29
    },
    {
      "epoch": 0.19,
      "learning_rate": 1.8677539646179705e-06,
      "logits/chosen": -2.88179349899292,
      "logits/rejected": -2.929935932159424,
      "loss": 0.5827,
      "policy_logps/chosen": -163.58660888671875,
      "policy_logps/rejected": -131.79258728027344,
      "referece_logps/chosen": -163.90985107421875,
      "referece_logps/rejected": -129.81314086914062,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.0323248989880085,
      "rewards/margins": 0.23026807606220245,
      "rewards/rejected": -0.19794318079948425,
      "step": 30
    },
    {
      "epoch": 0.2,
      "learning_rate": 1.8572268255071718e-06,
      "logits/chosen": -2.9416298866271973,
      "logits/rejected": -2.961357831954956,
      "loss": 0.5977,
      "policy_logps/chosen": -98.88802337646484,
      "policy_logps/rejected": -94.90333557128906,
      "referece_logps/chosen": -99.61119842529297,
      "referece_logps/rejected": -93.1702651977539,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.07231828570365906,
      "rewards/margins": 0.24562585353851318,
      "rewards/rejected": -0.17330753803253174,
      "step": 31
    },
    {
      "epoch": 0.2,
      "learning_rate": 1.8463286419478252e-06,
      "logits/chosen": -2.9383907318115234,
      "logits/rejected": -2.880384922027588,
      "loss": 0.5772,
      "policy_logps/chosen": -118.6290283203125,
      "policy_logps/rejected": -126.94461822509766,
      "referece_logps/chosen": -118.98780822753906,
      "referece_logps/rejected": -125.21376037597656,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.03587843477725983,
      "rewards/margins": 0.20896492898464203,
      "rewards/rejected": -0.173086479306221,
      "step": 32
    },
    {
      "epoch": 0.21,
      "learning_rate": 1.835064131140081e-06,
      "logits/chosen": -2.908090829849243,
      "logits/rejected": -2.9141459465026855,
      "loss": 0.5723,
      "policy_logps/chosen": -132.4810333251953,
      "policy_logps/rejected": -130.16603088378906,
      "referece_logps/chosen": -132.7151336669922,
      "referece_logps/rejected": -127.14649963378906,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.02341010421514511,
      "rewards/margins": 0.32536280155181885,
      "rewards/rejected": -0.3019527196884155,
      "step": 33
    },
    {
      "epoch": 0.22,
      "learning_rate": 1.8234381688461941e-06,
      "logits/chosen": -2.9611878395080566,
      "logits/rejected": -2.966728448867798,
      "loss": 0.5734,
      "policy_logps/chosen": -119.45679473876953,
      "policy_logps/rejected": -118.76419830322266,
      "referece_logps/chosen": -118.86337280273438,
      "referece_logps/rejected": -115.91516876220703,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.059341806918382645,
      "rewards/margins": 0.22556202113628387,
      "rewards/rejected": -0.28490379452705383,
      "step": 34
    },
    {
      "epoch": 0.22,
      "learning_rate": 1.8114557872800905e-06,
      "logits/chosen": -2.967761993408203,
      "logits/rejected": -2.900844097137451,
      "loss": 0.5633,
      "policy_logps/chosen": -130.78848266601562,
      "policy_logps/rejected": -142.6917724609375,
      "referece_logps/chosen": -130.37750244140625,
      "referece_logps/rejected": -139.14178466796875,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.041097551584243774,
      "rewards/margins": 0.31390050053596497,
      "rewards/rejected": -0.35499805212020874,
      "step": 35
    },
    {
      "epoch": 0.23,
      "learning_rate": 1.7991221729292058e-06,
      "logits/chosen": -2.8585638999938965,
      "logits/rejected": -2.90903377532959,
      "loss": 0.5649,
      "policy_logps/chosen": -137.8734893798828,
      "policy_logps/rejected": -127.91438293457031,
      "referece_logps/chosen": -137.9367218017578,
      "referece_logps/rejected": -125.2115478515625,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.0063229575753211975,
      "rewards/margins": 0.27660617232322693,
      "rewards/rejected": -0.27028322219848633,
      "step": 36
    },
    {
      "epoch": 0.24,
      "learning_rate": 1.7864426643095536e-06,
      "logits/chosen": -2.9357798099517822,
      "logits/rejected": -2.900618553161621,
      "loss": 0.5592,
      "policy_logps/chosen": -151.2121124267578,
      "policy_logps/rejected": -134.10804748535156,
      "referece_logps/chosen": -150.83583068847656,
      "referece_logps/rejected": -131.0611572265625,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.03762848675251007,
      "rewards/margins": 0.26705947518348694,
      "rewards/rejected": -0.3046879470348358,
      "step": 37
    },
    {
      "epoch": 0.24,
      "learning_rate": 1.7734227496549878e-06,
      "logits/chosen": -2.9032468795776367,
      "logits/rejected": -2.9027702808380127,
      "loss": 0.5684,
      "policy_logps/chosen": -104.47734069824219,
      "policy_logps/rejected": -106.55696105957031,
      "referece_logps/chosen": -105.3730697631836,
      "referece_logps/rejected": -104.14241027832031,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.0895722359418869,
      "rewards/margins": 0.3310272991657257,
      "rewards/rejected": -0.241455078125,
      "step": 38
    },
    {
      "epoch": 0.25,
      "learning_rate": 1.7600680645416582e-06,
      "logits/chosen": -2.9712843894958496,
      "logits/rejected": -2.894430637359619,
      "loss": 0.5499,
      "policy_logps/chosen": -126.83047485351562,
      "policy_logps/rejected": -138.54771423339844,
      "referece_logps/chosen": -126.56389617919922,
      "referece_logps/rejected": -135.42103576660156,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.026657823473215103,
      "rewards/margins": 0.2860097885131836,
      "rewards/rejected": -0.312667578458786,
      "step": 39
    },
    {
      "epoch": 0.26,
      "learning_rate": 1.7463843894486936e-06,
      "logits/chosen": -2.942542552947998,
      "logits/rejected": -2.9863080978393555,
      "loss": 0.5534,
      "policy_logps/chosen": -93.51473999023438,
      "policy_logps/rejected": -94.46737670898438,
      "referece_logps/chosen": -93.94270324707031,
      "referece_logps/rejected": -91.81558227539062,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.04279506206512451,
      "rewards/margins": 0.3079749345779419,
      "rewards/rejected": -0.2651798725128174,
      "step": 40
    },
    {
      "epoch": 0.26,
      "learning_rate": 1.7323776472561625e-06,
      "logits/chosen": -2.9191946983337402,
      "logits/rejected": -2.935122013092041,
      "loss": 0.5682,
      "policy_logps/chosen": -128.17813110351562,
      "policy_logps/rejected": -139.06613159179688,
      "referece_logps/chosen": -127.54570770263672,
      "referece_logps/rejected": -135.201904296875,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.06324195861816406,
      "rewards/margins": 0.32317861914634705,
      "rewards/rejected": -0.3864205777645111,
      "step": 41
    },
    {
      "epoch": 0.27,
      "learning_rate": 1.7180539006813969e-06,
      "logits/chosen": -2.920085906982422,
      "logits/rejected": -2.906773567199707,
      "loss": 0.5454,
      "policy_logps/chosen": -126.83687591552734,
      "policy_logps/rejected": -114.17430877685547,
      "referece_logps/chosen": -126.77012634277344,
      "referece_logps/rejected": -110.9568862915039,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.006674099713563919,
      "rewards/margins": 0.31506818532943726,
      "rewards/rejected": -0.3217422664165497,
      "step": 42
    },
    {
      "epoch": 0.27,
      "learning_rate": 1.7034193496547902e-06,
      "logits/chosen": -2.8620564937591553,
      "logits/rejected": -2.9222469329833984,
      "loss": 0.534,
      "policy_logps/chosen": -125.01854705810547,
      "policy_logps/rejected": -116.95589447021484,
      "referece_logps/chosen": -125.00530242919922,
      "referece_logps/rejected": -113.22078704833984,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.0013249870389699936,
      "rewards/margins": 0.3721860647201538,
      "rewards/rejected": -0.37351107597351074,
      "step": 43
    },
    {
      "epoch": 0.28,
      "learning_rate": 1.6884803286362e-06,
      "logits/chosen": -2.8868565559387207,
      "logits/rejected": -2.91511869430542,
      "loss": 0.5459,
      "policy_logps/chosen": -150.9369354248047,
      "policy_logps/rejected": -144.1173858642578,
      "referece_logps/chosen": -150.93002319335938,
      "referece_logps/rejected": -139.8741455078125,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.0006910450756549835,
      "rewards/margins": 0.42363405227661133,
      "rewards/rejected": -0.4243250787258148,
      "step": 44
    },
    {
      "epoch": 0.29,
      "learning_rate": 1.673243303873124e-06,
      "logits/chosen": -2.9313180446624756,
      "logits/rejected": -2.9371540546417236,
      "loss": 0.5569,
      "policy_logps/chosen": -122.50914764404297,
      "policy_logps/rejected": -110.94712829589844,
      "referece_logps/chosen": -122.10940551757812,
      "referece_logps/rejected": -108.29950714111328,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.03997454792261124,
      "rewards/margins": 0.22478806972503662,
      "rewards/rejected": -0.26476261019706726,
      "step": 45
    },
    {
      "epoch": 0.29,
      "learning_rate": 1.6577148706018328e-06,
      "logits/chosen": -2.894786834716797,
      "logits/rejected": -2.977565288543701,
      "loss": 0.5775,
      "policy_logps/chosen": -114.12425231933594,
      "policy_logps/rejected": -113.44467163085938,
      "referece_logps/chosen": -112.93356323242188,
      "referece_logps/rejected": -110.42334747314453,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.11906924843788147,
      "rewards/margins": 0.18306350708007812,
      "rewards/rejected": -0.3021327555179596,
      "step": 46
    },
    {
      "epoch": 0.3,
      "learning_rate": 1.6419017501926656e-06,
      "logits/chosen": -2.90000581741333,
      "logits/rejected": -2.9254188537597656,
      "loss": 0.5176,
      "policy_logps/chosen": -128.19607543945312,
      "policy_logps/rejected": -122.17625427246094,
      "referece_logps/chosen": -128.63787841796875,
      "referece_logps/rejected": -118.01435852050781,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.04418013244867325,
      "rewards/margins": 0.4603692889213562,
      "rewards/rejected": -0.41618919372558594,
      "step": 47
    },
    {
      "epoch": 0.31,
      "learning_rate": 1.6258107872407374e-06,
      "logits/chosen": -2.9072225093841553,
      "logits/rejected": -2.9065234661102295,
      "loss": 0.5243,
      "policy_logps/chosen": -125.48599243164062,
      "policy_logps/rejected": -129.7559814453125,
      "referece_logps/chosen": -126.17718505859375,
      "referece_logps/rejected": -127.1695785522461,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.06911970674991608,
      "rewards/margins": 0.32776087522506714,
      "rewards/rejected": -0.25864115357398987,
      "step": 48
    },
    {
      "epoch": 0.31,
      "learning_rate": 1.6094489466033042e-06,
      "logits/chosen": -2.917146682739258,
      "logits/rejected": -2.9667577743530273,
      "loss": 0.5559,
      "policy_logps/chosen": -108.18141174316406,
      "policy_logps/rejected": -99.39623260498047,
      "referece_logps/chosen": -108.76286315917969,
      "referece_logps/rejected": -96.53907775878906,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.0581454373896122,
      "rewards/margins": 0.34386110305786133,
      "rewards/rejected": -0.28571566939353943,
      "step": 49
    },
    {
      "epoch": 0.32,
      "learning_rate": 1.5928233103850727e-06,
      "logits/chosen": -2.906874179840088,
      "logits/rejected": -2.919196844100952,
      "loss": 0.5364,
      "policy_logps/chosen": -161.1468048095703,
      "policy_logps/rejected": -143.97592163085938,
      "referece_logps/chosen": -160.62081909179688,
      "referece_logps/rejected": -140.28585815429688,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.052598677575588226,
      "rewards/margins": 0.31640806794166565,
      "rewards/rejected": -0.3690067529678345,
      "step": 50
    },
    {
      "epoch": 0.33,
      "learning_rate": 1.575941074872766e-06,
      "logits/chosen": -2.957002878189087,
      "logits/rejected": -2.9672722816467285,
      "loss": 0.5172,
      "policy_logps/chosen": -119.42903900146484,
      "policy_logps/rejected": -116.86624145507812,
      "referece_logps/chosen": -120.13378143310547,
      "referece_logps/rejected": -113.2657241821289,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.07047442346811295,
      "rewards/margins": 0.4305253326892853,
      "rewards/rejected": -0.36005088686943054,
      "step": 51
    },
    {
      "epoch": 0.33,
      "learning_rate": 1.5588095474202594e-06,
      "logits/chosen": -2.8591084480285645,
      "logits/rejected": -2.8590025901794434,
      "loss": 0.4933,
      "policy_logps/chosen": -157.062744140625,
      "policy_logps/rejected": -151.7144775390625,
      "referece_logps/chosen": -156.53114318847656,
      "referece_logps/rejected": -147.15911865234375,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.05316000431776047,
      "rewards/margins": 0.4023759067058563,
      "rewards/rejected": -0.4555359482765198,
      "step": 52
    },
    {
      "epoch": 0.34,
      "learning_rate": 1.5414361432856474e-06,
      "logits/chosen": -2.9438529014587402,
      "logits/rejected": -2.9987499713897705,
      "loss": 0.4979,
      "policy_logps/chosen": -117.94309997558594,
      "policy_logps/rejected": -108.414306640625,
      "referece_logps/chosen": -118.55531311035156,
      "referece_logps/rejected": -104.54682922363281,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.061221349984407425,
      "rewards/margins": 0.4479690194129944,
      "rewards/rejected": -0.38674765825271606,
      "step": 53
    },
    {
      "epoch": 0.34,
      "learning_rate": 1.5238283824216013e-06,
      "logits/chosen": -2.9244260787963867,
      "logits/rejected": -2.9373199939727783,
      "loss": 0.4961,
      "policy_logps/chosen": -119.873779296875,
      "policy_logps/rejected": -108.0604248046875,
      "referece_logps/chosen": -120.80511474609375,
      "referece_logps/rejected": -104.79627990722656,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.09313352406024933,
      "rewards/margins": 0.4195476770401001,
      "rewards/rejected": -0.32641416788101196,
      "step": 54
    },
    {
      "epoch": 0.35,
      "learning_rate": 1.5059938862204125e-06,
      "logits/chosen": -2.920050621032715,
      "logits/rejected": -2.9494502544403076,
      "loss": 0.5339,
      "policy_logps/chosen": -128.6884307861328,
      "policy_logps/rejected": -124.91862487792969,
      "referece_logps/chosen": -127.86734008789062,
      "referece_logps/rejected": -120.62930297851562,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.08210951089859009,
      "rewards/margins": 0.34682315587997437,
      "rewards/rejected": -0.42893266677856445,
      "step": 55
    },
    {
      "epoch": 0.36,
      "learning_rate": 1.4879403742151283e-06,
      "logits/chosen": -2.907794713973999,
      "logits/rejected": -2.9134182929992676,
      "loss": 0.5389,
      "policy_logps/chosen": -129.15188598632812,
      "policy_logps/rejected": -135.58035278320312,
      "referece_logps/chosen": -127.82069396972656,
      "referece_logps/rejected": -131.39085388183594,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.13311973214149475,
      "rewards/margins": 0.28583019971847534,
      "rewards/rejected": -0.4189499020576477,
      "step": 56
    },
    {
      "epoch": 0.36,
      "learning_rate": 1.4696756607382058e-06,
      "logits/chosen": -2.9352166652679443,
      "logits/rejected": -2.9330966472625732,
      "loss": 0.543,
      "policy_logps/chosen": -125.51739501953125,
      "policy_logps/rejected": -128.10951232910156,
      "referece_logps/chosen": -124.24830627441406,
      "referece_logps/rejected": -123.51809692382812,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.12690886855125427,
      "rewards/margins": 0.33223241567611694,
      "rewards/rejected": -0.4591412842273712,
      "step": 57
    },
    {
      "epoch": 0.37,
      "learning_rate": 1.4512076515391374e-06,
      "logits/chosen": -2.941624164581299,
      "logits/rejected": -2.9078333377838135,
      "loss": 0.4758,
      "policy_logps/chosen": -109.23553466796875,
      "policy_logps/rejected": -104.69176483154297,
      "referece_logps/chosen": -110.22058868408203,
      "referece_logps/rejected": -100.0502700805664,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.09850560873746872,
      "rewards/margins": 0.5626559257507324,
      "rewards/rejected": -0.4641503393650055,
      "step": 58
    },
    {
      "epoch": 0.38,
      "learning_rate": 1.432544340362501e-06,
      "logits/chosen": -2.972963809967041,
      "logits/rejected": -2.9390883445739746,
      "loss": 0.4904,
      "policy_logps/chosen": -95.29005432128906,
      "policy_logps/rejected": -121.40338897705078,
      "referece_logps/chosen": -95.24687194824219,
      "referece_logps/rejected": -116.24635314941406,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.004318548366427422,
      "rewards/margins": 0.5113850235939026,
      "rewards/rejected": -0.515703558921814,
      "step": 59
    },
    {
      "epoch": 0.38,
      "learning_rate": 1.4136938054879282e-06,
      "logits/chosen": -2.9395623207092285,
      "logits/rejected": -2.953756809234619,
      "loss": 0.5286,
      "policy_logps/chosen": -122.0515365600586,
      "policy_logps/rejected": -128.63021850585938,
      "referece_logps/chosen": -120.7651596069336,
      "referece_logps/rejected": -123.8788070678711,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1286384016275406,
      "rewards/margins": 0.34650319814682007,
      "rewards/rejected": -0.47514158487319946,
      "step": 60
    },
    {
      "epoch": 0.39,
      "learning_rate": 1.3946642062334763e-06,
      "logits/chosen": -2.9107141494750977,
      "logits/rejected": -2.9332330226898193,
      "loss": 0.4832,
      "policy_logps/chosen": -121.9334716796875,
      "policy_logps/rejected": -119.00639343261719,
      "referece_logps/chosen": -122.2487564086914,
      "referece_logps/rejected": -114.04248809814453,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.03152900189161301,
      "rewards/margins": 0.5279202461242676,
      "rewards/rejected": -0.49639129638671875,
      "step": 61
    },
    {
      "epoch": 0.4,
      "learning_rate": 1.37546377942393e-06,
      "logits/chosen": -2.950266122817993,
      "logits/rejected": -2.9494924545288086,
      "loss": 0.5178,
      "policy_logps/chosen": -104.2540283203125,
      "policy_logps/rejected": -125.53955841064453,
      "referece_logps/chosen": -104.66864776611328,
      "referece_logps/rejected": -121.41145324707031,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.04146187752485275,
      "rewards/margins": 0.45427215099334717,
      "rewards/rejected": -0.4128102958202362,
      "step": 62
    },
    {
      "epoch": 0.4,
      "learning_rate": 1.3561008358255469e-06,
      "logits/chosen": -2.917292594909668,
      "logits/rejected": -2.952850818634033,
      "loss": 0.5263,
      "policy_logps/chosen": -118.34769439697266,
      "policy_logps/rejected": -112.80188751220703,
      "referece_logps/chosen": -118.5436782836914,
      "referece_logps/rejected": -109.08131408691406,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.01959807053208351,
      "rewards/margins": 0.39165574312210083,
      "rewards/rejected": -0.3720576763153076,
      "step": 63
    },
    {
      "epoch": 0.41,
      "learning_rate": 1.3365837565488062e-06,
      "logits/chosen": -2.92209529876709,
      "logits/rejected": -2.9668972492218018,
      "loss": 0.4813,
      "policy_logps/chosen": -161.57546997070312,
      "policy_logps/rejected": -137.75039672851562,
      "referece_logps/chosen": -161.21104431152344,
      "referece_logps/rejected": -130.89552307128906,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.03644174337387085,
      "rewards/margins": 0.6490457057952881,
      "rewards/rejected": -0.6854873895645142,
      "step": 64
    },
    {
      "epoch": 0.41,
      "learning_rate": 1.3169209894207027e-06,
      "logits/chosen": -2.9616472721099854,
      "logits/rejected": -2.961239814758301,
      "loss": 0.4486,
      "policy_logps/chosen": -154.44467163085938,
      "policy_logps/rejected": -158.21372985839844,
      "referece_logps/chosen": -154.07650756835938,
      "referece_logps/rejected": -151.34902954101562,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.03681756183505058,
      "rewards/margins": 0.6496531367301941,
      "rewards/rejected": -0.6864707469940186,
      "step": 65
    },
    {
      "epoch": 0.42,
      "learning_rate": 1.2971210453281673e-06,
      "logits/chosen": -2.892361640930176,
      "logits/rejected": -2.8887600898742676,
      "loss": 0.4939,
      "policy_logps/chosen": -121.81805419921875,
      "policy_logps/rejected": -114.29660034179688,
      "referece_logps/chosen": -122.0481948852539,
      "referece_logps/rejected": -110.4261474609375,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.023014426231384277,
      "rewards/margins": 0.41005975008010864,
      "rewards/rejected": -0.38704538345336914,
      "step": 66
    },
    {
      "epoch": 0.43,
      "learning_rate": 1.2771924945341906e-06,
      "logits/chosen": -2.91727352142334,
      "logits/rejected": -2.922391891479492,
      "loss": 0.4868,
      "policy_logps/chosen": -111.62061309814453,
      "policy_logps/rejected": -106.5035400390625,
      "referece_logps/chosen": -112.6988525390625,
      "referece_logps/rejected": -101.50572967529297,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.10782448947429657,
      "rewards/margins": 0.6076046228408813,
      "rewards/rejected": -0.49978014826774597,
      "step": 67
    },
    {
      "epoch": 0.43,
      "learning_rate": 1.257143962968246e-06,
      "logits/chosen": -2.9374663829803467,
      "logits/rejected": -2.9512386322021484,
      "loss": 0.5144,
      "policy_logps/chosen": -149.2052001953125,
      "policy_logps/rejected": -130.6556396484375,
      "referece_logps/chosen": -147.80715942382812,
      "referece_logps/rejected": -125.28959655761719,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.13980263471603394,
      "rewards/margins": 0.39680200815200806,
      "rewards/rejected": -0.536604642868042,
      "step": 68
    },
    {
      "epoch": 0.44,
      "learning_rate": 1.236984128492619e-06,
      "logits/chosen": -2.9574966430664062,
      "logits/rejected": -2.9541845321655273,
      "loss": 0.4811,
      "policy_logps/chosen": -105.58607482910156,
      "policy_logps/rejected": -107.0927734375,
      "referece_logps/chosen": -106.00244903564453,
      "referece_logps/rejected": -103.1064453125,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.041637033224105835,
      "rewards/margins": 0.44027072191238403,
      "rewards/rejected": -0.3986337184906006,
      "step": 69
    },
    {
      "epoch": 0.45,
      "learning_rate": 1.2167217171462566e-06,
      "logits/chosen": -2.96567964553833,
      "logits/rejected": -2.9865453243255615,
      "loss": 0.4782,
      "policy_logps/chosen": -137.23434448242188,
      "policy_logps/rejected": -116.43749237060547,
      "referece_logps/chosen": -137.0900421142578,
      "referece_logps/rejected": -110.79519653320312,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.014430008828639984,
      "rewards/margins": 0.5497984886169434,
      "rewards/rejected": -0.5642285346984863,
      "step": 70
    },
    {
      "epoch": 0.45,
      "learning_rate": 1.1963654993677643e-06,
      "logits/chosen": -2.9047865867614746,
      "logits/rejected": -2.9375927448272705,
      "loss": 0.4831,
      "policy_logps/chosen": -142.0609588623047,
      "policy_logps/rejected": -138.23785400390625,
      "referece_logps/chosen": -140.9311981201172,
      "referece_logps/rejected": -132.38949584960938,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.1129767894744873,
      "rewards/margins": 0.47185903787612915,
      "rewards/rejected": -0.5848358869552612,
      "step": 71
    },
    {
      "epoch": 0.46,
      "learning_rate": 1.1759242861991854e-06,
      "logits/chosen": -2.9874184131622314,
      "logits/rejected": -2.9638171195983887,
      "loss": 0.4436,
      "policy_logps/chosen": -111.93683624267578,
      "policy_logps/rejected": -113.12979888916016,
      "referece_logps/chosen": -111.51397705078125,
      "referece_logps/rejected": -107.90960693359375,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.042286355048418045,
      "rewards/margins": 0.4797336161136627,
      "rewards/rejected": -0.5220199227333069,
      "step": 72
    },
    {
      "epoch": 0.47,
      "learning_rate": 1.155406925472205e-06,
      "logits/chosen": -2.927311420440674,
      "logits/rejected": -2.921337366104126,
      "loss": 0.4284,
      "policy_logps/chosen": -147.83377075195312,
      "policy_logps/rejected": -148.07089233398438,
      "referece_logps/chosen": -147.18499755859375,
      "referece_logps/rejected": -140.68211364746094,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.0648760199546814,
      "rewards/margins": 0.6740000247955322,
      "rewards/rejected": -0.7388760447502136,
      "step": 73
    },
    {
      "epoch": 0.47,
      "learning_rate": 1.1348222979784287e-06,
      "logits/chosen": -2.9232394695281982,
      "logits/rejected": -2.9430999755859375,
      "loss": 0.5002,
      "policy_logps/chosen": -143.42274475097656,
      "policy_logps/rejected": -143.17144775390625,
      "referece_logps/chosen": -141.336669921875,
      "referece_logps/rejected": -136.30712890625,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.20860746502876282,
      "rewards/margins": 0.4778253436088562,
      "rewards/rejected": -0.6864327788352966,
      "step": 74
    },
    {
      "epoch": 0.48,
      "learning_rate": 1.1141793136253986e-06,
      "logits/chosen": -2.8802056312561035,
      "logits/rejected": -2.9181809425354004,
      "loss": 0.5141,
      "policy_logps/chosen": -155.84146118164062,
      "policy_logps/rejected": -145.16802978515625,
      "referece_logps/chosen": -154.25274658203125,
      "referece_logps/rejected": -140.40045166015625,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.15887098014354706,
      "rewards/margins": 0.31788796186447144,
      "rewards/rejected": -0.4767589569091797,
      "step": 75
    },
    {
      "epoch": 0.48,
      "learning_rate": 1.09348690758e-06,
      "logits/chosen": -2.921013355255127,
      "logits/rejected": -2.949887275695801,
      "loss": 0.4978,
      "policy_logps/chosen": -140.55694580078125,
      "policy_logps/rejected": -137.7884521484375,
      "referece_logps/chosen": -139.3011474609375,
      "referece_logps/rejected": -130.94146728515625,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.12557877600193024,
      "rewards/margins": 0.5591215491294861,
      "rewards/rejected": -0.6847003102302551,
      "step": 76
    },
    {
      "epoch": 0.49,
      "learning_rate": 1.072754036400944e-06,
      "logits/chosen": -2.9696455001831055,
      "logits/rejected": -2.9857373237609863,
      "loss": 0.417,
      "policy_logps/chosen": -113.93096923828125,
      "policy_logps/rejected": -107.09919738769531,
      "referece_logps/chosen": -115.27886962890625,
      "referece_logps/rejected": -102.10234069824219,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.1347896158695221,
      "rewards/margins": 0.6344748139381409,
      "rewards/rejected": -0.49968522787094116,
      "step": 77
    },
    {
      "epoch": 0.5,
      "learning_rate": 1.0519896741619803e-06,
      "logits/chosen": -2.919851303100586,
      "logits/rejected": -2.9218404293060303,
      "loss": 0.4688,
      "policy_logps/chosen": -155.71444702148438,
      "policy_logps/rejected": -144.62152099609375,
      "referece_logps/chosen": -155.02752685546875,
      "referece_logps/rejected": -138.43026733398438,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.0686924085021019,
      "rewards/margins": 0.5504311323165894,
      "rewards/rejected": -0.6191235184669495,
      "step": 78
    },
    {
      "epoch": 0.5,
      "learning_rate": 1.031202808567539e-06,
      "logits/chosen": -2.8995673656463623,
      "logits/rejected": -2.930441379547119,
      "loss": 0.4539,
      "policy_logps/chosen": -155.89486694335938,
      "policy_logps/rejected": -133.66708374023438,
      "referece_logps/chosen": -154.44532775878906,
      "referece_logps/rejected": -126.08793640136719,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.14495408535003662,
      "rewards/margins": 0.6129606366157532,
      "rewards/rejected": -0.7579147815704346,
      "step": 79
    },
    {
      "epoch": 0.51,
      "learning_rate": 1.0104024370624642e-06,
      "logits/chosen": -2.9370193481445312,
      "logits/rejected": -2.9806201457977295,
      "loss": 0.4106,
      "policy_logps/chosen": -134.17471313476562,
      "policy_logps/rejected": -112.07872772216797,
      "referece_logps/chosen": -134.83750915527344,
      "referece_logps/rejected": -105.02826690673828,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.06628021597862244,
      "rewards/margins": 0.7713260650634766,
      "rewards/rejected": -0.7050458192825317,
      "step": 80
    },
    {
      "epoch": 0.52,
      "learning_rate": 9.895975629375357e-07,
      "logits/chosen": -2.9489431381225586,
      "logits/rejected": -2.931128978729248,
      "loss": 0.4902,
      "policy_logps/chosen": -132.9930877685547,
      "policy_logps/rejected": -148.03472900390625,
      "referece_logps/chosen": -131.95632934570312,
      "referece_logps/rejected": -141.67227172851562,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.1036761999130249,
      "rewards/margins": 0.5325698852539062,
      "rewards/rejected": -0.6362460851669312,
      "step": 81
    },
    {
      "epoch": 0.52,
      "learning_rate": 9.687971914324607e-07,
      "logits/chosen": -2.911447525024414,
      "logits/rejected": -2.9363858699798584,
      "loss": 0.4865,
      "policy_logps/chosen": -136.85560607910156,
      "policy_logps/rejected": -110.8121337890625,
      "referece_logps/chosen": -135.94354248046875,
      "referece_logps/rejected": -105.58758544921875,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.09120647609233856,
      "rewards/margins": 0.4312480390071869,
      "rewards/rejected": -0.5224545001983643,
      "step": 82
    },
    {
      "epoch": 0.53,
      "learning_rate": 9.480103258380197e-07,
      "logits/chosen": -2.9371180534362793,
      "logits/rejected": -2.9503769874572754,
      "loss": 0.4557,
      "policy_logps/chosen": -150.19725036621094,
      "policy_logps/rejected": -149.049072265625,
      "referece_logps/chosen": -147.80014038085938,
      "referece_logps/rejected": -141.38333129882812,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.23970948159694672,
      "rewards/margins": 0.5268632173538208,
      "rewards/rejected": -0.7665727138519287,
      "step": 83
    },
    {
      "epoch": 0.54,
      "learning_rate": 9.272459635990562e-07,
      "logits/chosen": -2.914623737335205,
      "logits/rejected": -2.909700393676758,
      "loss": 0.4753,
      "policy_logps/chosen": -149.44277954101562,
      "policy_logps/rejected": -158.23593139648438,
      "referece_logps/chosen": -147.42015075683594,
      "referece_logps/rejected": -149.85214233398438,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.2022620439529419,
      "rewards/margins": 0.6361156702041626,
      "rewards/rejected": -0.838377833366394,
      "step": 84
    },
    {
      "epoch": 0.54,
      "learning_rate": 9.065130924199998e-07,
      "logits/chosen": -2.964315414428711,
      "logits/rejected": -2.9234838485717773,
      "loss": 0.5092,
      "policy_logps/chosen": -133.36880493164062,
      "policy_logps/rejected": -146.08169555664062,
      "referece_logps/chosen": -132.2157440185547,
      "referece_logps/rejected": -140.08538818359375,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.11530620604753494,
      "rewards/margins": 0.4843238890171051,
      "rewards/rejected": -0.5996301174163818,
      "step": 85
    },
    {
      "epoch": 0.55,
      "learning_rate": 8.858206863746017e-07,
      "logits/chosen": -2.909698486328125,
      "logits/rejected": -2.918955087661743,
      "loss": 0.4525,
      "policy_logps/chosen": -132.2825469970703,
      "policy_logps/rejected": -136.70962524414062,
      "referece_logps/chosen": -130.87799072265625,
      "referece_logps/rejected": -130.4683837890625,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.1404554843902588,
      "rewards/margins": 0.48366838693618774,
      "rewards/rejected": -0.6241238713264465,
      "step": 86
    },
    {
      "epoch": 0.56,
      "learning_rate": 8.651777020215712e-07,
      "logits/chosen": -2.8775806427001953,
      "logits/rejected": -2.876236915588379,
      "loss": 0.4829,
      "policy_logps/chosen": -138.22640991210938,
      "policy_logps/rejected": -135.67745971679688,
      "referece_logps/chosen": -137.68087768554688,
      "referece_logps/rejected": -129.38088989257812,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.05455498397350311,
      "rewards/margins": 0.5751017928123474,
      "rewards/rejected": -0.6296567916870117,
      "step": 87
    },
    {
      "epoch": 0.56,
      "learning_rate": 8.445930745277951e-07,
      "logits/chosen": -2.9498825073242188,
      "logits/rejected": -2.9456593990325928,
      "loss": 0.4666,
      "policy_logps/chosen": -99.36846160888672,
      "policy_logps/rejected": -110.91441345214844,
      "referece_logps/chosen": -97.86239624023438,
      "referece_logps/rejected": -104.65141296386719,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.15060700476169586,
      "rewards/margins": 0.47569307684898376,
      "rewards/rejected": -0.6263000965118408,
      "step": 88
    },
    {
      "epoch": 0.57,
      "learning_rate": 8.240757138008148e-07,
      "logits/chosen": -2.9100852012634277,
      "logits/rejected": -2.9250974655151367,
      "loss": 0.4921,
      "policy_logps/chosen": -144.23516845703125,
      "policy_logps/rejected": -140.12838745117188,
      "referece_logps/chosen": -142.4854736328125,
      "referece_logps/rejected": -133.91094970703125,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.17496907711029053,
      "rewards/margins": 0.4467761516571045,
      "rewards/rejected": -0.621745228767395,
      "step": 89
    },
    {
      "epoch": 0.57,
      "learning_rate": 8.036345006322358e-07,
      "logits/chosen": -2.9227583408355713,
      "logits/rejected": -2.936587333679199,
      "loss": 0.4516,
      "policy_logps/chosen": -130.54054260253906,
      "policy_logps/rejected": -137.36415100097656,
      "referece_logps/chosen": -129.6831512451172,
      "referece_logps/rejected": -129.63467407226562,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.08573976159095764,
      "rewards/margins": 0.6872075796127319,
      "rewards/rejected": -0.772947371006012,
      "step": 90
    },
    {
      "epoch": 0.58,
      "learning_rate": 7.832782828537435e-07,
      "logits/chosen": -2.946342706680298,
      "logits/rejected": -2.9744181632995605,
      "loss": 0.4716,
      "policy_logps/chosen": -155.42410278320312,
      "policy_logps/rejected": -155.65284729003906,
      "referece_logps/chosen": -153.79293823242188,
      "referece_logps/rejected": -145.5734100341797,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.16311725974082947,
      "rewards/margins": 0.8448256850242615,
      "rewards/rejected": -1.0079429149627686,
      "step": 91
    },
    {
      "epoch": 0.59,
      "learning_rate": 7.630158715073812e-07,
      "logits/chosen": -2.9480092525482178,
      "logits/rejected": -2.971871852874756,
      "loss": 0.4588,
      "policy_logps/chosen": -117.96566772460938,
      "policy_logps/rejected": -120.06287384033203,
      "referece_logps/chosen": -117.35060119628906,
      "referece_logps/rejected": -114.15763092041016,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.06150689721107483,
      "rewards/margins": 0.5290161371231079,
      "rewards/rejected": -0.5905230045318604,
      "step": 92
    },
    {
      "epoch": 0.59,
      "learning_rate": 7.428560370317541e-07,
      "logits/chosen": -2.9216105937957764,
      "logits/rejected": -2.973964214324951,
      "loss": 0.4124,
      "policy_logps/chosen": -118.88021850585938,
      "policy_logps/rejected": -116.6799087524414,
      "referece_logps/chosen": -118.77764892578125,
      "referece_logps/rejected": -110.41334533691406,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.010256083682179451,
      "rewards/margins": 0.6163986921310425,
      "rewards/rejected": -0.6266547441482544,
      "step": 93
    },
    {
      "epoch": 0.6,
      "learning_rate": 7.228075054658095e-07,
      "logits/chosen": -2.914088249206543,
      "logits/rejected": -2.949735164642334,
      "loss": 0.5151,
      "policy_logps/chosen": -156.11376953125,
      "policy_logps/rejected": -136.4941864013672,
      "referece_logps/chosen": -154.32318115234375,
      "referece_logps/rejected": -129.20169067382812,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.17905890941619873,
      "rewards/margins": 0.5501898527145386,
      "rewards/rejected": -0.7292487621307373,
      "step": 94
    },
    {
      "epoch": 0.61,
      "learning_rate": 7.028789546718325e-07,
      "logits/chosen": -2.9285836219787598,
      "logits/rejected": -2.9289567470550537,
      "loss": 0.46,
      "policy_logps/chosen": -138.6322021484375,
      "policy_logps/rejected": -140.56346130371094,
      "referece_logps/chosen": -137.0457763671875,
      "referece_logps/rejected": -134.56263732910156,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.15864019095897675,
      "rewards/margins": 0.44144219160079956,
      "rewards/rejected": -0.6000823974609375,
      "step": 95
    },
    {
      "epoch": 0.61,
      "learning_rate": 6.830790105792973e-07,
      "logits/chosen": -2.9379961490631104,
      "logits/rejected": -2.9724767208099365,
      "loss": 0.4291,
      "policy_logps/chosen": -101.97422790527344,
      "policy_logps/rejected": -120.30264282226562,
      "referece_logps/chosen": -102.00912475585938,
      "referece_logps/rejected": -112.79841613769531,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.0034902598708868027,
      "rewards/margins": 0.7539129257202148,
      "rewards/rejected": -0.7504226565361023,
      "step": 96
    },
    {
      "epoch": 0.62,
      "learning_rate": 6.634162434511938e-07,
      "logits/chosen": -2.9047234058380127,
      "logits/rejected": -2.9031195640563965,
      "loss": 0.4965,
      "policy_logps/chosen": -143.34796142578125,
      "policy_logps/rejected": -137.45285034179688,
      "referece_logps/chosen": -140.2106170654297,
      "referece_logps/rejected": -128.766845703125,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.3137343227863312,
      "rewards/margins": 0.5548651814460754,
      "rewards/rejected": -0.868599534034729,
      "step": 97
    },
    {
      "epoch": 0.63,
      "learning_rate": 6.43899164174453e-07,
      "logits/chosen": -2.9683728218078613,
      "logits/rejected": -2.9473183155059814,
      "loss": 0.4951,
      "policy_logps/chosen": -99.94713592529297,
      "policy_logps/rejected": -99.26773834228516,
      "referece_logps/chosen": -100.4857406616211,
      "referece_logps/rejected": -93.8021011352539,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.05386023223400116,
      "rewards/margins": 0.6004235148429871,
      "rewards/rejected": -0.5465632677078247,
      "step": 98
    },
    {
      "epoch": 0.63,
      "learning_rate": 6.245362205760703e-07,
      "logits/chosen": -2.912815570831299,
      "logits/rejected": -2.941206216812134,
      "loss": 0.4346,
      "policy_logps/chosen": -149.0521240234375,
      "policy_logps/rejected": -140.37326049804688,
      "referece_logps/chosen": -146.72096252441406,
      "referece_logps/rejected": -131.8609161376953,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.23311704397201538,
      "rewards/margins": 0.6181185245513916,
      "rewards/rejected": -0.8512355089187622,
      "step": 99
    },
    {
      "epoch": 0.64,
      "learning_rate": 6.053357937665236e-07,
      "logits/chosen": -2.9297289848327637,
      "logits/rejected": -2.9020700454711914,
      "loss": 0.4962,
      "policy_logps/chosen": -130.0217742919922,
      "policy_logps/rejected": -139.19395446777344,
      "referece_logps/chosen": -128.82655334472656,
      "referece_logps/rejected": -132.361572265625,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.11952205747365952,
      "rewards/margins": 0.563715934753418,
      "rewards/rejected": -0.6832380294799805,
      "step": 100
    }
  ],
  "max_steps": 156,
  "num_train_epochs": 1,
  "total_flos": 0.0,
  "trial_name": null,
  "trial_params": null
}