{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 0.9984301412872841,
  "eval_steps": 100,
  "global_step": 477,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.02,
      "learning_rate": 6.25e-07,
      "logits/chosen": 0.2953376770019531,
      "logits/rejected": 0.3199407160282135,
      "logps/chosen": -295.2257080078125,
      "logps/rejected": -290.8670959472656,
      "loss": 0.1859,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": 0.00015557199367322028,
      "rewards/margins": 0.0003326966252643615,
      "rewards/rejected": -0.00017712473345454782,
      "step": 10
    },
    {
      "epoch": 0.04,
      "learning_rate": 1.25e-06,
      "logits/chosen": 0.26482802629470825,
      "logits/rejected": 0.33679550886154175,
      "logps/chosen": -318.49652099609375,
      "logps/rejected": -291.0345458984375,
      "loss": 0.1869,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.002573266625404358,
      "rewards/margins": 0.011565858498215675,
      "rewards/rejected": -0.014139125123620033,
      "step": 20
    },
    {
      "epoch": 0.06,
      "learning_rate": 1.875e-06,
      "logits/chosen": 0.3993307650089264,
      "logits/rejected": 0.45834311842918396,
      "logps/chosen": -301.6546936035156,
      "logps/rejected": -295.427734375,
      "loss": 0.1703,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.05365907400846481,
      "rewards/margins": 0.07345211505889893,
      "rewards/rejected": -0.12711118161678314,
      "step": 30
    },
    {
      "epoch": 0.08,
      "learning_rate": 2.5e-06,
      "logits/chosen": 0.35423707962036133,
      "logits/rejected": 0.4464724063873291,
      "logps/chosen": -333.11279296875,
      "logps/rejected": -334.0306091308594,
      "loss": 0.1389,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.2748720049858093,
      "rewards/margins": 0.14294137060642242,
      "rewards/rejected": -0.41781339049339294,
      "step": 40
    },
    {
      "epoch": 0.1,
      "learning_rate": 2.999839121261416e-06,
      "logits/chosen": 0.35561248660087585,
      "logits/rejected": 0.3796083331108093,
      "logps/chosen": -374.3013916015625,
      "logps/rejected": -392.155517578125,
      "loss": 0.0917,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.6061999201774597,
      "rewards/margins": 0.2705405652523041,
      "rewards/rejected": -0.8767404556274414,
      "step": 50
    },
    {
      "epoch": 0.13,
      "learning_rate": 2.994211988057582e-06,
      "logits/chosen": 0.17622683942317963,
      "logits/rejected": 0.313021719455719,
      "logps/chosen": -346.9971618652344,
      "logps/rejected": -349.66387939453125,
      "loss": 0.0903,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.5485752820968628,
      "rewards/margins": 0.28624793887138367,
      "rewards/rejected": -0.8348232507705688,
      "step": 60
    },
    {
      "epoch": 0.15,
      "learning_rate": 2.9805753939568693e-06,
      "logits/chosen": 0.1983145773410797,
      "logits/rejected": 0.23167672753334045,
      "logps/chosen": -356.7235412597656,
      "logps/rejected": -386.5905456542969,
      "loss": 0.0815,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.5771406888961792,
      "rewards/margins": 0.28289300203323364,
      "rewards/rejected": -0.8600338101387024,
      "step": 70
    },
    {
      "epoch": 0.17,
      "learning_rate": 2.959002435526626e-06,
      "logits/chosen": 0.35488566756248474,
      "logits/rejected": 0.4059143662452698,
      "logps/chosen": -371.07623291015625,
      "logps/rejected": -389.4164123535156,
      "loss": 0.0655,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.755893349647522,
      "rewards/margins": 0.5531964302062988,
      "rewards/rejected": -1.3090897798538208,
      "step": 80
    },
    {
      "epoch": 0.19,
      "learning_rate": 2.929608750821129e-06,
      "logits/chosen": 0.2614609897136688,
      "logits/rejected": 0.4048990309238434,
      "logps/chosen": -387.1451110839844,
      "logps/rejected": -373.1450500488281,
      "loss": 0.0699,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.7152177095413208,
      "rewards/margins": 0.39531436562538147,
      "rewards/rejected": -1.1105320453643799,
      "step": 90
    },
    {
      "epoch": 0.21,
      "learning_rate": 2.892551899524109e-06,
      "logits/chosen": 0.35849034786224365,
      "logits/rejected": 0.3978693187236786,
      "logps/chosen": -395.839599609375,
      "logps/rejected": -456.36016845703125,
      "loss": 0.0642,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -1.0831067562103271,
      "rewards/margins": 0.505523681640625,
      "rewards/rejected": -1.5886304378509521,
      "step": 100
    },
    {
      "epoch": 0.21,
      "eval_logits/chosen": 0.4300641715526581,
      "eval_logits/rejected": 0.45788079500198364,
      "eval_logps/chosen": -443.42120361328125,
      "eval_logps/rejected": -556.7063598632812,
      "eval_loss": 0.04867542162537575,
      "eval_rewards/accuracies": 0.71875,
      "eval_rewards/chosen": -1.3983652591705322,
      "eval_rewards/margins": 1.0524473190307617,
      "eval_rewards/rejected": -2.450812578201294,
      "eval_runtime": 73.3058,
      "eval_samples_per_second": 27.283,
      "eval_steps_per_second": 0.437,
      "step": 100
    },
    {
      "epoch": 0.23,
      "learning_rate": 2.848030518377739e-06,
      "logits/chosen": 0.2927325367927551,
      "logits/rejected": 0.3830544054508209,
      "logps/chosen": -459.0276794433594,
      "logps/rejected": -510.6993103027344,
      "loss": 0.0405,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.4556727409362793,
      "rewards/margins": 0.8043819665908813,
      "rewards/rejected": -2.260054588317871,
      "step": 110
    },
    {
      "epoch": 0.25,
      "learning_rate": 2.7962832564252724e-06,
      "logits/chosen": 0.15991483628749847,
      "logits/rejected": 0.2417908012866974,
      "logps/chosen": -477.8702087402344,
      "logps/rejected": -537.2095947265625,
      "loss": 0.0489,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -1.243250846862793,
      "rewards/margins": 0.6618646383285522,
      "rewards/rejected": -1.9051154851913452,
      "step": 120
    },
    {
      "epoch": 0.27,
      "learning_rate": 2.7375874957747644e-06,
      "logits/chosen": 0.06199001148343086,
      "logits/rejected": 0.19189774990081787,
      "logps/chosen": -411.61602783203125,
      "logps/rejected": -418.01019287109375,
      "loss": 0.0665,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.8180139660835266,
      "rewards/margins": 0.39107373356819153,
      "rewards/rejected": -1.209087610244751,
      "step": 130
    },
    {
      "epoch": 0.29,
      "learning_rate": 2.672257864741005e-06,
      "logits/chosen": 0.16753675043582916,
      "logits/rejected": 0.2094758003950119,
      "logps/chosen": -402.02484130859375,
      "logps/rejected": -429.792724609375,
      "loss": 0.0608,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.8829413652420044,
      "rewards/margins": 0.4748268127441406,
      "rewards/rejected": -1.3577684164047241,
      "step": 140
    },
    {
      "epoch": 0.31,
      "learning_rate": 2.600644551335706e-06,
      "logits/chosen": 0.12230317294597626,
      "logits/rejected": 0.2780611515045166,
      "logps/chosen": -399.07012939453125,
      "logps/rejected": -447.2755432128906,
      "loss": 0.0409,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -1.1799192428588867,
      "rewards/margins": 0.7245117425918579,
      "rewards/rejected": -1.9044307470321655,
      "step": 150
    },
    {
      "epoch": 0.33,
      "learning_rate": 2.5231314261461732e-06,
      "logits/chosen": 0.15223169326782227,
      "logits/rejected": 0.2353695183992386,
      "logps/chosen": -423.8455505371094,
      "logps/rejected": -493.4649353027344,
      "loss": 0.0439,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -1.3035107851028442,
      "rewards/margins": 0.7260502576828003,
      "rewards/rejected": -2.0295610427856445,
      "step": 160
    },
    {
      "epoch": 0.36,
      "learning_rate": 2.440133984664454e-06,
      "logits/chosen": 0.07280103117227554,
      "logits/rejected": 0.15081152319908142,
      "logps/chosen": -472.6494140625,
      "logps/rejected": -495.68768310546875,
      "loss": 0.0448,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -1.1380369663238525,
      "rewards/margins": 0.558665931224823,
      "rewards/rejected": -1.6967029571533203,
      "step": 170
    },
    {
      "epoch": 0.38,
      "learning_rate": 2.3520971200967337e-06,
      "logits/chosen": 0.1431509107351303,
      "logits/rejected": 0.12718148529529572,
      "logps/chosen": -421.31591796875,
      "logps/rejected": -481.0022888183594,
      "loss": 0.0398,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -1.277320146560669,
      "rewards/margins": 0.7172085642814636,
      "rewards/rejected": -1.9945287704467773,
      "step": 180
    },
    {
      "epoch": 0.4,
      "learning_rate": 2.2594927385914546e-06,
      "logits/chosen": 0.1423339694738388,
      "logits/rejected": 0.20066264271736145,
      "logps/chosen": -385.1085205078125,
      "logps/rejected": -413.58660888671875,
      "loss": 0.044,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -1.1722856760025024,
      "rewards/margins": 0.5436893105506897,
      "rewards/rejected": -1.715975046157837,
      "step": 190
    },
    {
      "epoch": 0.42,
      "learning_rate": 2.1628172296692954e-06,
      "logits/chosen": 0.08901094645261765,
      "logits/rejected": 0.13271097838878632,
      "logps/chosen": -385.1289367675781,
      "logps/rejected": -454.6835021972656,
      "loss": 0.047,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -1.0956504344940186,
      "rewards/margins": 0.6720448732376099,
      "rewards/rejected": -1.767695426940918,
      "step": 200
    },
    {
      "epoch": 0.42,
      "eval_logits/chosen": 0.11227821558713913,
      "eval_logits/rejected": 0.12742303311824799,
      "eval_logps/chosen": -415.04937744140625,
      "eval_logps/rejected": -496.232666015625,
      "eval_loss": 0.0460943840444088,
      "eval_rewards/accuracies": 0.7421875,
      "eval_rewards/chosen": -1.1146469116210938,
      "eval_rewards/margins": 0.7314289212226868,
      "eval_rewards/rejected": -1.8460756540298462,
      "eval_runtime": 73.4078,
      "eval_samples_per_second": 27.245,
      "eval_steps_per_second": 0.436,
      "step": 200
    },
    {
      "epoch": 0.44,
      "learning_rate": 2.062588805414343e-06,
      "logits/chosen": -0.006222477648407221,
      "logits/rejected": 0.02312941662967205,
      "logps/chosen": -435.7500915527344,
      "logps/rejected": -491.56561279296875,
      "loss": 0.0379,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -1.1337292194366455,
      "rewards/margins": 0.7197057008743286,
      "rewards/rejected": -1.8534349203109741,
      "step": 210
    },
    {
      "epoch": 0.46,
      "learning_rate": 1.9593447226892386e-06,
      "logits/chosen": -0.17459270358085632,
      "logits/rejected": -0.05563253164291382,
      "logps/chosen": -466.9278259277344,
      "logps/rejected": -490.50567626953125,
      "loss": 0.033,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -1.4599158763885498,
      "rewards/margins": 0.615450382232666,
      "rewards/rejected": -2.075366497039795,
      "step": 220
    },
    {
      "epoch": 0.48,
      "learning_rate": 1.853638403264141e-06,
      "logits/chosen": -0.037995658814907074,
      "logits/rejected": -0.011381956748664379,
      "logps/chosen": -448.68939208984375,
      "logps/rejected": -532.2055053710938,
      "loss": 0.0365,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.6667238473892212,
      "rewards/margins": 0.8385307192802429,
      "rewards/rejected": -2.5052547454833984,
      "step": 230
    },
    {
      "epoch": 0.5,
      "learning_rate": 1.7460364672965328e-06,
      "logits/chosen": -0.0721597820520401,
      "logits/rejected": -0.08736585080623627,
      "logps/chosen": -470.03045654296875,
      "logps/rejected": -542.5087280273438,
      "loss": 0.0484,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -1.167931318283081,
      "rewards/margins": 0.7865667939186096,
      "rewards/rejected": -1.954498052597046,
      "step": 240
    },
    {
      "epoch": 0.52,
      "learning_rate": 1.637115696063402e-06,
      "logits/chosen": -0.07914544641971588,
      "logits/rejected": -0.05659206956624985,
      "logps/chosen": -430.75726318359375,
      "logps/rejected": -472.98284912109375,
      "loss": 0.0475,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -1.1496978998184204,
      "rewards/margins": 0.6665709018707275,
      "rewards/rejected": -1.8162685632705688,
      "step": 250
    },
    {
      "epoch": 0.54,
      "learning_rate": 1.5274599402265162e-06,
      "logits/chosen": -0.23472614586353302,
      "logits/rejected": -0.11506778001785278,
      "logps/chosen": -493.5909118652344,
      "logps/rejected": -528.9447021484375,
      "loss": 0.0421,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.5179877281188965,
      "rewards/margins": 0.8330425024032593,
      "rewards/rejected": -2.3510303497314453,
      "step": 260
    },
    {
      "epoch": 0.57,
      "learning_rate": 1.4176569902035088e-06,
      "logits/chosen": -0.17047128081321716,
      "logits/rejected": -0.09592470526695251,
      "logps/chosen": -456.50042724609375,
      "logps/rejected": -494.05059814453125,
      "loss": 0.0513,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -1.2792171239852905,
      "rewards/margins": 0.6969503164291382,
      "rewards/rejected": -1.9761674404144287,
      "step": 270
    },
    {
      "epoch": 0.59,
      "learning_rate": 1.308295425420593e-06,
      "logits/chosen": -0.24125584959983826,
      "logits/rejected": -0.21810472011566162,
      "logps/chosen": -445.63916015625,
      "logps/rejected": -531.8533325195312,
      "loss": 0.0463,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -1.3838244676589966,
      "rewards/margins": 0.7514439225196838,
      "rewards/rejected": -2.135268211364746,
      "step": 280
    },
    {
      "epoch": 0.61,
      "learning_rate": 1.1999614593359337e-06,
      "logits/chosen": -0.2246803492307663,
      "logits/rejected": -0.19805817306041718,
      "logps/chosen": -426.7688903808594,
      "logps/rejected": -472.63134765625,
      "loss": 0.0407,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -1.2982797622680664,
      "rewards/margins": 0.6589063405990601,
      "rewards/rejected": -1.9571861028671265,
      "step": 290
    },
    {
      "epoch": 0.63,
      "learning_rate": 1.0932357971453745e-06,
      "logits/chosen": -0.28935036063194275,
      "logits/rejected": -0.16529107093811035,
      "logps/chosen": -460.61138916015625,
      "logps/rejected": -460.7236328125,
      "loss": 0.0401,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -1.4895880222320557,
      "rewards/margins": 0.5250975489616394,
      "rewards/rejected": -2.0146853923797607,
      "step": 300
    },
    {
      "epoch": 0.63,
      "eval_logits/chosen": -0.1928369402885437,
      "eval_logits/rejected": -0.17653387784957886,
      "eval_logps/chosen": -431.743896484375,
      "eval_logps/rejected": -528.1251831054688,
      "eval_loss": 0.04077731445431709,
      "eval_rewards/accuracies": 0.71484375,
      "eval_rewards/chosen": -1.2815920114517212,
      "eval_rewards/margins": 0.8834086656570435,
      "eval_rewards/rejected": -2.1650009155273438,
      "eval_runtime": 75.4051,
      "eval_samples_per_second": 26.523,
      "eval_steps_per_second": 0.424,
      "step": 300
    },
    {
      "epoch": 0.65,
      "learning_rate": 9.886905230142433e-07,
      "logits/chosen": -0.19448669254779816,
      "logits/rejected": -0.2653646469116211,
      "logps/chosen": -470.20654296875,
      "logps/rejected": -532.0836181640625,
      "loss": 0.0436,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -1.4866724014282227,
      "rewards/margins": 0.6113816499710083,
      "rewards/rejected": -2.0980541706085205,
      "step": 310
    },
    {
      "epoch": 0.67,
      "learning_rate": 8.868860335206678e-07,
      "logits/chosen": -0.274607390165329,
      "logits/rejected": -0.18271437287330627,
      "logps/chosen": -476.87591552734375,
      "logps/rejected": -476.43304443359375,
      "loss": 0.0457,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.2201156616210938,
      "rewards/margins": 0.4418094754219055,
      "rewards/rejected": -1.6619250774383545,
      "step": 320
    },
    {
      "epoch": 0.69,
      "learning_rate": 7.883680337481599e-07,
      "logits/chosen": -0.17901699244976044,
      "logits/rejected": -0.19661930203437805,
      "logps/chosen": -414.8687438964844,
      "logps/rejected": -532.6778564453125,
      "loss": 0.0437,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.1104294061660767,
      "rewards/margins": 0.8232589960098267,
      "rewards/rejected": -1.9336884021759033,
      "step": 330
    },
    {
      "epoch": 0.71,
      "learning_rate": 6.936646121293654e-07,
      "logits/chosen": -0.21059110760688782,
      "logits/rejected": -0.22660131752490997,
      "logps/chosen": -434.880859375,
      "logps/rejected": -541.180908203125,
      "loss": 0.0366,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -1.4544843435287476,
      "rewards/margins": 0.8150385618209839,
      "rewards/rejected": -2.2695229053497314,
      "step": 340
    },
    {
      "epoch": 0.73,
      "learning_rate": 6.032834097207889e-07,
      "logits/chosen": -0.17869403958320618,
      "logits/rejected": -0.1880768984556198,
      "logps/chosen": -464.3429260253906,
      "logps/rejected": -565.3632202148438,
      "loss": 0.0355,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -1.5690315961837769,
      "rewards/margins": 0.8189191818237305,
      "rewards/rejected": -2.387950897216797,
      "step": 350
    },
    {
      "epoch": 0.75,
      "learning_rate": 5.177088990820725e-07,
      "logits/chosen": -0.2201962023973465,
      "logits/rejected": -0.17389468848705292,
      "logps/chosen": -494.4137268066406,
      "logps/rejected": -554.2338256835938,
      "loss": 0.0348,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -1.467558741569519,
      "rewards/margins": 0.9708374738693237,
      "rewards/rejected": -2.4383959770202637,
      "step": 360
    },
    {
      "epoch": 0.77,
      "learning_rate": 4.3739978734594494e-07,
      "logits/chosen": -0.2049982100725174,
      "logits/rejected": -0.13553449511528015,
      "logps/chosen": -510.56610107421875,
      "logps/rejected": -544.7944946289062,
      "loss": 0.0343,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -1.5760037899017334,
      "rewards/margins": 0.8068605661392212,
      "rewards/rejected": -2.382864236831665,
      "step": 370
    },
    {
      "epoch": 0.8,
      "learning_rate": 3.627865573992087e-07,
      "logits/chosen": -0.23506097495555878,
      "logits/rejected": -0.1945139467716217,
      "logps/chosen": -473.996337890625,
      "logps/rejected": -546.2703857421875,
      "loss": 0.0411,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -1.4194319248199463,
      "rewards/margins": 0.9198926091194153,
      "rewards/rejected": -2.339324474334717,
      "step": 380
    },
    {
      "epoch": 0.82,
      "learning_rate": 2.9426916035484166e-07,
      "logits/chosen": -0.18075546622276306,
      "logits/rejected": -0.12812264263629913,
      "logps/chosen": -451.8818359375,
      "logps/rejected": -511.1201171875,
      "loss": 0.0428,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -1.2848039865493774,
      "rewards/margins": 0.7686089277267456,
      "rewards/rejected": -2.053412675857544,
      "step": 390
    },
    {
      "epoch": 0.84,
      "learning_rate": 2.322148716843081e-07,
      "logits/chosen": -0.15861022472381592,
      "logits/rejected": -0.1434582620859146,
      "logps/chosen": -434.2018127441406,
      "logps/rejected": -514.0694580078125,
      "loss": 0.0435,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -1.3711330890655518,
      "rewards/margins": 0.6456871032714844,
      "rewards/rejected": -2.016820192337036,
      "step": 400
    },
    {
      "epoch": 0.84,
      "eval_logits/chosen": -0.08009739220142365,
      "eval_logits/rejected": -0.07131167501211166,
      "eval_logps/chosen": -420.7032470703125,
      "eval_logps/rejected": -517.1834106445312,
      "eval_loss": 0.04270826652646065,
      "eval_rewards/accuracies": 0.7265625,
      "eval_rewards/chosen": -1.1711856126785278,
      "eval_rewards/margins": 0.8843976259231567,
      "eval_rewards/rejected": -2.0555832386016846,
      "eval_runtime": 75.7653,
      "eval_samples_per_second": 26.397,
      "eval_steps_per_second": 0.422,
      "step": 400
    },
    {
      "epoch": 0.86,
      "learning_rate": 1.7695632250191002e-07,
      "logits/chosen": -0.16302308440208435,
      "logits/rejected": -0.1545868217945099,
      "logps/chosen": -431.54083251953125,
      "logps/rejected": -492.18975830078125,
      "loss": 0.0419,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.1856681108474731,
      "rewards/margins": 0.7893311381340027,
      "rewards/rejected": -1.9749990701675415,
      "step": 410
    },
    {
      "epoch": 0.88,
      "learning_rate": 1.2878971655412515e-07,
      "logits/chosen": -0.15071699023246765,
      "logits/rejected": -0.1457306295633316,
      "logps/chosen": -424.42547607421875,
      "logps/rejected": -474.6815490722656,
      "loss": 0.0388,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -1.1596481800079346,
      "rewards/margins": 0.791610836982727,
      "rewards/rejected": -1.9512590169906616,
      "step": 420
    },
    {
      "epoch": 0.9,
      "learning_rate": 8.797324247145411e-08,
      "logits/chosen": -0.16887474060058594,
      "logits/rejected": -0.11855659633874893,
      "logps/chosen": -482.5933532714844,
      "logps/rejected": -524.8614501953125,
      "loss": 0.0433,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -1.3214952945709229,
      "rewards/margins": 0.7315466403961182,
      "rewards/rejected": -2.053041934967041,
      "step": 430
    },
    {
      "epoch": 0.92,
      "learning_rate": 5.472568979361853e-08,
      "logits/chosen": -0.1303589642047882,
      "logits/rejected": -0.1282893717288971,
      "logps/chosen": -428.30877685546875,
      "logps/rejected": -485.59185791015625,
      "loss": 0.0406,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -1.3397411108016968,
      "rewards/margins": 0.8013665080070496,
      "rewards/rejected": -2.1411080360412598,
      "step": 440
    },
    {
      "epoch": 0.94,
      "learning_rate": 2.922527618666465e-08,
      "logits/chosen": -0.20207130908966064,
      "logits/rejected": -0.07939404994249344,
      "logps/chosen": -461.0880432128906,
      "logps/rejected": -495.314453125,
      "loss": 0.0414,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -1.184661626815796,
      "rewards/margins": 0.7377935647964478,
      "rewards/rejected": -1.922455072402954,
      "step": 450
    },
    {
      "epoch": 0.96,
      "learning_rate": 1.1608692138469379e-08,
      "logits/chosen": -0.18077249825000763,
      "logits/rejected": -0.11532274633646011,
      "logps/chosen": -430.862060546875,
      "logps/rejected": -464.39385986328125,
      "loss": 0.0406,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -1.2867480516433716,
      "rewards/margins": 0.6147353053092957,
      "rewards/rejected": -1.9014835357666016,
      "step": 460
    },
    {
      "epoch": 0.98,
      "learning_rate": 1.970368253390198e-09,
      "logits/chosen": -0.21146085858345032,
      "logits/rejected": -0.20344269275665283,
      "logps/chosen": -457.4112243652344,
      "logps/rejected": -528.634521484375,
      "loss": 0.039,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -1.1956093311309814,
      "rewards/margins": 0.7480987906455994,
      "rewards/rejected": -1.943708062171936,
      "step": 470
    },
    {
      "epoch": 1.0,
      "step": 477,
      "total_flos": 0.0,
      "train_loss": 0.05794826206171288,
      "train_runtime": 4397.3222,
      "train_samples_per_second": 13.903,
      "train_steps_per_second": 0.108
    }
  ],
  "logging_steps": 10,
  "max_steps": 477,
  "num_train_epochs": 1,
  "save_steps": 1000,
  "total_flos": 0.0,
  "trial_name": null,
  "trial_params": null
}