{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 2.0,
  "eval_steps": 100,
  "global_step": 658,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.0,
      "learning_rate": 7.575757575757576e-08,
      "logits/chosen": -3.0401854515075684,
      "logits/rejected": -2.9935803413391113,
      "logps/chosen": -33.6235466003418,
      "logps/rejected": -31.46235466003418,
      "loss": 0.0011,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 1
    },
    {
      "epoch": 0.03,
      "learning_rate": 7.575757575757576e-07,
      "logits/chosen": -3.0845224857330322,
      "logits/rejected": -2.939974784851074,
      "logps/chosen": -187.66502380371094,
      "logps/rejected": -172.21742248535156,
      "loss": 0.0011,
      "rewards/accuracies": 0.4444444477558136,
      "rewards/chosen": 4.157695730100386e-05,
      "rewards/margins": -0.0002048778405878693,
      "rewards/rejected": 0.0002464548160787672,
      "step": 10
    },
    {
      "epoch": 0.06,
      "learning_rate": 1.5151515151515152e-06,
      "logits/chosen": -2.975489616394043,
      "logits/rejected": -2.8098347187042236,
      "logps/chosen": -101.97224426269531,
      "logps/rejected": -79.41976928710938,
      "loss": 0.001,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.0006187028484418988,
      "rewards/margins": 0.0002881823165807873,
      "rewards/rejected": 0.0003305206191726029,
      "step": 20
    },
    {
      "epoch": 0.09,
      "learning_rate": 2.2727272727272728e-06,
      "logits/chosen": -2.9616599082946777,
      "logits/rejected": -2.853991746902466,
      "logps/chosen": -93.24244689941406,
      "logps/rejected": -93.37409973144531,
      "loss": 0.0011,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.0018816631054505706,
      "rewards/margins": 0.000262443587416783,
      "rewards/rejected": 0.001619219547137618,
      "step": 30
    },
    {
      "epoch": 0.12,
      "learning_rate": 3.0303030303030305e-06,
      "logits/chosen": -2.957383871078491,
      "logits/rejected": -2.8873894214630127,
      "logps/chosen": -98.40808868408203,
      "logps/rejected": -90.6903305053711,
      "loss": 0.0011,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.0038822232745587826,
      "rewards/margins": 0.0006505203782580793,
      "rewards/rejected": 0.0032317028380930424,
      "step": 40
    },
    {
      "epoch": 0.15,
      "learning_rate": 3.7878787878787882e-06,
      "logits/chosen": -2.793536424636841,
      "logits/rejected": -2.701014995574951,
      "logps/chosen": -76.89764404296875,
      "logps/rejected": -76.88601684570312,
      "loss": 0.0009,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": 0.0003440352447796613,
      "rewards/margins": 0.003660330083221197,
      "rewards/rejected": -0.003316295798867941,
      "step": 50
    },
    {
      "epoch": 0.18,
      "learning_rate": 4.5454545454545455e-06,
      "logits/chosen": -2.8517611026763916,
      "logits/rejected": -2.701003313064575,
      "logps/chosen": -111.06632232666016,
      "logps/rejected": -97.74620056152344,
      "loss": 0.001,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": 0.0011594895040616393,
      "rewards/margins": 0.00348883168771863,
      "rewards/rejected": -0.0023293420672416687,
      "step": 60
    },
    {
      "epoch": 0.21,
      "learning_rate": 4.999436790436924e-06,
      "logits/chosen": -3.074284791946411,
      "logits/rejected": -2.859107494354248,
      "logps/chosen": -175.5392608642578,
      "logps/rejected": -165.30654907226562,
      "loss": 0.0011,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.0042682006023824215,
      "rewards/margins": 0.004156609531491995,
      "rewards/rejected": 0.00011159134737681597,
      "step": 70
    },
    {
      "epoch": 0.24,
      "learning_rate": 4.993103596812269e-06,
      "logits/chosen": -2.7597286701202393,
      "logits/rejected": -2.6338610649108887,
      "logps/chosen": -117.37264251708984,
      "logps/rejected": -96.65496826171875,
      "loss": 0.0014,
      "rewards/accuracies": 0.30000001192092896,
      "rewards/chosen": -0.004695770796388388,
      "rewards/margins": -0.0023340280167758465,
      "rewards/rejected": -0.002361743012443185,
      "step": 80
    },
    {
      "epoch": 0.27,
      "learning_rate": 4.979751088147192e-06,
      "logits/chosen": -2.7671074867248535,
      "logits/rejected": -2.5734329223632812,
      "logps/chosen": -124.0124740600586,
      "logps/rejected": -117.49992370605469,
      "loss": 0.0013,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.0014447696739807725,
      "rewards/margins": 0.006112386472523212,
      "rewards/rejected": -0.0075571550987660885,
      "step": 90
    },
    {
      "epoch": 0.3,
      "learning_rate": 4.95941685833271e-06,
      "logits/chosen": -2.841676712036133,
      "logits/rejected": -2.6840147972106934,
      "logps/chosen": -136.9166259765625,
      "logps/rejected": -142.25997924804688,
      "loss": 0.0012,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.005834830459207296,
      "rewards/margins": 0.0021175735164433718,
      "rewards/rejected": -0.007952402345836163,
      "step": 100
    },
    {
      "epoch": 0.3,
      "eval_logits/chosen": -2.3582723140716553,
      "eval_logits/rejected": -2.1644442081451416,
      "eval_logps/chosen": -281.1500244140625,
      "eval_logps/rejected": -259.7554626464844,
      "eval_loss": 0.001627120072953403,
      "eval_rewards/accuracies": 0.5034999847412109,
      "eval_rewards/chosen": -0.016425278037786484,
      "eval_rewards/margins": -0.00047451091813854873,
      "eval_rewards/rejected": -0.0159507654607296,
      "eval_runtime": 1420.6531,
      "eval_samples_per_second": 1.408,
      "eval_steps_per_second": 0.704,
      "step": 100
    },
    {
      "epoch": 0.33,
      "learning_rate": 4.9321581582449365e-06,
      "logits/chosen": -2.8993427753448486,
      "logits/rejected": -2.786620616912842,
      "logps/chosen": -118.65157318115234,
      "logps/rejected": -127.34912109375,
      "loss": 0.0011,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.0042579686269164085,
      "rewards/margins": 0.004396714735776186,
      "rewards/rejected": -0.008654682897031307,
      "step": 110
    },
    {
      "epoch": 0.36,
      "learning_rate": 4.898051734555676e-06,
      "logits/chosen": -3.0458617210388184,
      "logits/rejected": -2.99739670753479,
      "logps/chosen": -95.90953063964844,
      "logps/rejected": -102.84895324707031,
      "loss": 0.0013,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.004254769999533892,
      "rewards/margins": -0.0005313962465152144,
      "rewards/rejected": -0.0037233736366033554,
      "step": 120
    },
    {
      "epoch": 0.4,
      "learning_rate": 4.857193613652711e-06,
      "logits/chosen": -3.0125811100006104,
      "logits/rejected": -2.8767762184143066,
      "logps/chosen": -66.88941955566406,
      "logps/rejected": -71.67733001708984,
      "loss": 0.0012,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.004949026275426149,
      "rewards/margins": 0.0032058332581073046,
      "rewards/rejected": 0.001743193482980132,
      "step": 130
    },
    {
      "epoch": 0.43,
      "learning_rate": 4.809698831278217e-06,
      "logits/chosen": -2.9552953243255615,
      "logits/rejected": -2.7150585651397705,
      "logps/chosen": -75.46265411376953,
      "logps/rejected": -72.64187622070312,
      "loss": 0.001,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": 0.00027392804622650146,
      "rewards/margins": 0.016122380271553993,
      "rewards/rejected": -0.01584845408797264,
      "step": 140
    },
    {
      "epoch": 0.46,
      "learning_rate": 4.7557011086464634e-06,
      "logits/chosen": -2.9132320880889893,
      "logits/rejected": -2.64630126953125,
      "logps/chosen": -76.73291778564453,
      "logps/rejected": -97.9271011352539,
      "loss": 0.0013,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.00335231376811862,
      "rewards/margins": 0.01173357479274273,
      "rewards/rejected": -0.015085889026522636,
      "step": 150
    },
    {
      "epoch": 0.49,
      "learning_rate": 4.695352475952706e-06,
      "logits/chosen": -3.013190746307373,
      "logits/rejected": -2.926192045211792,
      "logps/chosen": -73.78190612792969,
      "logps/rejected": -84.00215148925781,
      "loss": 0.0016,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.006495082285255194,
      "rewards/margins": 0.01742926612496376,
      "rewards/rejected": -0.023924347013235092,
      "step": 160
    },
    {
      "epoch": 0.52,
      "learning_rate": 4.6288228443332786e-06,
      "logits/chosen": -2.892408847808838,
      "logits/rejected": -2.785109758377075,
      "logps/chosen": -88.50574493408203,
      "logps/rejected": -63.979164123535156,
      "loss": 0.0014,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.015971587970852852,
      "rewards/margins": 0.0037319730035960674,
      "rewards/rejected": -0.019703561440110207,
      "step": 170
    },
    {
      "epoch": 0.55,
      "learning_rate": 4.556299527482029e-06,
      "logits/chosen": -3.024970293045044,
      "logits/rejected": -2.9614624977111816,
      "logps/chosen": -95.4378433227539,
      "logps/rejected": -86.34746551513672,
      "loss": 0.0009,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.0011760194320231676,
      "rewards/margins": 0.010892638936638832,
      "rewards/rejected": -0.009716618806123734,
      "step": 180
    },
    {
      "epoch": 0.58,
      "learning_rate": 4.4779867142699715e-06,
      "logits/chosen": -3.0479438304901123,
      "logits/rejected": -2.903231620788574,
      "logps/chosen": -138.85531616210938,
      "logps/rejected": -146.4200439453125,
      "loss": 0.0017,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": 0.013675776310265064,
      "rewards/margins": 0.003341741394251585,
      "rewards/rejected": 0.010334033519029617,
      "step": 190
    },
    {
      "epoch": 0.61,
      "learning_rate": 4.394104893853007e-06,
      "logits/chosen": -2.9416115283966064,
      "logits/rejected": -2.8143601417541504,
      "logps/chosen": -85.66619110107422,
      "logps/rejected": -84.1985092163086,
      "loss": 0.0011,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.003594033420085907,
      "rewards/margins": 0.005169789306819439,
      "rewards/rejected": -0.0015757561195641756,
      "step": 200
    },
    {
      "epoch": 0.61,
      "eval_logits/chosen": -2.378091812133789,
      "eval_logits/rejected": -2.183748960494995,
      "eval_logps/chosen": -280.3858337402344,
      "eval_logps/rejected": -258.9316711425781,
      "eval_loss": 0.0018177788006141782,
      "eval_rewards/accuracies": 0.4814999997615814,
      "eval_rewards/chosen": -0.008783474564552307,
      "eval_rewards/margins": -0.0010705279419198632,
      "eval_rewards/rejected": -0.007712946273386478,
      "eval_runtime": 1420.2588,
      "eval_samples_per_second": 1.408,
      "eval_steps_per_second": 0.704,
      "step": 200
    },
    {
      "epoch": 0.64,
      "learning_rate": 4.3048902348863116e-06,
      "logits/chosen": -2.9989707469940186,
      "logits/rejected": -2.8650078773498535,
      "logps/chosen": -102.37027740478516,
      "logps/rejected": -118.4636001586914,
      "loss": 0.0012,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.001182993990369141,
      "rewards/margins": 0.004763666074723005,
      "rewards/rejected": -0.003580672200769186,
      "step": 210
    },
    {
      "epoch": 0.67,
      "learning_rate": 4.210593920593201e-06,
      "logits/chosen": -2.9398374557495117,
      "logits/rejected": -2.8116965293884277,
      "logps/chosen": -70.69538879394531,
      "logps/rejected": -75.53007507324219,
      "loss": 0.0013,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.005323179066181183,
      "rewards/margins": 0.004396263975650072,
      "rewards/rejected": 0.0009269150905311108,
      "step": 220
    },
    {
      "epoch": 0.7,
      "learning_rate": 4.111481441560598e-06,
      "logits/chosen": -2.972592830657959,
      "logits/rejected": -2.852461099624634,
      "logps/chosen": -160.5238037109375,
      "logps/rejected": -129.62954711914062,
      "loss": 0.0013,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": 0.00035965832648798823,
      "rewards/margins": 0.005134374834597111,
      "rewards/rejected": -0.004774716217070818,
      "step": 230
    },
    {
      "epoch": 0.73,
      "learning_rate": 4.007831848252212e-06,
      "logits/chosen": -3.1746320724487305,
      "logits/rejected": -2.9887349605560303,
      "logps/chosen": -124.45001220703125,
      "logps/rejected": -101.69696807861328,
      "loss": 0.0009,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.019443057477474213,
      "rewards/margins": 0.012666161172091961,
      "rewards/rejected": 0.0067768944427371025,
      "step": 240
    },
    {
      "epoch": 0.76,
      "learning_rate": 3.899936965343989e-06,
      "logits/chosen": -3.0935516357421875,
      "logits/rejected": -3.028534412384033,
      "logps/chosen": -131.24337768554688,
      "logps/rejected": -116.88172912597656,
      "loss": 0.0016,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": 0.005977076943963766,
      "rewards/margins": 0.005966751836240292,
      "rewards/rejected": 1.0324455615773331e-05,
      "step": 250
    },
    {
      "epoch": 0.79,
      "learning_rate": 3.7881005700938635e-06,
      "logits/chosen": -2.7336764335632324,
      "logits/rejected": -2.7675654888153076,
      "logps/chosen": -95.82691955566406,
      "logps/rejected": -99.57430267333984,
      "loss": 0.0008,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 0.012143732979893684,
      "rewards/margins": 0.008911015465855598,
      "rewards/rejected": 0.003232717514038086,
      "step": 260
    },
    {
      "epoch": 0.82,
      "learning_rate": 3.6726375370590927e-06,
      "logits/chosen": -2.959117889404297,
      "logits/rejected": -2.841716766357422,
      "logps/chosen": -132.79884338378906,
      "logps/rejected": -122.66941833496094,
      "loss": 0.0015,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.003037696471437812,
      "rewards/margins": 0.00202806293964386,
      "rewards/rejected": 0.0010096338810399175,
      "step": 270
    },
    {
      "epoch": 0.85,
      "learning_rate": 3.553872951569236e-06,
      "logits/chosen": -3.0903332233428955,
      "logits/rejected": -3.009970188140869,
      "logps/chosen": -90.32659912109375,
      "logps/rejected": -92.47020721435547,
      "loss": 0.0012,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.005868277512490749,
      "rewards/margins": 0.011209800839424133,
      "rewards/rejected": -0.005341522395610809,
      "step": 280
    },
    {
      "epoch": 0.88,
      "learning_rate": 3.432141194450772e-06,
      "logits/chosen": -2.920701503753662,
      "logits/rejected": -2.721304416656494,
      "logps/chosen": -111.44612121582031,
      "logps/rejected": -111.91583251953125,
      "loss": 0.0007,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.004392694681882858,
      "rewards/margins": 0.016653168946504593,
      "rewards/rejected": -0.02104586362838745,
      "step": 290
    },
    {
      "epoch": 0.91,
      "learning_rate": 3.307785000580313e-06,
      "logits/chosen": -2.757887363433838,
      "logits/rejected": -2.5684075355529785,
      "logps/chosen": -128.5594024658203,
      "logps/rejected": -115.97843933105469,
      "loss": 0.0015,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": -0.006182619370520115,
      "rewards/margins": 0.0031226237770169973,
      "rewards/rejected": -0.009305243380367756,
      "step": 300
    },
    {
      "epoch": 0.91,
      "eval_logits/chosen": -2.374006748199463,
      "eval_logits/rejected": -2.1796298027038574,
      "eval_logps/chosen": -281.17401123046875,
      "eval_logps/rejected": -259.6520690917969,
      "eval_loss": 0.0019075373420491815,
      "eval_rewards/accuracies": 0.4805000126361847,
      "eval_rewards/chosen": -0.016665350645780563,
      "eval_rewards/margins": -0.0017483622068539262,
      "eval_rewards/rejected": -0.014916987158358097,
      "eval_runtime": 1421.116,
      "eval_samples_per_second": 1.407,
      "eval_steps_per_second": 0.704,
      "step": 300
    },
    {
      "epoch": 0.94,
      "learning_rate": 3.1811544939170573e-06,
      "logits/chosen": -2.7975335121154785,
      "logits/rejected": -2.667598009109497,
      "logps/chosen": -161.1660919189453,
      "logps/rejected": -133.93838500976562,
      "loss": 0.0012,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.010928490199148655,
      "rewards/margins": 0.00381992245092988,
      "rewards/rejected": -0.014748412184417248,
      "step": 310
    },
    {
      "epoch": 0.97,
      "learning_rate": 3.052606201731325e-06,
      "logits/chosen": -2.93320894241333,
      "logits/rejected": -2.7140159606933594,
      "logps/chosen": -79.52749633789062,
      "logps/rejected": -84.45579528808594,
      "loss": 0.0008,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.0026647502090781927,
      "rewards/margins": 0.009594769217073917,
      "rewards/rejected": -0.01225951872766018,
      "step": 320
    },
    {
      "epoch": 1.0,
      "learning_rate": 2.9225020508046233e-06,
      "logits/chosen": -2.901916742324829,
      "logits/rejected": -2.736692190170288,
      "logps/chosen": -115.4826431274414,
      "logps/rejected": -130.37356567382812,
      "loss": 0.0313,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.017747070640325546,
      "rewards/margins": 0.05266844481229782,
      "rewards/rejected": -0.034921370446681976,
      "step": 330
    },
    {
      "epoch": 1.03,
      "learning_rate": 2.7912083484274266e-06,
      "logits/chosen": -2.7787222862243652,
      "logits/rejected": -2.63339900970459,
      "logps/chosen": -137.5430450439453,
      "logps/rejected": -127.07392883300781,
      "loss": 0.0436,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": 0.0036686602979898453,
      "rewards/margins": 0.05118006467819214,
      "rewards/rejected": -0.04751140996813774,
      "step": 340
    },
    {
      "epoch": 1.06,
      "learning_rate": 2.659094751063666e-06,
      "logits/chosen": -2.9705729484558105,
      "logits/rejected": -2.832822322845459,
      "logps/chosen": -143.50698852539062,
      "logps/rejected": -140.02352905273438,
      "loss": 0.0133,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.008256057277321815,
      "rewards/margins": 0.05859958380460739,
      "rewards/rejected": -0.05034352466464043,
      "step": 350
    },
    {
      "epoch": 1.09,
      "learning_rate": 2.526533223585641e-06,
      "logits/chosen": -2.8163113594055176,
      "logits/rejected": -2.697537660598755,
      "logps/chosen": -85.76821899414062,
      "logps/rejected": -81.71627807617188,
      "loss": 0.0073,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.010457667522132397,
      "rewards/margins": 0.053769804537296295,
      "rewards/rejected": -0.043312136083841324,
      "step": 360
    },
    {
      "epoch": 1.12,
      "learning_rate": 2.39389699200963e-06,
      "logits/chosen": -3.000455141067505,
      "logits/rejected": -2.8073432445526123,
      "logps/chosen": -107.18000793457031,
      "logps/rejected": -118.2176284790039,
      "loss": 0.019,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": 0.008137053810060024,
      "rewards/margins": 0.07659684866666794,
      "rewards/rejected": -0.06845979392528534,
      "step": 370
    },
    {
      "epoch": 1.16,
      "learning_rate": 2.2615594926807554e-06,
      "logits/chosen": -2.9433727264404297,
      "logits/rejected": -2.817605495452881,
      "logps/chosen": -84.96995544433594,
      "logps/rejected": -95.87684631347656,
      "loss": 0.0277,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 0.01651051640510559,
      "rewards/margins": 0.06000148132443428,
      "rewards/rejected": -0.04349096864461899,
      "step": 380
    },
    {
      "epoch": 1.19,
      "learning_rate": 2.129893320865672e-06,
      "logits/chosen": -3.0510168075561523,
      "logits/rejected": -2.938931465148926,
      "logps/chosen": -76.3388442993164,
      "logps/rejected": -62.002540588378906,
      "loss": 0.0302,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 0.04483187943696976,
      "rewards/margins": 0.10971853882074356,
      "rewards/rejected": -0.0648866593837738,
      "step": 390
    },
    {
      "epoch": 1.22,
      "learning_rate": 1.9992691817133025e-06,
      "logits/chosen": -3.05735445022583,
      "logits/rejected": -2.815657138824463,
      "logps/chosen": -251.65219116210938,
      "logps/rejected": -255.02658081054688,
      "loss": 0.0397,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.03865816444158554,
      "rewards/margins": 0.04983866214752197,
      "rewards/rejected": -0.08849682658910751,
      "step": 400
    },
    {
      "epoch": 1.22,
      "eval_logits/chosen": -2.356828451156616,
      "eval_logits/rejected": -2.1631951332092285,
      "eval_logps/chosen": -287.29345703125,
      "eval_logps/rejected": -264.43231201171875,
      "eval_loss": 0.007440177723765373,
      "eval_rewards/accuracies": 0.41600000858306885,
      "eval_rewards/chosen": -0.0778598040342331,
      "eval_rewards/margins": -0.015140472911298275,
      "eval_rewards/rejected": -0.06271932274103165,
      "eval_runtime": 1421.1528,
      "eval_samples_per_second": 1.407,
      "eval_steps_per_second": 0.704,
      "step": 400
    },
    {
      "epoch": 1.25,
      "learning_rate": 1.8700548465371877e-06,
      "logits/chosen": -3.085784435272217,
      "logits/rejected": -2.8534493446350098,
      "logps/chosen": -150.0517120361328,
      "logps/rejected": -130.98390197753906,
      "loss": 0.0304,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.026314150542020798,
      "rewards/margins": 0.05200232192873955,
      "rewards/rejected": -0.07831647992134094,
      "step": 410
    },
    {
      "epoch": 1.28,
      "learning_rate": 1.742614117358029e-06,
      "logits/chosen": -2.8179147243499756,
      "logits/rejected": -2.72475528717041,
      "logps/chosen": -85.7872543334961,
      "logps/rejected": -98.0990219116211,
      "loss": 0.0839,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.052498847246170044,
      "rewards/margins": 0.009759088046848774,
      "rewards/rejected": -0.06225793436169624,
      "step": 420
    },
    {
      "epoch": 1.31,
      "learning_rate": 1.617305802621748e-06,
      "logits/chosen": -2.978959083557129,
      "logits/rejected": -2.9247641563415527,
      "logps/chosen": -94.34339904785156,
      "logps/rejected": -103.36552429199219,
      "loss": 0.0221,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": 0.004420995246618986,
      "rewards/margins": 0.09026098251342773,
      "rewards/rejected": -0.08583998680114746,
      "step": 430
    },
    {
      "epoch": 1.34,
      "learning_rate": 1.4944827069769125e-06,
      "logits/chosen": -2.94343900680542,
      "logits/rejected": -2.8201680183410645,
      "logps/chosen": -164.69044494628906,
      "logps/rejected": -141.93528747558594,
      "loss": 0.004,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.01349672395735979,
      "rewards/margins": 0.0265080276876688,
      "rewards/rejected": -0.040004752576351166,
      "step": 440
    },
    {
      "epoch": 1.37,
      "learning_rate": 1.3744906379558165e-06,
      "logits/chosen": -2.9248745441436768,
      "logits/rejected": -2.9322562217712402,
      "logps/chosen": -103.63645935058594,
      "logps/rejected": -123.60746765136719,
      "loss": 0.0383,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 0.014714512042701244,
      "rewards/margins": 0.11696286499500275,
      "rewards/rejected": -0.10224835574626923,
      "step": 450
    },
    {
      "epoch": 1.4,
      "learning_rate": 1.257667432355893e-06,
      "logits/chosen": -2.974544048309326,
      "logits/rejected": -2.780351161956787,
      "logps/chosen": -76.25318145751953,
      "logps/rejected": -95.62451934814453,
      "loss": 0.0157,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 0.0021516154520213604,
      "rewards/margins": 0.10200606286525726,
      "rewards/rejected": -0.09985444694757462,
      "step": 460
    },
    {
      "epoch": 1.43,
      "learning_rate": 1.1443420050626624e-06,
      "logits/chosen": -3.061354875564575,
      "logits/rejected": -2.9399447441101074,
      "logps/chosen": -71.61457824707031,
      "logps/rejected": -85.53517150878906,
      "loss": 0.047,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.00521849375218153,
      "rewards/margins": 0.06701686233282089,
      "rewards/rejected": -0.0722353607416153,
      "step": 470
    },
    {
      "epoch": 1.46,
      "learning_rate": 1.0348334229922677e-06,
      "logits/chosen": -3.0284695625305176,
      "logits/rejected": -2.7861952781677246,
      "logps/chosen": -84.50325012207031,
      "logps/rejected": -80.56702423095703,
      "loss": 0.0331,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.010606551542878151,
      "rewards/margins": 0.08576520532369614,
      "rewards/rejected": -0.07515865564346313,
      "step": 480
    },
    {
      "epoch": 1.49,
      "learning_rate": 9.294500067608941e-07,
      "logits/chosen": -2.929229974746704,
      "logits/rejected": -2.727160692214966,
      "logps/chosen": -106.28855895996094,
      "logps/rejected": -96.00597381591797,
      "loss": 0.0183,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.032337434589862823,
      "rewards/margins": 0.024760348722338676,
      "rewards/rejected": -0.05709778517484665,
      "step": 490
    },
    {
      "epoch": 1.52,
      "learning_rate": 8.284884626103165e-07,
      "logits/chosen": -2.9482462406158447,
      "logits/rejected": -2.7884809970855713,
      "logps/chosen": -77.4437255859375,
      "logps/rejected": -83.18536376953125,
      "loss": 0.0305,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 0.025916963815689087,
      "rewards/margins": 0.11217900365591049,
      "rewards/rejected": -0.086262047290802,
      "step": 500
    },
    {
      "epoch": 1.52,
      "eval_logits/chosen": -2.3874802589416504,
      "eval_logits/rejected": -2.1901655197143555,
      "eval_logps/chosen": -288.4842224121094,
      "eval_logps/rejected": -264.83880615234375,
      "eval_loss": 0.011713836342096329,
      "eval_rewards/accuracies": 0.3944999873638153,
      "eval_rewards/chosen": -0.08976726979017258,
      "eval_rewards/margins": -0.022982925176620483,
      "eval_rewards/rejected": -0.06678435206413269,
      "eval_runtime": 1421.6286,
      "eval_samples_per_second": 1.407,
      "eval_steps_per_second": 0.703,
      "step": 500
    },
    {
      "epoch": 1.55,
      "learning_rate": 7.322330470336314e-07,
      "logits/chosen": -3.0825541019439697,
      "logits/rejected": -2.9925718307495117,
      "logps/chosen": -95.14485931396484,
      "logps/rejected": -104.95149993896484,
      "loss": 0.0318,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 0.004621284082531929,
      "rewards/margins": 0.12566979229450226,
      "rewards/rejected": -0.12104851007461548,
      "step": 510
    },
    {
      "epoch": 1.58,
      "learning_rate": 6.409547664531734e-07,
      "logits/chosen": -2.9336235523223877,
      "logits/rejected": -2.8529655933380127,
      "logps/chosen": -130.2211151123047,
      "logps/rejected": -122.72911071777344,
      "loss": 0.0063,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": 0.002978086471557617,
      "rewards/margins": 0.05790657550096512,
      "rewards/rejected": -0.0549284890294075,
      "step": 520
    },
    {
      "epoch": 1.61,
      "learning_rate": 5.549106142039018e-07,
      "logits/chosen": -3.001368522644043,
      "logits/rejected": -2.9931020736694336,
      "logps/chosen": -122.59478759765625,
      "logps/rejected": -139.2528839111328,
      "loss": 0.0343,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.03542733192443848,
      "rewards/margins": 0.028016597032546997,
      "rewards/rejected": -0.06344393640756607,
      "step": 530
    },
    {
      "epoch": 1.64,
      "learning_rate": 4.743428469705336e-07,
      "logits/chosen": -2.895446300506592,
      "logits/rejected": -2.805732011795044,
      "logps/chosen": -77.88665771484375,
      "logps/rejected": -86.38732147216797,
      "loss": 0.0152,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 0.012998471967875957,
      "rewards/margins": 0.09006098657846451,
      "rewards/rejected": -0.07706250250339508,
      "step": 540
    },
    {
      "epoch": 1.67,
      "learning_rate": 3.994783027156143e-07,
      "logits/chosen": -2.8439111709594727,
      "logits/rejected": -2.765376567840576,
      "logps/chosen": -124.60064697265625,
      "logps/rejected": -115.8946533203125,
      "loss": 0.0116,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.0016067728865891695,
      "rewards/margins": 0.08273078501224518,
      "rewards/rejected": -0.08433755487203598,
      "step": 550
    },
    {
      "epoch": 1.7,
      "learning_rate": 3.3052776201888266e-07,
      "logits/chosen": -2.8251261711120605,
      "logits/rejected": -2.69478178024292,
      "logps/chosen": -85.72721862792969,
      "logps/rejected": -84.4536361694336,
      "loss": 0.0146,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.007551318965852261,
      "rewards/margins": 0.08319269120693207,
      "rewards/rejected": -0.07564137130975723,
      "step": 560
    },
    {
      "epoch": 1.73,
      "learning_rate": 2.676853546260791e-07,
      "logits/chosen": -2.8717586994171143,
      "logits/rejected": -2.805624485015869,
      "logps/chosen": -102.53792572021484,
      "logps/rejected": -89.75617980957031,
      "loss": 0.0086,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.008890565484762192,
      "rewards/margins": 0.04497462511062622,
      "rewards/rejected": -0.05386519432067871,
      "step": 570
    },
    {
      "epoch": 1.76,
      "learning_rate": 2.111280128780638e-07,
      "logits/chosen": -3.0598039627075195,
      "logits/rejected": -2.9559364318847656,
      "logps/chosen": -84.91885375976562,
      "logps/rejected": -75.99688720703125,
      "loss": 0.0162,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.025450268760323524,
      "rewards/margins": 0.03124385140836239,
      "rewards/rejected": -0.056694112718105316,
      "step": 580
    },
    {
      "epoch": 1.79,
      "learning_rate": 1.610149735590949e-07,
      "logits/chosen": -2.929579496383667,
      "logits/rejected": -2.8060431480407715,
      "logps/chosen": -60.950294494628906,
      "logps/rejected": -59.30121994018555,
      "loss": 0.0299,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.03121623769402504,
      "rewards/margins": 0.015263142995536327,
      "rewards/rejected": -0.04647938907146454,
      "step": 590
    },
    {
      "epoch": 1.82,
      "learning_rate": 1.1748732956682023e-07,
      "logits/chosen": -3.107717275619507,
      "logits/rejected": -2.9001567363739014,
      "logps/chosen": -90.45118713378906,
      "logps/rejected": -84.7816390991211,
      "loss": 0.0366,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.021300766617059708,
      "rewards/margins": 0.0555335208773613,
      "rewards/rejected": -0.0768342837691307,
      "step": 600
    },
    {
      "epoch": 1.82,
      "eval_logits/chosen": -2.3873283863067627,
      "eval_logits/rejected": -2.1899969577789307,
      "eval_logps/chosen": -288.27227783203125,
      "eval_logps/rejected": -264.630126953125,
      "eval_loss": 0.011489564552903175,
      "eval_rewards/accuracies": 0.4000000059604645,
      "eval_rewards/chosen": -0.08764798194169998,
      "eval_rewards/margins": -0.02295052260160446,
      "eval_rewards/rejected": -0.06469745188951492,
      "eval_runtime": 1422.4891,
      "eval_samples_per_second": 1.406,
      "eval_steps_per_second": 0.703,
      "step": 600
    },
    {
      "epoch": 1.85,
      "learning_rate": 8.066763266625283e-08,
      "logits/chosen": -2.9506607055664062,
      "logits/rejected": -2.825108051300049,
      "logps/chosen": -100.47017669677734,
      "logps/rejected": -127.78892517089844,
      "loss": 0.0349,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.004581383429467678,
      "rewards/margins": 0.1253204494714737,
      "rewards/rejected": -0.12073905766010284,
      "step": 610
    },
    {
      "epoch": 1.88,
      "learning_rate": 5.065954844616722e-08,
      "logits/chosen": -2.9829344749450684,
      "logits/rejected": -2.7227559089660645,
      "logps/chosen": -101.5233154296875,
      "logps/rejected": -99.42388916015625,
      "loss": 0.027,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.0389946773648262,
      "rewards/margins": 0.016298165544867516,
      "rewards/rejected": -0.05529283359646797,
      "step": 620
    },
    {
      "epoch": 1.91,
      "learning_rate": 2.7547564449386666e-08,
      "logits/chosen": -2.7666523456573486,
      "logits/rejected": -2.719158887863159,
      "logps/chosen": -215.42599487304688,
      "logps/rejected": -217.2696075439453,
      "loss": 0.0308,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.015156986191868782,
      "rewards/margins": 0.07782644778490067,
      "rewards/rejected": -0.0929834321141243,
      "step": 630
    },
    {
      "epoch": 1.95,
      "learning_rate": 1.1396752298723501e-08,
      "logits/chosen": -2.9985458850860596,
      "logits/rejected": -2.927014112472534,
      "logps/chosen": -109.7343521118164,
      "logps/rejected": -111.255859375,
      "loss": 0.0344,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": 0.020465310662984848,
      "rewards/margins": 0.10245206207036972,
      "rewards/rejected": -0.08198676258325577,
      "step": 640
    },
    {
      "epoch": 1.98,
      "learning_rate": 2.252584488296461e-09,
      "logits/chosen": -2.9206175804138184,
      "logits/rejected": -2.8202061653137207,
      "logps/chosen": -157.34161376953125,
      "logps/rejected": -171.29029846191406,
      "loss": 0.0403,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.005368991754949093,
      "rewards/margins": 0.11630947887897491,
      "rewards/rejected": -0.12167844921350479,
      "step": 650
    },
    {
      "epoch": 2.0,
      "step": 658,
      "total_flos": 0.0,
      "train_loss": 0.014390302914878392,
      "train_runtime": 11750.1681,
      "train_samples_per_second": 0.112,
      "train_steps_per_second": 0.056
    }
  ],
  "logging_steps": 10,
  "max_steps": 658,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 2,
  "save_steps": 100,
  "total_flos": 0.0,
  "train_batch_size": 1,
  "trial_name": null,
  "trial_params": null
}