{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 1.9968602825745683,
  "eval_steps": 100,
  "global_step": 954,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.0,
      "learning_rate": 5.208333333333333e-08,
      "logits/chosen": 0.952304482460022,
      "logits/rejected": 0.5888463854789734,
      "logps/chosen": -223.79486083984375,
      "logps/rejected": -209.482666015625,
      "loss": 0.6931,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 1
    },
    {
      "epoch": 0.02,
      "learning_rate": 5.208333333333334e-07,
      "logits/chosen": 0.836148738861084,
      "logits/rejected": 0.8545415997505188,
      "logps/chosen": -236.217529296875,
      "logps/rejected": -221.92974853515625,
      "loss": 0.6932,
      "rewards/accuracies": 0.4444444477558136,
      "rewards/chosen": 0.00045002170372754335,
      "rewards/margins": 0.0005102179129607975,
      "rewards/rejected": -6.019628563080914e-05,
      "step": 10
    },
    {
      "epoch": 0.04,
      "learning_rate": 1.0416666666666667e-06,
      "logits/chosen": 0.8339964151382446,
      "logits/rejected": 0.9286314249038696,
      "logps/chosen": -254.7573699951172,
      "logps/rejected": -247.9052734375,
      "loss": 0.6931,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -0.0004581348621286452,
      "rewards/margins": 0.00032393785659223795,
      "rewards/rejected": -0.0007820727187208831,
      "step": 20
    },
    {
      "epoch": 0.06,
      "learning_rate": 1.5625e-06,
      "logits/chosen": 0.8594380617141724,
      "logits/rejected": 0.916830837726593,
      "logps/chosen": -260.4742736816406,
      "logps/rejected": -232.0736846923828,
      "loss": 0.6929,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": 0.0009176501189358532,
      "rewards/margins": 0.0013712994987145066,
      "rewards/rejected": -0.00045364940888248384,
      "step": 30
    },
    {
      "epoch": 0.08,
      "learning_rate": 2.0833333333333334e-06,
      "logits/chosen": 0.8113815188407898,
      "logits/rejected": 0.9032772183418274,
      "logps/chosen": -280.20867919921875,
      "logps/rejected": -228.60598754882812,
      "loss": 0.693,
      "rewards/accuracies": 0.4937500059604645,
      "rewards/chosen": 3.258117794757709e-05,
      "rewards/margins": 0.0006167444516904652,
      "rewards/rejected": -0.0005841633537784219,
      "step": 40
    },
    {
      "epoch": 0.1,
      "learning_rate": 2.604166666666667e-06,
      "logits/chosen": 0.8574679493904114,
      "logits/rejected": 0.9363411068916321,
      "logps/chosen": -257.6195373535156,
      "logps/rejected": -219.19448852539062,
      "loss": 0.6928,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": 0.0007853487622924149,
      "rewards/margins": 0.001927538076415658,
      "rewards/rejected": -0.0011421891395002604,
      "step": 50
    },
    {
      "epoch": 0.13,
      "learning_rate": 3.125e-06,
      "logits/chosen": 0.8745417594909668,
      "logits/rejected": 0.901624858379364,
      "logps/chosen": -237.000244140625,
      "logps/rejected": -237.52395629882812,
      "loss": 0.6929,
      "rewards/accuracies": 0.48750001192092896,
      "rewards/chosen": 1.0722555089159869e-05,
      "rewards/margins": 0.0009478582069277763,
      "rewards/rejected": -0.0009371357737109065,
      "step": 60
    },
    {
      "epoch": 0.15,
      "learning_rate": 3.6458333333333333e-06,
      "logits/chosen": 0.8507300615310669,
      "logits/rejected": 0.8796240091323853,
      "logps/chosen": -260.81085205078125,
      "logps/rejected": -227.607421875,
      "loss": 0.6931,
      "rewards/accuracies": 0.48750001192092896,
      "rewards/chosen": -0.0008873953483998775,
      "rewards/margins": -0.0005922773270867765,
      "rewards/rejected": -0.00029511802131310105,
      "step": 70
    },
    {
      "epoch": 0.17,
      "learning_rate": 4.166666666666667e-06,
      "logits/chosen": 0.8868101239204407,
      "logits/rejected": 0.9193674921989441,
      "logps/chosen": -251.10498046875,
      "logps/rejected": -231.2660675048828,
      "loss": 0.6929,
      "rewards/accuracies": 0.518750011920929,
      "rewards/chosen": -0.00020074064377695322,
      "rewards/margins": 0.0003954143321607262,
      "rewards/rejected": -0.0005961551214568317,
      "step": 80
    },
    {
      "epoch": 0.19,
      "learning_rate": 4.6875000000000004e-06,
      "logits/chosen": 0.8633348345756531,
      "logits/rejected": 0.9175950884819031,
      "logps/chosen": -225.3305206298828,
      "logps/rejected": -241.3758087158203,
      "loss": 0.6931,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.0006292141624726355,
      "rewards/margins": 0.0004391434194985777,
      "rewards/rejected": -0.0010683576110750437,
      "step": 90
    },
    {
      "epoch": 0.21,
      "learning_rate": 4.999731868769027e-06,
      "logits/chosen": 0.9241905212402344,
      "logits/rejected": 0.9234986305236816,
      "logps/chosen": -242.17343139648438,
      "logps/rejected": -221.4640350341797,
      "loss": 0.6929,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -0.00037607658305205405,
      "rewards/margins": 0.0006910150987096131,
      "rewards/rejected": -0.0010670917108654976,
      "step": 100
    },
    {
      "epoch": 0.21,
      "eval_logits/chosen": 0.8145024180412292,
      "eval_logits/rejected": 0.9167963862419128,
      "eval_logps/chosen": -260.2240295410156,
      "eval_logps/rejected": -231.53602600097656,
      "eval_loss": 0.6927996873855591,
      "eval_rewards/accuracies": 0.5320000052452087,
      "eval_rewards/chosen": 0.00020143474102951586,
      "eval_rewards/margins": 0.0012005382450297475,
      "eval_rewards/rejected": -0.000999103649519384,
      "eval_runtime": 326.2051,
      "eval_samples_per_second": 6.131,
      "eval_steps_per_second": 0.383,
      "step": 100
    },
    {
      "epoch": 0.23,
      "learning_rate": 4.996716052911017e-06,
      "logits/chosen": 0.8340839147567749,
      "logits/rejected": 0.8756756782531738,
      "logps/chosen": -264.01495361328125,
      "logps/rejected": -219.5539093017578,
      "loss": 0.6927,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": 7.205537986010313e-05,
      "rewards/margins": 0.0008332778816111386,
      "rewards/rejected": -0.0007612224435433745,
      "step": 110
    },
    {
      "epoch": 0.25,
      "learning_rate": 4.9903533134293035e-06,
      "logits/chosen": 0.8605300784111023,
      "logits/rejected": 0.9697392582893372,
      "logps/chosen": -254.9059295654297,
      "logps/rejected": -219.1450958251953,
      "loss": 0.6924,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": 0.0002610751544125378,
      "rewards/margins": 0.002276130486279726,
      "rewards/rejected": -0.002015055390074849,
      "step": 120
    },
    {
      "epoch": 0.27,
      "learning_rate": 4.9806521797692184e-06,
      "logits/chosen": 0.877278208732605,
      "logits/rejected": 0.8787814974784851,
      "logps/chosen": -264.6486511230469,
      "logps/rejected": -247.06497192382812,
      "loss": 0.6921,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -0.00013167767610866576,
      "rewards/margins": 0.002138237003237009,
      "rewards/rejected": -0.0022699148394167423,
      "step": 130
    },
    {
      "epoch": 0.29,
      "learning_rate": 4.967625656594782e-06,
      "logits/chosen": 0.8538626432418823,
      "logits/rejected": 0.9312636256217957,
      "logps/chosen": -222.04147338867188,
      "logps/rejected": -232.23892211914062,
      "loss": 0.6921,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -0.0011232274118810892,
      "rewards/margins": 0.0009854320669546723,
      "rewards/rejected": -0.0021086593624204397,
      "step": 140
    },
    {
      "epoch": 0.31,
      "learning_rate": 4.95129120635556e-06,
      "logits/chosen": 0.8747261166572571,
      "logits/rejected": 0.9058718681335449,
      "logps/chosen": -258.2791442871094,
      "logps/rejected": -215.7833251953125,
      "loss": 0.6919,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.0004979403456673026,
      "rewards/margins": 0.002242505783215165,
      "rewards/rejected": -0.0027404462452977896,
      "step": 150
    },
    {
      "epoch": 0.33,
      "learning_rate": 4.93167072587771e-06,
      "logits/chosen": 0.7788752317428589,
      "logits/rejected": 0.8470233082771301,
      "logps/chosen": -257.824951171875,
      "logps/rejected": -250.79281616210938,
      "loss": 0.6914,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.0017024253029376268,
      "rewards/margins": 0.0027075002435594797,
      "rewards/rejected": -0.0044099255464971066,
      "step": 160
    },
    {
      "epoch": 0.36,
      "learning_rate": 4.908790517010637e-06,
      "logits/chosen": 0.9306305050849915,
      "logits/rejected": 0.9719738960266113,
      "logps/chosen": -239.07101440429688,
      "logps/rejected": -252.9778289794922,
      "loss": 0.6911,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.002183904405683279,
      "rewards/margins": 0.002954904455691576,
      "rewards/rejected": -0.005138809327036142,
      "step": 170
    },
    {
      "epoch": 0.38,
      "learning_rate": 4.882681251368549e-06,
      "logits/chosen": 0.853600800037384,
      "logits/rejected": 0.8488330841064453,
      "logps/chosen": -270.89105224609375,
      "logps/rejected": -257.0163269042969,
      "loss": 0.6906,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.0012318979715928435,
      "rewards/margins": 0.004222923889756203,
      "rewards/rejected": -0.0054548210464417934,
      "step": 180
    },
    {
      "epoch": 0.4,
      "learning_rate": 4.853377929214243e-06,
      "logits/chosen": 0.7871135473251343,
      "logits/rejected": 0.8603521585464478,
      "logps/chosen": -252.1376190185547,
      "logps/rejected": -245.2152862548828,
      "loss": 0.6898,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.0007233443320728838,
      "rewards/margins": 0.007929561659693718,
      "rewards/rejected": -0.008652905933558941,
      "step": 190
    },
    {
      "epoch": 0.42,
      "learning_rate": 4.8209198325401815e-06,
      "logits/chosen": 0.8782480955123901,
      "logits/rejected": 0.8951404690742493,
      "logps/chosen": -237.61087036132812,
      "logps/rejected": -233.7824249267578,
      "loss": 0.6893,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.002221657894551754,
      "rewards/margins": 0.00844704918563366,
      "rewards/rejected": -0.010668707080185413,
      "step": 200
    },
    {
      "epoch": 0.42,
      "eval_logits/chosen": 0.8204554915428162,
      "eval_logits/rejected": 0.9234155416488647,
      "eval_logps/chosen": -260.62249755859375,
      "eval_logps/rejected": -232.77418518066406,
      "eval_loss": 0.6890601515769958,
      "eval_rewards/accuracies": 0.6499999761581421,
      "eval_rewards/chosen": -0.0037833875976502895,
      "eval_rewards/margins": 0.009597329422831535,
      "eval_rewards/rejected": -0.013380718417465687,
      "eval_runtime": 326.0716,
      "eval_samples_per_second": 6.134,
      "eval_steps_per_second": 0.383,
      "step": 200
    },
    {
      "epoch": 0.44,
      "learning_rate": 4.785350472409792e-06,
      "logits/chosen": 0.906767725944519,
      "logits/rejected": 0.9123601913452148,
      "logps/chosen": -233.3804168701172,
      "logps/rejected": -228.94284057617188,
      "loss": 0.689,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.00779892411082983,
      "rewards/margins": 0.007471003569662571,
      "rewards/rejected": -0.015269925817847252,
      "step": 210
    },
    {
      "epoch": 0.46,
      "learning_rate": 4.746717530629565e-06,
      "logits/chosen": 0.8570274114608765,
      "logits/rejected": 0.8925272226333618,
      "logps/chosen": -259.993896484375,
      "logps/rejected": -239.8516082763672,
      "loss": 0.6887,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.006393183022737503,
      "rewards/margins": 0.01187940128147602,
      "rewards/rejected": -0.018272582441568375,
      "step": 220
    },
    {
      "epoch": 0.48,
      "learning_rate": 4.7050727958301505e-06,
      "logits/chosen": 0.8952150344848633,
      "logits/rejected": 0.8783475756645203,
      "logps/chosen": -244.82919311523438,
      "logps/rejected": -230.5172882080078,
      "loss": 0.687,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.008937228471040726,
      "rewards/margins": 0.014644329436123371,
      "rewards/rejected": -0.023581556975841522,
      "step": 230
    },
    {
      "epoch": 0.5,
      "learning_rate": 4.660472094042121e-06,
      "logits/chosen": 0.857439398765564,
      "logits/rejected": 0.8656377792358398,
      "logps/chosen": -282.5680847167969,
      "logps/rejected": -236.55111694335938,
      "loss": 0.6853,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.012710051611065865,
      "rewards/margins": 0.017861289903521538,
      "rewards/rejected": -0.030571341514587402,
      "step": 240
    },
    {
      "epoch": 0.52,
      "learning_rate": 4.612975213859487e-06,
      "logits/chosen": 0.8455727696418762,
      "logits/rejected": 0.9185819625854492,
      "logps/chosen": -268.9493713378906,
      "logps/rejected": -243.8383331298828,
      "loss": 0.6852,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.014121539890766144,
      "rewards/margins": 0.015637289732694626,
      "rewards/rejected": -0.02975882962346077,
      "step": 250
    },
    {
      "epoch": 0.54,
      "learning_rate": 4.5626458262912745e-06,
      "logits/chosen": 0.858254075050354,
      "logits/rejected": 0.8988651037216187,
      "logps/chosen": -274.9979553222656,
      "logps/rejected": -262.4846496582031,
      "loss": 0.6842,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.008313321508467197,
      "rewards/margins": 0.024480264633893967,
      "rewards/rejected": -0.03279358521103859,
      "step": 260
    },
    {
      "epoch": 0.57,
      "learning_rate": 4.509551399408598e-06,
      "logits/chosen": 0.9360445737838745,
      "logits/rejected": 0.9437822103500366,
      "logps/chosen": -253.78689575195312,
      "logps/rejected": -211.69509887695312,
      "loss": 0.683,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.015140311792492867,
      "rewards/margins": 0.022892246022820473,
      "rewards/rejected": -0.03803255409002304,
      "step": 270
    },
    {
      "epoch": 0.59,
      "learning_rate": 4.453763107901676e-06,
      "logits/chosen": 0.9350343942642212,
      "logits/rejected": 0.8994883298873901,
      "logps/chosen": -247.9213104248047,
      "logps/rejected": -256.08056640625,
      "loss": 0.6842,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.025496864691376686,
      "rewards/margins": 0.012935856357216835,
      "rewards/rejected": -0.03843272104859352,
      "step": 280
    },
    {
      "epoch": 0.61,
      "learning_rate": 4.3953557376679856e-06,
      "logits/chosen": 0.8583111763000488,
      "logits/rejected": 0.8643442988395691,
      "logps/chosen": -262.2920837402344,
      "logps/rejected": -258.9486083984375,
      "loss": 0.6827,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.027247220277786255,
      "rewards/margins": 0.015835126861929893,
      "rewards/rejected": -0.0430823490023613,
      "step": 290
    },
    {
      "epoch": 0.63,
      "learning_rate": 4.33440758555951e-06,
      "logits/chosen": 0.8355463743209839,
      "logits/rejected": 0.9161020517349243,
      "logps/chosen": -250.9317626953125,
      "logps/rejected": -247.6198272705078,
      "loss": 0.6809,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.018615344539284706,
      "rewards/margins": 0.033209316432476044,
      "rewards/rejected": -0.0518246591091156,
      "step": 300
    },
    {
      "epoch": 0.63,
      "eval_logits/chosen": 0.8092363476753235,
      "eval_logits/rejected": 0.9150914549827576,
      "eval_logps/chosen": -263.3646545410156,
      "eval_logps/rejected": -237.54312133789062,
      "eval_loss": 0.6809768080711365,
      "eval_rewards/accuracies": 0.6679999828338623,
      "eval_rewards/chosen": -0.03120502457022667,
      "eval_rewards/margins": 0.029865048825740814,
      "eval_rewards/rejected": -0.061070073395967484,
      "eval_runtime": 325.9788,
      "eval_samples_per_second": 6.135,
      "eval_steps_per_second": 0.383,
      "step": 300
    },
    {
      "epoch": 0.65,
      "learning_rate": 4.2710003544234255e-06,
      "logits/chosen": 0.855351448059082,
      "logits/rejected": 0.8703336715698242,
      "logps/chosen": -238.54061889648438,
      "logps/rejected": -230.7489776611328,
      "loss": 0.6787,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.036315977573394775,
      "rewards/margins": 0.025386247783899307,
      "rewards/rejected": -0.061702221632003784,
      "step": 310
    },
    {
      "epoch": 0.67,
      "learning_rate": 4.205219043576955e-06,
      "logits/chosen": 0.8615506887435913,
      "logits/rejected": 0.8854458928108215,
      "logps/chosen": -227.1722869873047,
      "logps/rejected": -220.97412109375,
      "loss": 0.6785,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.03975047916173935,
      "rewards/margins": 0.028606727719306946,
      "rewards/rejected": -0.06835721433162689,
      "step": 320
    },
    {
      "epoch": 0.69,
      "learning_rate": 4.137151834863213e-06,
      "logits/chosen": 0.8440972566604614,
      "logits/rejected": 0.8271803855895996,
      "logps/chosen": -256.4277038574219,
      "logps/rejected": -264.7651062011719,
      "loss": 0.6787,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.04135146737098694,
      "rewards/margins": 0.039213813841342926,
      "rewards/rejected": -0.08056528866291046,
      "step": 330
    },
    {
      "epoch": 0.71,
      "learning_rate": 4.066889974440757e-06,
      "logits/chosen": 0.8700094223022461,
      "logits/rejected": 0.8766329884529114,
      "logps/chosen": -269.4642333984375,
      "logps/rejected": -255.2469024658203,
      "loss": 0.6737,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.03877495974302292,
      "rewards/margins": 0.04113053157925606,
      "rewards/rejected": -0.07990548759698868,
      "step": 340
    },
    {
      "epoch": 0.73,
      "learning_rate": 3.994527650465352e-06,
      "logits/chosen": 0.8648789525032043,
      "logits/rejected": 0.8878329396247864,
      "logps/chosen": -227.1282958984375,
      "logps/rejected": -212.8804931640625,
      "loss": 0.6755,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.05504993349313736,
      "rewards/margins": 0.040687672793865204,
      "rewards/rejected": -0.09573759883642197,
      "step": 350
    },
    {
      "epoch": 0.75,
      "learning_rate": 3.92016186682789e-06,
      "logits/chosen": 0.7903240919113159,
      "logits/rejected": 0.8026289939880371,
      "logps/chosen": -214.0880889892578,
      "logps/rejected": -244.24618530273438,
      "loss": 0.6752,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.06658105552196503,
      "rewards/margins": 0.03603743016719818,
      "rewards/rejected": -0.1026184931397438,
      "step": 360
    },
    {
      "epoch": 0.77,
      "learning_rate": 3.843892313117724e-06,
      "logits/chosen": 0.8764356374740601,
      "logits/rejected": 0.9107359051704407,
      "logps/chosen": -273.88873291015625,
      "logps/rejected": -251.10812377929688,
      "loss": 0.6758,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.06244384124875069,
      "rewards/margins": 0.04542430490255356,
      "rewards/rejected": -0.10786814987659454,
      "step": 370
    },
    {
      "epoch": 0.8,
      "learning_rate": 3.7658212309857576e-06,
      "logits/chosen": 0.8514490127563477,
      "logits/rejected": 0.8745658993721008,
      "logps/chosen": -244.4405059814453,
      "logps/rejected": -224.3061981201172,
      "loss": 0.6708,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.06743437796831131,
      "rewards/margins": 0.048164453357458115,
      "rewards/rejected": -0.11559884250164032,
      "step": 380
    },
    {
      "epoch": 0.82,
      "learning_rate": 3.686053277086401e-06,
      "logits/chosen": 0.8000567555427551,
      "logits/rejected": 0.8884351849555969,
      "logps/chosen": -268.4157409667969,
      "logps/rejected": -247.18115234375,
      "loss": 0.6696,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.07314668595790863,
      "rewards/margins": 0.049585141241550446,
      "rewards/rejected": -0.12273182719945908,
      "step": 390
    },
    {
      "epoch": 0.84,
      "learning_rate": 3.604695382782159e-06,
      "logits/chosen": 0.8107595443725586,
      "logits/rejected": 0.8327080607414246,
      "logps/chosen": -284.17218017578125,
      "logps/rejected": -263.6735534667969,
      "loss": 0.6671,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.0770660936832428,
      "rewards/margins": 0.0486028790473938,
      "rewards/rejected": -0.1256689727306366,
      "step": 400
    },
    {
      "epoch": 0.84,
      "eval_logits/chosen": 0.7712985873222351,
      "eval_logits/rejected": 0.8790242671966553,
      "eval_logps/chosen": -268.7867431640625,
      "eval_logps/rejected": -245.51242065429688,
      "eval_loss": 0.6722846627235413,
      "eval_rewards/accuracies": 0.6639999747276306,
      "eval_rewards/chosen": -0.0854262262582779,
      "eval_rewards/margins": 0.055336710065603256,
      "eval_rewards/rejected": -0.14076292514801025,
      "eval_runtime": 325.7205,
      "eval_samples_per_second": 6.14,
      "eval_steps_per_second": 0.384,
      "step": 400
    },
    {
      "epoch": 0.86,
      "learning_rate": 3.5218566107988872e-06,
      "logits/chosen": 0.8288625478744507,
      "logits/rejected": 0.8551315069198608,
      "logps/chosen": -276.74462890625,
      "logps/rejected": -247.576416015625,
      "loss": 0.6718,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.08322989195585251,
      "rewards/margins": 0.06066075712442398,
      "rewards/rejected": -0.1438906490802765,
      "step": 410
    },
    {
      "epoch": 0.88,
      "learning_rate": 3.437648009023905e-06,
      "logits/chosen": 0.7773796916007996,
      "logits/rejected": 0.8443312644958496,
      "logps/chosen": -222.3345184326172,
      "logps/rejected": -218.86825561523438,
      "loss": 0.6696,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.10068164020776749,
      "rewards/margins": 0.04352349415421486,
      "rewards/rejected": -0.14420512318611145,
      "step": 420
    },
    {
      "epoch": 0.9,
      "learning_rate": 3.352182461642929e-06,
      "logits/chosen": 0.8025181889533997,
      "logits/rejected": 0.8646955490112305,
      "logps/chosen": -243.1133270263672,
      "logps/rejected": -232.02047729492188,
      "loss": 0.6638,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.08811922371387482,
      "rewards/margins": 0.06207479164004326,
      "rewards/rejected": -0.15019401907920837,
      "step": 430
    },
    {
      "epoch": 0.92,
      "learning_rate": 3.265574537815398e-06,
      "logits/chosen": 0.8092749714851379,
      "logits/rejected": 0.8513230085372925,
      "logps/chosen": -288.71636962890625,
      "logps/rejected": -256.467529296875,
      "loss": 0.6676,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.0855972021818161,
      "rewards/margins": 0.07402561604976654,
      "rewards/rejected": -0.15962281823158264,
      "step": 440
    },
    {
      "epoch": 0.94,
      "learning_rate": 3.177940338091043e-06,
      "logits/chosen": 0.8230969309806824,
      "logits/rejected": 0.8771616816520691,
      "logps/chosen": -262.885009765625,
      "logps/rejected": -236.41366577148438,
      "loss": 0.6661,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.10893376171588898,
      "rewards/margins": 0.05591960996389389,
      "rewards/rejected": -0.16485336422920227,
      "step": 450
    },
    {
      "epoch": 0.96,
      "learning_rate": 3.089397338773569e-06,
      "logits/chosen": 0.8003429174423218,
      "logits/rejected": 0.8373273015022278,
      "logps/chosen": -269.59130859375,
      "logps/rejected": -243.3413543701172,
      "loss": 0.6635,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.12066192924976349,
      "rewards/margins": 0.07056786119937897,
      "rewards/rejected": -0.19122979044914246,
      "step": 460
    },
    {
      "epoch": 0.98,
      "learning_rate": 3.0000642344401115e-06,
      "logits/chosen": 0.7690973877906799,
      "logits/rejected": 0.8160909414291382,
      "logps/chosen": -250.65017700195312,
      "logps/rejected": -241.54763793945312,
      "loss": 0.6624,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.12458320707082748,
      "rewards/margins": 0.06558409333229065,
      "rewards/rejected": -0.19016727805137634,
      "step": 470
    },
    {
      "epoch": 1.0,
      "learning_rate": 2.9100607788275547e-06,
      "logits/chosen": 0.8020121455192566,
      "logits/rejected": 0.8617356419563293,
      "logps/chosen": -256.21771240234375,
      "logps/rejected": -253.4419708251953,
      "loss": 0.6658,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.1322631537914276,
      "rewards/margins": 0.06032481789588928,
      "rewards/rejected": -0.1925879716873169,
      "step": 480
    },
    {
      "epoch": 1.03,
      "learning_rate": 2.8195076242990124e-06,
      "logits/chosen": 0.7743502259254456,
      "logits/rejected": 0.8271788358688354,
      "logps/chosen": -252.33804321289062,
      "logps/rejected": -235.423828125,
      "loss": 0.6624,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.14343151450157166,
      "rewards/margins": 0.07069016247987747,
      "rewards/rejected": -0.21412166953086853,
      "step": 490
    },
    {
      "epoch": 1.05,
      "learning_rate": 2.72852616010567e-06,
      "logits/chosen": 0.7551236748695374,
      "logits/rejected": 0.7969120144844055,
      "logps/chosen": -266.84375,
      "logps/rejected": -254.37460327148438,
      "loss": 0.6627,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.1373104304075241,
      "rewards/margins": 0.07557874917984009,
      "rewards/rejected": -0.212889164686203,
      "step": 500
    },
    {
      "epoch": 1.05,
      "eval_logits/chosen": 0.7216536998748779,
      "eval_logits/rejected": 0.8293780088424683,
      "eval_logps/chosen": -275.1849365234375,
      "eval_logps/rejected": -254.370361328125,
      "eval_loss": 0.664509654045105,
      "eval_rewards/accuracies": 0.6679999828338623,
      "eval_rewards/chosen": -0.14940780401229858,
      "eval_rewards/margins": 0.07993472367525101,
      "eval_rewards/rejected": -0.2293425351381302,
      "eval_runtime": 325.7214,
      "eval_samples_per_second": 6.14,
      "eval_steps_per_second": 0.384,
      "step": 500
    },
    {
      "epoch": 1.07,
      "learning_rate": 2.637238349660819e-06,
      "logits/chosen": 0.7647446393966675,
      "logits/rejected": 0.8582034111022949,
      "logps/chosen": -251.79904174804688,
      "logps/rejected": -218.86587524414062,
      "loss": 0.665,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.16131970286369324,
      "rewards/margins": 0.06838177144527435,
      "rewards/rejected": -0.2297014743089676,
      "step": 510
    },
    {
      "epoch": 1.09,
      "learning_rate": 2.5457665670441937e-06,
      "logits/chosen": 0.8116461038589478,
      "logits/rejected": 0.8353972434997559,
      "logps/chosen": -263.5601501464844,
      "logps/rejected": -247.66909790039062,
      "loss": 0.6591,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.14197978377342224,
      "rewards/margins": 0.09031648933887482,
      "rewards/rejected": -0.23229627311229706,
      "step": 520
    },
    {
      "epoch": 1.11,
      "learning_rate": 2.4542334329558075e-06,
      "logits/chosen": 0.7308142781257629,
      "logits/rejected": 0.7632499933242798,
      "logps/chosen": -257.2229919433594,
      "logps/rejected": -250.84469604492188,
      "loss": 0.6607,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.15741421282291412,
      "rewards/margins": 0.07882841676473618,
      "rewards/rejected": -0.2362426221370697,
      "step": 530
    },
    {
      "epoch": 1.13,
      "learning_rate": 2.3627616503391813e-06,
      "logits/chosen": 0.7235521674156189,
      "logits/rejected": 0.7589094042778015,
      "logps/chosen": -274.9698791503906,
      "logps/rejected": -238.1727752685547,
      "loss": 0.6614,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.16767871379852295,
      "rewards/margins": 0.07072968035936356,
      "rewards/rejected": -0.2384084016084671,
      "step": 540
    },
    {
      "epoch": 1.15,
      "learning_rate": 2.271473839894331e-06,
      "logits/chosen": 0.7224727869033813,
      "logits/rejected": 0.7499375939369202,
      "logps/chosen": -283.6424560546875,
      "logps/rejected": -272.5013122558594,
      "loss": 0.6593,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.17033258080482483,
      "rewards/margins": 0.07838577777147293,
      "rewards/rejected": -0.24871835112571716,
      "step": 550
    },
    {
      "epoch": 1.17,
      "learning_rate": 2.1804923757009885e-06,
      "logits/chosen": 0.6960411667823792,
      "logits/rejected": 0.7274879813194275,
      "logps/chosen": -269.6304016113281,
      "logps/rejected": -250.3975830078125,
      "loss": 0.6614,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.18492145836353302,
      "rewards/margins": 0.07485240697860718,
      "rewards/rejected": -0.259773850440979,
      "step": 560
    },
    {
      "epoch": 1.19,
      "learning_rate": 2.089939221172446e-06,
      "logits/chosen": 0.7146167755126953,
      "logits/rejected": 0.7337481379508972,
      "logps/chosen": -288.8470153808594,
      "logps/rejected": -258.131591796875,
      "loss": 0.664,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.19552810490131378,
      "rewards/margins": 0.09874384850263596,
      "rewards/rejected": -0.29427194595336914,
      "step": 570
    },
    {
      "epoch": 1.21,
      "learning_rate": 1.9999357655598894e-06,
      "logits/chosen": 0.7304507493972778,
      "logits/rejected": 0.7039176225662231,
      "logps/chosen": -259.52008056640625,
      "logps/rejected": -250.97146606445312,
      "loss": 0.6591,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.19664093852043152,
      "rewards/margins": 0.058397077023983,
      "rewards/rejected": -0.2550380527973175,
      "step": 580
    },
    {
      "epoch": 1.23,
      "learning_rate": 1.9106026612264316e-06,
      "logits/chosen": 0.7533366084098816,
      "logits/rejected": 0.8193215131759644,
      "logps/chosen": -236.71243286132812,
      "logps/rejected": -238.93679809570312,
      "loss": 0.653,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.1914629340171814,
      "rewards/margins": 0.07271396368741989,
      "rewards/rejected": -0.2641769051551819,
      "step": 590
    },
    {
      "epoch": 1.26,
      "learning_rate": 1.8220596619089576e-06,
      "logits/chosen": 0.7307643890380859,
      "logits/rejected": 0.7107076048851013,
      "logps/chosen": -281.09710693359375,
      "logps/rejected": -258.6228942871094,
      "loss": 0.6476,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.19486048817634583,
      "rewards/margins": 0.10412123054265976,
      "rewards/rejected": -0.2989817261695862,
      "step": 600
    },
    {
      "epoch": 1.26,
      "eval_logits/chosen": 0.6828243136405945,
      "eval_logits/rejected": 0.7883425354957581,
      "eval_logps/chosen": -280.03369140625,
      "eval_logps/rejected": -261.11236572265625,
      "eval_loss": 0.659087061882019,
      "eval_rewards/accuracies": 0.6639999747276306,
      "eval_rewards/chosen": -0.1978950798511505,
      "eval_rewards/margins": 0.09886746108531952,
      "eval_rewards/rejected": -0.2967625558376312,
      "eval_runtime": 325.9412,
      "eval_samples_per_second": 6.136,
      "eval_steps_per_second": 0.384,
      "step": 600
    },
    {
      "epoch": 1.28,
      "learning_rate": 1.7344254621846018e-06,
      "logits/chosen": 0.700169026851654,
      "logits/rejected": 0.7478510141372681,
      "logps/chosen": -283.26641845703125,
      "logps/rejected": -278.6813049316406,
      "loss": 0.6514,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.191552072763443,
      "rewards/margins": 0.09749529510736465,
      "rewards/rejected": -0.28904733061790466,
      "step": 610
    },
    {
      "epoch": 1.3,
      "learning_rate": 1.647817538357072e-06,
      "logits/chosen": 0.7241575121879578,
      "logits/rejected": 0.7943159341812134,
      "logps/chosen": -267.4305725097656,
      "logps/rejected": -256.3984069824219,
      "loss": 0.6591,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.2104833871126175,
      "rewards/margins": 0.07614756375551224,
      "rewards/rejected": -0.2866309583187103,
      "step": 620
    },
    {
      "epoch": 1.32,
      "learning_rate": 1.5623519909760953e-06,
      "logits/chosen": 0.6998089551925659,
      "logits/rejected": 0.7587723731994629,
      "logps/chosen": -264.82110595703125,
      "logps/rejected": -267.1747741699219,
      "loss": 0.6551,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.22911302745342255,
      "rewards/margins": 0.07119645923376083,
      "rewards/rejected": -0.30030950903892517,
      "step": 630
    },
    {
      "epoch": 1.34,
      "learning_rate": 1.4781433892011132e-06,
      "logits/chosen": 0.7701424360275269,
      "logits/rejected": 0.7888699173927307,
      "logps/chosen": -262.2229309082031,
      "logps/rejected": -286.0958557128906,
      "loss": 0.658,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.2087526023387909,
      "rewards/margins": 0.07823307812213898,
      "rewards/rejected": -0.28698569536209106,
      "step": 640
    },
    {
      "epoch": 1.36,
      "learning_rate": 1.3953046172178413e-06,
      "logits/chosen": 0.7079032063484192,
      "logits/rejected": 0.7274273037910461,
      "logps/chosen": -276.0498352050781,
      "logps/rejected": -277.2353820800781,
      "loss": 0.6536,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.23646607995033264,
      "rewards/margins": 0.09612749516963959,
      "rewards/rejected": -0.33259356021881104,
      "step": 650
    },
    {
      "epoch": 1.38,
      "learning_rate": 1.3139467229135999e-06,
      "logits/chosen": 0.6826360821723938,
      "logits/rejected": 0.751067578792572,
      "logps/chosen": -241.80044555664062,
      "logps/rejected": -275.70989990234375,
      "loss": 0.6556,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.22094163298606873,
      "rewards/margins": 0.08276458084583282,
      "rewards/rejected": -0.30370622873306274,
      "step": 660
    },
    {
      "epoch": 1.4,
      "learning_rate": 1.2341787690142436e-06,
      "logits/chosen": 0.6527940630912781,
      "logits/rejected": 0.7365330457687378,
      "logps/chosen": -320.76385498046875,
      "logps/rejected": -271.78790283203125,
      "loss": 0.6552,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.20969745516777039,
      "rewards/margins": 0.11965203285217285,
      "rewards/rejected": -0.32934948801994324,
      "step": 670
    },
    {
      "epoch": 1.42,
      "learning_rate": 1.1561076868822756e-06,
      "logits/chosen": 0.6766701936721802,
      "logits/rejected": 0.7141178250312805,
      "logps/chosen": -265.7345886230469,
      "logps/rejected": -245.7202606201172,
      "loss": 0.6518,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.22580020129680634,
      "rewards/margins": 0.0855962485074997,
      "rewards/rejected": -0.31139642000198364,
      "step": 680
    },
    {
      "epoch": 1.44,
      "learning_rate": 1.079838133172111e-06,
      "logits/chosen": 0.7425110936164856,
      "logits/rejected": 0.7384303212165833,
      "logps/chosen": -278.8888854980469,
      "logps/rejected": -266.8258972167969,
      "loss": 0.6598,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.21307018399238586,
      "rewards/margins": 0.08619161695241928,
      "rewards/rejected": -0.29926180839538574,
      "step": 690
    },
    {
      "epoch": 1.47,
      "learning_rate": 1.0054723495346484e-06,
      "logits/chosen": 0.6221153736114502,
      "logits/rejected": 0.6998614072799683,
      "logps/chosen": -280.8089599609375,
      "logps/rejected": -257.3545837402344,
      "loss": 0.6488,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.22230124473571777,
      "rewards/margins": 0.09477819502353668,
      "rewards/rejected": -0.31707945466041565,
      "step": 700
    },
    {
      "epoch": 1.47,
      "eval_logits/chosen": 0.651095986366272,
      "eval_logits/rejected": 0.7549068927764893,
      "eval_logps/chosen": -283.3440246582031,
      "eval_logps/rejected": -265.57830810546875,
      "eval_loss": 0.6559058427810669,
      "eval_rewards/accuracies": 0.6620000004768372,
      "eval_rewards/chosen": -0.23099879920482635,
      "eval_rewards/margins": 0.11042327433824539,
      "eval_rewards/rejected": -0.34142205119132996,
      "eval_runtime": 325.7526,
      "eval_samples_per_second": 6.14,
      "eval_steps_per_second": 0.384,
      "step": 700
    },
    {
      "epoch": 1.49,
      "learning_rate": 9.331100255592437e-07,
      "logits/chosen": 0.7064228653907776,
      "logits/rejected": 0.7716813683509827,
      "logps/chosen": -300.17694091796875,
      "logps/rejected": -261.4855651855469,
      "loss": 0.6475,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.2144012451171875,
      "rewards/margins": 0.12820136547088623,
      "rewards/rejected": -0.34260261058807373,
      "step": 710
    },
    {
      "epoch": 1.51,
      "learning_rate": 8.628481651367876e-07,
      "logits/chosen": 0.6913928985595703,
      "logits/rejected": 0.7301486730575562,
      "logps/chosen": -286.8462829589844,
      "logps/rejected": -276.6960754394531,
      "loss": 0.6457,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.23222024738788605,
      "rewards/margins": 0.10797496885061264,
      "rewards/rejected": -0.3401952087879181,
      "step": 720
    },
    {
      "epoch": 1.53,
      "learning_rate": 7.947809564230446e-07,
      "logits/chosen": 0.7156798839569092,
      "logits/rejected": 0.7468422055244446,
      "logps/chosen": -288.5824890136719,
      "logps/rejected": -286.32293701171875,
      "loss": 0.6439,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.23393268883228302,
      "rewards/margins": 0.12296883016824722,
      "rewards/rejected": -0.35690149664878845,
      "step": 730
    },
    {
      "epoch": 1.55,
      "learning_rate": 7.289996455765749e-07,
      "logits/chosen": 0.668700098991394,
      "logits/rejected": 0.6845619678497314,
      "logps/chosen": -299.6327819824219,
      "logps/rejected": -283.09295654296875,
      "loss": 0.6584,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.2437756061553955,
      "rewards/margins": 0.07863186299800873,
      "rewards/rejected": -0.32240745425224304,
      "step": 740
    },
    {
      "epoch": 1.57,
      "learning_rate": 6.655924144404907e-07,
      "logits/chosen": 0.6234613656997681,
      "logits/rejected": 0.6867505311965942,
      "logps/chosen": -281.64007568359375,
      "logps/rejected": -268.5241394042969,
      "loss": 0.6515,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.21235065162181854,
      "rewards/margins": 0.12807723879814148,
      "rewards/rejected": -0.34042784571647644,
      "step": 750
    },
    {
      "epoch": 1.59,
      "learning_rate": 6.046442623320145e-07,
      "logits/chosen": 0.7112501859664917,
      "logits/rejected": 0.7498377561569214,
      "logps/chosen": -281.40875244140625,
      "logps/rejected": -251.820068359375,
      "loss": 0.6496,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.26181793212890625,
      "rewards/margins": 0.08466657996177673,
      "rewards/rejected": -0.3464844822883606,
      "step": 760
    },
    {
      "epoch": 1.61,
      "learning_rate": 5.462368920983249e-07,
      "logits/chosen": 0.6798941493034363,
      "logits/rejected": 0.7346147298812866,
      "logps/chosen": -266.92315673828125,
      "logps/rejected": -247.5948028564453,
      "loss": 0.6566,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.25860121846199036,
      "rewards/margins": 0.05472899600863457,
      "rewards/rejected": -0.31333020329475403,
      "step": 770
    },
    {
      "epoch": 1.63,
      "learning_rate": 4.904486005914027e-07,
      "logits/chosen": 0.6729373931884766,
      "logits/rejected": 0.7290818095207214,
      "logps/chosen": -307.30926513671875,
      "logps/rejected": -292.6742248535156,
      "loss": 0.6463,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.2319561243057251,
      "rewards/margins": 0.1268167346715927,
      "rewards/rejected": -0.3587728440761566,
      "step": 780
    },
    {
      "epoch": 1.65,
      "learning_rate": 4.373541737087264e-07,
      "logits/chosen": 0.6597692966461182,
      "logits/rejected": 0.7631260752677917,
      "logps/chosen": -288.7844543457031,
      "logps/rejected": -258.93450927734375,
      "loss": 0.6505,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.2365461140871048,
      "rewards/margins": 0.12159979343414307,
      "rewards/rejected": -0.35814589262008667,
      "step": 790
    },
    {
      "epoch": 1.67,
      "learning_rate": 3.8702478614051353e-07,
      "logits/chosen": 0.6355623006820679,
      "logits/rejected": 0.64263516664505,
      "logps/chosen": -261.3099670410156,
      "logps/rejected": -265.4583435058594,
      "loss": 0.6449,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.23573680222034454,
      "rewards/margins": 0.13916215300559998,
      "rewards/rejected": -0.3748989403247833,
      "step": 800
    },
    {
      "epoch": 1.67,
      "eval_logits/chosen": 0.6347466111183167,
      "eval_logits/rejected": 0.7372007966041565,
      "eval_logps/chosen": -285.4225769042969,
      "eval_logps/rejected": -268.3813781738281,
      "eval_loss": 0.6541773676872253,
      "eval_rewards/accuracies": 0.656000018119812,
      "eval_rewards/chosen": -0.2517840266227722,
      "eval_rewards/margins": 0.11766859143972397,
      "eval_rewards/rejected": -0.3694525957107544,
      "eval_runtime": 325.7549,
      "eval_samples_per_second": 6.14,
      "eval_steps_per_second": 0.384,
      "step": 800
    },
    {
      "epoch": 1.7,
      "learning_rate": 3.3952790595787986e-07,
      "logits/chosen": 0.7447446584701538,
      "logits/rejected": 0.7175893783569336,
      "logps/chosen": -275.09906005859375,
      "logps/rejected": -265.5423889160156,
      "loss": 0.6508,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.25975173711776733,
      "rewards/margins": 0.08070627599954605,
      "rewards/rejected": -0.3404580354690552,
      "step": 810
    },
    {
      "epoch": 1.72,
      "learning_rate": 2.9492720416985004e-07,
      "logits/chosen": 0.5959675908088684,
      "logits/rejected": 0.6474281549453735,
      "logps/chosen": -294.9203186035156,
      "logps/rejected": -284.7016906738281,
      "loss": 0.6455,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.2046935260295868,
      "rewards/margins": 0.13686785101890564,
      "rewards/rejected": -0.34156137704849243,
      "step": 820
    },
    {
      "epoch": 1.74,
      "learning_rate": 2.5328246937043526e-07,
      "logits/chosen": 0.6916497349739075,
      "logits/rejected": 0.7139278650283813,
      "logps/chosen": -264.2368469238281,
      "logps/rejected": -253.6294403076172,
      "loss": 0.6506,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.28065750002861023,
      "rewards/margins": 0.10091479122638702,
      "rewards/rejected": -0.38157230615615845,
      "step": 830
    },
    {
      "epoch": 1.76,
      "learning_rate": 2.1464952759020857e-07,
      "logits/chosen": 0.6801538467407227,
      "logits/rejected": 0.7134484648704529,
      "logps/chosen": -274.87481689453125,
      "logps/rejected": -282.79766845703125,
      "loss": 0.6522,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.2523757517337799,
      "rewards/margins": 0.11816386878490448,
      "rewards/rejected": -0.3705395758152008,
      "step": 840
    },
    {
      "epoch": 1.78,
      "learning_rate": 1.790801674598186e-07,
      "logits/chosen": 0.6462175846099854,
      "logits/rejected": 0.6658031940460205,
      "logps/chosen": -292.3714599609375,
      "logps/rejected": -269.7221984863281,
      "loss": 0.6522,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.24048607051372528,
      "rewards/margins": 0.10906896740198135,
      "rewards/rejected": -0.34955504536628723,
      "step": 850
    },
    {
      "epoch": 1.8,
      "learning_rate": 1.4662207078575685e-07,
      "logits/chosen": 0.6442946791648865,
      "logits/rejected": 0.7183449864387512,
      "logps/chosen": -256.558837890625,
      "logps/rejected": -274.9573669433594,
      "loss": 0.6476,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.2482720911502838,
      "rewards/margins": 0.1144469827413559,
      "rewards/rejected": -0.3627190887928009,
      "step": 860
    },
    {
      "epoch": 1.82,
      "learning_rate": 1.1731874863145143e-07,
      "logits/chosen": 0.68730229139328,
      "logits/rejected": 0.7417377829551697,
      "logps/chosen": -285.8447570800781,
      "logps/rejected": -258.12408447265625,
      "loss": 0.6483,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.24029162526130676,
      "rewards/margins": 0.10951662063598633,
      "rewards/rejected": -0.3498082160949707,
      "step": 870
    },
    {
      "epoch": 1.84,
      "learning_rate": 9.120948298936422e-08,
      "logits/chosen": 0.7039914131164551,
      "logits/rejected": 0.720660388469696,
      "logps/chosen": -275.22564697265625,
      "logps/rejected": -259.50921630859375,
      "loss": 0.6391,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.2585623860359192,
      "rewards/margins": 0.10217638313770294,
      "rewards/rejected": -0.36073875427246094,
      "step": 880
    },
    {
      "epoch": 1.86,
      "learning_rate": 6.832927412229017e-08,
      "logits/chosen": 0.634792685508728,
      "logits/rejected": 0.7409111857414246,
      "logps/chosen": -285.78076171875,
      "logps/rejected": -263.0660095214844,
      "loss": 0.6441,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.24097540974617004,
      "rewards/margins": 0.12255308777093887,
      "rewards/rejected": -0.3635285198688507,
      "step": 890
    },
    {
      "epoch": 1.88,
      "learning_rate": 4.870879364444109e-08,
      "logits/chosen": 0.6492586731910706,
      "logits/rejected": 0.7190669775009155,
      "logps/chosen": -275.6029357910156,
      "logps/rejected": -243.29281616210938,
      "loss": 0.6487,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.24271516501903534,
      "rewards/margins": 0.14071747660636902,
      "rewards/rejected": -0.38343262672424316,
      "step": 900
    },
    {
      "epoch": 1.88,
      "eval_logits/chosen": 0.6298695802688599,
      "eval_logits/rejected": 0.7320161461830139,
      "eval_logps/chosen": -285.9532165527344,
      "eval_logps/rejected": -269.07244873046875,
      "eval_loss": 0.6538792848587036,
      "eval_rewards/accuracies": 0.656000018119812,
      "eval_rewards/chosen": -0.25709065794944763,
      "eval_rewards/margins": 0.11927253752946854,
      "eval_rewards/rejected": -0.37636318802833557,
      "eval_runtime": 326.0646,
      "eval_samples_per_second": 6.134,
      "eval_steps_per_second": 0.383,
      "step": 900
    },
    {
      "epoch": 1.9,
      "learning_rate": 3.237434340521789e-08,
      "logits/chosen": 0.6427167654037476,
      "logits/rejected": 0.7207670211791992,
      "logps/chosen": -286.7334289550781,
      "logps/rejected": -282.4278259277344,
      "loss": 0.6455,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.24073800444602966,
      "rewards/margins": 0.1223975196480751,
      "rewards/rejected": -0.36313554644584656,
      "step": 910
    },
    {
      "epoch": 1.93,
      "learning_rate": 1.93478202307823e-08,
      "logits/chosen": 0.6422888040542603,
      "logits/rejected": 0.6643663644790649,
      "logps/chosen": -267.14398193359375,
      "logps/rejected": -278.8194274902344,
      "loss": 0.6473,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.257813036441803,
      "rewards/margins": 0.0857832059264183,
      "rewards/rejected": -0.34359627962112427,
      "step": 920
    },
    {
      "epoch": 1.95,
      "learning_rate": 9.646686570697062e-09,
      "logits/chosen": 0.6971439123153687,
      "logits/rejected": 0.722652792930603,
      "logps/chosen": -283.13067626953125,
      "logps/rejected": -285.1507568359375,
      "loss": 0.6572,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.2655975818634033,
      "rewards/margins": 0.10348521173000336,
      "rewards/rejected": -0.36908283829689026,
      "step": 930
    },
    {
      "epoch": 1.97,
      "learning_rate": 3.283947088983663e-09,
      "logits/chosen": 0.6804937720298767,
      "logits/rejected": 0.6877144575119019,
      "logps/chosen": -265.1544494628906,
      "logps/rejected": -278.4422302246094,
      "loss": 0.6544,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.27914106845855713,
      "rewards/margins": 0.08945213258266449,
      "rewards/rejected": -0.3685932457447052,
      "step": 940
    },
    {
      "epoch": 1.99,
      "learning_rate": 2.681312309735229e-10,
      "logits/chosen": 0.6534776091575623,
      "logits/rejected": 0.7444522976875305,
      "logps/chosen": -257.7799377441406,
      "logps/rejected": -263.00030517578125,
      "loss": 0.654,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.2684522867202759,
      "rewards/margins": 0.08305440098047256,
      "rewards/rejected": -0.35150665044784546,
      "step": 950
    },
    {
      "epoch": 2.0,
      "step": 954,
      "total_flos": 0.0,
      "train_loss": 0.6680307045922589,
      "train_runtime": 18174.3674,
      "train_samples_per_second": 3.364,
      "train_steps_per_second": 0.052
    }
  ],
  "logging_steps": 10,
  "max_steps": 954,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 2,
  "save_steps": 100,
  "total_flos": 0.0,
  "train_batch_size": 4,
  "trial_name": null,
  "trial_params": null
}