{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 2.994601079784043,
  "eval_steps": 500,
  "global_step": 1248,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.02399520095980804,
      "grad_norm": 24.58741331565172,
      "learning_rate": 1.0000000000000002e-06,
      "logits/chosen": -0.5075146555900574,
      "logits/rejected": -0.31934085488319397,
      "logps/chosen": -1.394007921218872,
      "logps/rejected": -1.3630257844924927,
      "loss": 1.3501,
      "odds_ratio_loss": 0.8239962458610535,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.06970040500164032,
      "rewards/margins": -0.0015491036465391517,
      "rewards/rejected": -0.06815129518508911,
      "sft_loss": 1.394007921218872,
      "step": 10
    },
    {
      "epoch": 0.04799040191961608,
      "grad_norm": 4.281683015852783,
      "learning_rate": 3.5e-06,
      "logits/chosen": 0.08614908158779144,
      "logits/rejected": 0.3013238310813904,
      "logps/chosen": -1.3080074787139893,
      "logps/rejected": -1.334457278251648,
      "loss": 1.2858,
      "odds_ratio_loss": 0.7804475426673889,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.0654003769159317,
      "rewards/margins": 0.0013224859721958637,
      "rewards/rejected": -0.06672286242246628,
      "sft_loss": 1.3080074787139893,
      "step": 20
    },
    {
      "epoch": 0.07198560287942411,
      "grad_norm": 3.830958349381369,
      "learning_rate": 4.99986910314335e-06,
      "logits/chosen": 0.3485943675041199,
      "logits/rejected": 0.6042150855064392,
      "logps/chosen": -0.9540683627128601,
      "logps/rejected": -1.1750730276107788,
      "loss": 0.9904,
      "odds_ratio_loss": 0.6533687710762024,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.047703422605991364,
      "rewards/margins": 0.011050237342715263,
      "rewards/rejected": -0.05875365808606148,
      "sft_loss": 0.9540683627128601,
      "step": 30
    },
    {
      "epoch": 0.09598080383923216,
      "grad_norm": 3.6776666943951675,
      "learning_rate": 4.998396670920005e-06,
      "logits/chosen": 0.17601105570793152,
      "logits/rejected": 0.5272272229194641,
      "logps/chosen": -0.898045539855957,
      "logps/rejected": -1.0136868953704834,
      "loss": 0.9614,
      "odds_ratio_loss": 0.6860688328742981,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -0.04490227997303009,
      "rewards/margins": 0.005782057531177998,
      "rewards/rejected": -0.05068434029817581,
      "sft_loss": 0.898045539855957,
      "step": 40
    },
    {
      "epoch": 0.11997600479904019,
      "grad_norm": 2.636908991979515,
      "learning_rate": 4.995289152254744e-06,
      "logits/chosen": 0.2309066355228424,
      "logits/rejected": 0.22152824699878693,
      "logps/chosen": -0.9074997901916504,
      "logps/rejected": -1.0551084280014038,
      "loss": 0.9374,
      "odds_ratio_loss": 0.663613498210907,
      "rewards/accuracies": 0.48750001192092896,
      "rewards/chosen": -0.04537498578429222,
      "rewards/margins": 0.007380434311926365,
      "rewards/rejected": -0.05275542289018631,
      "sft_loss": 0.9074997901916504,
      "step": 50
    },
    {
      "epoch": 0.14397120575884823,
      "grad_norm": 1.8300107701302537,
      "learning_rate": 4.990548580876516e-06,
      "logits/chosen": 0.307407021522522,
      "logits/rejected": 0.37507694959640503,
      "logps/chosen": -0.9279610514640808,
      "logps/rejected": -0.986476719379425,
      "loss": 0.9464,
      "odds_ratio_loss": 0.7063499093055725,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.04639805108308792,
      "rewards/margins": 0.00292578199878335,
      "rewards/rejected": -0.04932383447885513,
      "sft_loss": 0.9279610514640808,
      "step": 60
    },
    {
      "epoch": 0.16796640671865626,
      "grad_norm": 3.8157191209486507,
      "learning_rate": 4.9841780592726385e-06,
      "logits/chosen": 0.19509825110435486,
      "logits/rejected": 0.2650177776813507,
      "logps/chosen": -0.9848098754882812,
      "logps/rejected": -1.0149097442626953,
      "loss": 0.9578,
      "odds_ratio_loss": 0.726799488067627,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.04924049228429794,
      "rewards/margins": 0.0015049913199618459,
      "rewards/rejected": -0.050745487213134766,
      "sft_loss": 0.9848098754882812,
      "step": 70
    },
    {
      "epoch": 0.19196160767846432,
      "grad_norm": 4.078587531391316,
      "learning_rate": 4.976181756658363e-06,
      "logits/chosen": 0.061622969806194305,
      "logits/rejected": 0.2444450408220291,
      "logps/chosen": -0.8894473910331726,
      "logps/rejected": -1.0614734888076782,
      "loss": 0.9675,
      "odds_ratio_loss": 0.6382969617843628,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.04447237029671669,
      "rewards/margins": 0.008601305074989796,
      "rewards/rejected": -0.05307367444038391,
      "sft_loss": 0.8894473910331726,
      "step": 80
    },
    {
      "epoch": 0.21595680863827235,
      "grad_norm": 2.9874023740770363,
      "learning_rate": 4.9665649062483115e-06,
      "logits/chosen": 0.6337467432022095,
      "logits/rejected": 0.7902036905288696,
      "logps/chosen": -0.9439412951469421,
      "logps/rejected": -0.9588793516159058,
      "loss": 0.9635,
      "odds_ratio_loss": 0.7716476917266846,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.047197069972753525,
      "rewards/margins": 0.0007468975381925702,
      "rewards/rejected": -0.047943972051143646,
      "sft_loss": 0.9439412951469421,
      "step": 90
    },
    {
      "epoch": 0.23995200959808038,
      "grad_norm": 2.3029148332001745,
      "learning_rate": 4.955333801831578e-06,
      "logits/chosen": 0.49920982122421265,
      "logits/rejected": 0.6337569355964661,
      "logps/chosen": -0.8333128094673157,
      "logps/rejected": -1.059599757194519,
      "loss": 0.9453,
      "odds_ratio_loss": 0.6517213582992554,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.041665639728307724,
      "rewards/margins": 0.011314347386360168,
      "rewards/rejected": -0.05297998711466789,
      "sft_loss": 0.8333128094673157,
      "step": 100
    },
    {
      "epoch": 0.26394721055788845,
      "grad_norm": 2.8766587489414395,
      "learning_rate": 4.9424957936527295e-06,
      "logits/chosen": -0.28645992279052734,
      "logits/rejected": 0.04107431694865227,
      "logps/chosen": -0.9429195523262024,
      "logps/rejected": -0.9936224222183228,
      "loss": 0.9526,
      "odds_ratio_loss": 0.705885112285614,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.04714598134160042,
      "rewards/margins": 0.002535139676183462,
      "rewards/rejected": -0.04968111589550972,
      "sft_loss": 0.9429195523262024,
      "step": 110
    },
    {
      "epoch": 0.28794241151769645,
      "grad_norm": 2.1411106644617703,
      "learning_rate": 4.92805928360141e-06,
      "logits/chosen": -0.29608479142189026,
      "logits/rejected": -0.21111997961997986,
      "logps/chosen": -0.888851523399353,
      "logps/rejected": -1.0842912197113037,
      "loss": 0.8904,
      "odds_ratio_loss": 0.5968859195709229,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.04444257169961929,
      "rewards/margins": 0.009771987795829773,
      "rewards/rejected": -0.054214559495449066,
      "sft_loss": 0.888851523399353,
      "step": 120
    },
    {
      "epoch": 0.3119376124775045,
      "grad_norm": 2.1891227152981347,
      "learning_rate": 4.912033719713687e-06,
      "logits/chosen": 0.49228960275650024,
      "logits/rejected": 0.5680336952209473,
      "logps/chosen": -0.9152839779853821,
      "logps/rejected": -1.0058788061141968,
      "loss": 0.9427,
      "odds_ratio_loss": 0.6943625807762146,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.04576420038938522,
      "rewards/margins": 0.004529745317995548,
      "rewards/rejected": -0.0502939410507679,
      "sft_loss": 0.9152839779853821,
      "step": 130
    },
    {
      "epoch": 0.3359328134373125,
      "grad_norm": 2.5131225459939,
      "learning_rate": 4.894429589988739e-06,
      "logits/chosen": -1.2468726634979248,
      "logits/rejected": -1.0485397577285767,
      "logps/chosen": -1.0104249715805054,
      "logps/rejected": -1.0477244853973389,
      "loss": 0.949,
      "odds_ratio_loss": 0.7160865068435669,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -0.05052124708890915,
      "rewards/margins": 0.0018649749690666795,
      "rewards/rejected": -0.05238622426986694,
      "sft_loss": 1.0104249715805054,
      "step": 140
    },
    {
      "epoch": 0.3599280143971206,
      "grad_norm": 2.696319834123575,
      "learning_rate": 4.875258415524945e-06,
      "logits/chosen": 0.039508234709501266,
      "logits/rejected": 0.23594827950000763,
      "logps/chosen": -0.904223620891571,
      "logps/rejected": -1.032157063484192,
      "loss": 0.9533,
      "odds_ratio_loss": 0.6739581823348999,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.04521118476986885,
      "rewards/margins": 0.0063966671004891396,
      "rewards/rejected": -0.051607854664325714,
      "sft_loss": 0.904223620891571,
      "step": 150
    },
    {
      "epoch": 0.38392321535692864,
      "grad_norm": 2.241170193835809,
      "learning_rate": 4.85453274297985e-06,
      "logits/chosen": 0.4507044851779938,
      "logits/rejected": 0.7088828682899475,
      "logps/chosen": -0.9252007603645325,
      "logps/rejected": -1.0105345249176025,
      "loss": 0.9187,
      "odds_ratio_loss": 0.6664329171180725,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.0462600402534008,
      "rewards/margins": 0.004266692791134119,
      "rewards/rejected": -0.050526730716228485,
      "sft_loss": 0.9252007603645325,
      "step": 160
    },
    {
      "epoch": 0.40791841631673664,
      "grad_norm": 1.759854296483571,
      "learning_rate": 4.832266136358951e-06,
      "logits/chosen": -0.12876208126544952,
      "logits/rejected": 0.014335835352540016,
      "logps/chosen": -0.8540490865707397,
      "logps/rejected": -0.9863293766975403,
      "loss": 0.926,
      "odds_ratio_loss": 0.6714656352996826,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.04270245134830475,
      "rewards/margins": 0.006614011712372303,
      "rewards/rejected": -0.04931646212935448,
      "sft_loss": 0.8540490865707397,
      "step": 170
    },
    {
      "epoch": 0.4319136172765447,
      "grad_norm": 2.793191882203603,
      "learning_rate": 4.808473168138675e-06,
      "logits/chosen": 0.3617595136165619,
      "logits/rejected": 0.3396950364112854,
      "logps/chosen": -0.8613064885139465,
      "logps/rejected": -1.0067331790924072,
      "loss": 0.9162,
      "odds_ratio_loss": 0.6582903861999512,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.04306532442569733,
      "rewards/margins": 0.007271329872310162,
      "rewards/rejected": -0.050336651504039764,
      "sft_loss": 0.8613064885139465,
      "step": 180
    },
    {
      "epoch": 0.4559088182363527,
      "grad_norm": 1.7774141067161418,
      "learning_rate": 4.783169409729363e-06,
      "logits/chosen": 0.9685203433036804,
      "logits/rejected": 1.1009634733200073,
      "logps/chosen": -0.8521540760993958,
      "logps/rejected": -0.9150575399398804,
      "loss": 0.9004,
      "odds_ratio_loss": 0.7224193811416626,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -0.04260770231485367,
      "rewards/margins": 0.0031451724935323,
      "rewards/rejected": -0.0457528755068779,
      "sft_loss": 0.8521540760993958,
      "step": 190
    },
    {
      "epoch": 0.47990401919616077,
      "grad_norm": 2.052107783396207,
      "learning_rate": 4.756371421284482e-06,
      "logits/chosen": 0.33597105741500854,
      "logits/rejected": 0.44187426567077637,
      "logps/chosen": -0.8725342750549316,
      "logps/rejected": -0.9003400802612305,
      "loss": 0.919,
      "odds_ratio_loss": 0.7135496735572815,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.04362671449780464,
      "rewards/margins": 0.0013902939390391111,
      "rewards/rejected": -0.04501700773835182,
      "sft_loss": 0.8725342750549316,
      "step": 200
    },
    {
      "epoch": 0.5038992201559688,
      "grad_norm": 2.3000145040966973,
      "learning_rate": 4.728096740862778e-06,
      "logits/chosen": 0.16287042200565338,
      "logits/rejected": 0.35098087787628174,
      "logps/chosen": -0.8514264822006226,
      "logps/rejected": -0.9913795590400696,
      "loss": 0.9096,
      "odds_ratio_loss": 0.6634506583213806,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.042571328580379486,
      "rewards/margins": 0.006997650023549795,
      "rewards/rejected": -0.04956897348165512,
      "sft_loss": 0.8514264822006226,
      "step": 210
    },
    {
      "epoch": 0.5278944211157769,
      "grad_norm": 1.581079267248328,
      "learning_rate": 4.698363872950406e-06,
      "logits/chosen": 0.298981636762619,
      "logits/rejected": 0.49268895387649536,
      "logps/chosen": -0.8895601034164429,
      "logps/rejected": -1.026539921760559,
      "loss": 0.8744,
      "odds_ratio_loss": 0.6685082316398621,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.04447800666093826,
      "rewards/margins": 0.0068489923141896725,
      "rewards/rejected": -0.051326997578144073,
      "sft_loss": 0.8895601034164429,
      "step": 220
    },
    {
      "epoch": 0.5518896220755849,
      "grad_norm": 1.7094822098553022,
      "learning_rate": 4.6671922763505915e-06,
      "logits/chosen": 0.34609514474868774,
      "logits/rejected": 0.5052930116653442,
      "logps/chosen": -0.863084614276886,
      "logps/rejected": -0.9836879968643188,
      "loss": 0.8905,
      "odds_ratio_loss": 0.6813028454780579,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.043154239654541016,
      "rewards/margins": 0.006030158139765263,
      "rewards/rejected": -0.049184400588274,
      "sft_loss": 0.863084614276886,
      "step": 230
    },
    {
      "epoch": 0.5758848230353929,
      "grad_norm": 1.9367159826113498,
      "learning_rate": 4.634602351448738e-06,
      "logits/chosen": 0.286350816488266,
      "logits/rejected": 0.3788919448852539,
      "logps/chosen": -0.8919585943222046,
      "logps/rejected": -0.9452742338180542,
      "loss": 0.9133,
      "odds_ratio_loss": 0.6905114650726318,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.04459793120622635,
      "rewards/margins": 0.0026657807175070047,
      "rewards/rejected": -0.04726371169090271,
      "sft_loss": 0.8919585943222046,
      "step": 240
    },
    {
      "epoch": 0.5998800239952009,
      "grad_norm": 2.0772847936555636,
      "learning_rate": 4.6006154268613015e-06,
      "logits/chosen": 0.4635019898414612,
      "logits/rejected": 0.5444530248641968,
      "logps/chosen": -0.8181222081184387,
      "logps/rejected": -0.9908831715583801,
      "loss": 0.8927,
      "odds_ratio_loss": 0.6295598149299622,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.04090610891580582,
      "rewards/margins": 0.008638045750558376,
      "rewards/rejected": -0.04954415559768677,
      "sft_loss": 0.8181222081184387,
      "step": 250
    },
    {
      "epoch": 0.623875224955009,
      "grad_norm": 2.084215689408855,
      "learning_rate": 4.565253745477187e-06,
      "logits/chosen": 0.40253886580467224,
      "logits/rejected": 0.4625183045864105,
      "logps/chosen": -0.9301355481147766,
      "logps/rejected": -1.0306508541107178,
      "loss": 0.9162,
      "odds_ratio_loss": 0.6872043609619141,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.04650677740573883,
      "rewards/margins": 0.005025765858590603,
      "rewards/rejected": -0.05153254419565201,
      "sft_loss": 0.9301355481147766,
      "step": 260
    },
    {
      "epoch": 0.647870425914817,
      "grad_norm": 1.9031984888179019,
      "learning_rate": 4.528540449900799e-06,
      "logits/chosen": 0.4078219532966614,
      "logits/rejected": 0.6789823174476624,
      "logps/chosen": -0.8785255551338196,
      "logps/rejected": -0.9139087796211243,
      "loss": 0.9176,
      "odds_ratio_loss": 0.7333613038063049,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.04392627626657486,
      "rewards/margins": 0.0017691642278805375,
      "rewards/rejected": -0.04569543898105621,
      "sft_loss": 0.8785255551338196,
      "step": 270
    },
    {
      "epoch": 0.671865626874625,
      "grad_norm": 2.3067419173621113,
      "learning_rate": 4.490499567306256e-06,
      "logits/chosen": 0.304252564907074,
      "logits/rejected": 0.5160123109817505,
      "logps/chosen": -0.8951358795166016,
      "logps/rejected": -0.9636558294296265,
      "loss": 0.8917,
      "odds_ratio_loss": 0.69621342420578,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -0.04475679248571396,
      "rewards/margins": 0.0034259993117302656,
      "rewards/rejected": -0.04818279296159744,
      "sft_loss": 0.8951358795166016,
      "step": 280
    },
    {
      "epoch": 0.6958608278344331,
      "grad_norm": 3.1297290877323003,
      "learning_rate": 4.451155993712711e-06,
      "logits/chosen": 0.25184166431427,
      "logits/rejected": 0.43299436569213867,
      "logps/chosen": -0.808620810508728,
      "logps/rejected": -0.9780584573745728,
      "loss": 0.9379,
      "odds_ratio_loss": 0.6151310205459595,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.04043104499578476,
      "rewards/margins": 0.008471880108118057,
      "rewards/rejected": -0.048902928829193115,
      "sft_loss": 0.808620810508728,
      "step": 290
    },
    {
      "epoch": 0.7198560287942412,
      "grad_norm": 2.001570442654457,
      "learning_rate": 4.410535477691041e-06,
      "logits/chosen": 0.6736063957214355,
      "logits/rejected": 0.8922637104988098,
      "logps/chosen": -0.8743098974227905,
      "logps/rejected": -1.0198915004730225,
      "loss": 0.8962,
      "odds_ratio_loss": 0.6545746326446533,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.043715499341487885,
      "rewards/margins": 0.0072790831327438354,
      "rewards/rejected": -0.05099458247423172,
      "sft_loss": 0.8743098974227905,
      "step": 300
    },
    {
      "epoch": 0.7438512297540492,
      "grad_norm": 3.088640251108737,
      "learning_rate": 4.368664603512586e-06,
      "logits/chosen": -0.10074709355831146,
      "logits/rejected": 0.08682968467473984,
      "logps/chosen": -0.7929955720901489,
      "logps/rejected": -0.9449365735054016,
      "loss": 0.8789,
      "odds_ratio_loss": 0.6474851369857788,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.03964977711439133,
      "rewards/margins": 0.007597046438604593,
      "rewards/rejected": -0.047246821224689484,
      "sft_loss": 0.7929955720901489,
      "step": 310
    },
    {
      "epoch": 0.7678464307138573,
      "grad_norm": 2.278875813822025,
      "learning_rate": 4.325570773750952e-06,
      "logits/chosen": -0.22130906581878662,
      "logits/rejected": -0.028980206698179245,
      "logps/chosen": -0.8826779127120972,
      "logps/rejected": -1.0213041305541992,
      "loss": 0.9204,
      "odds_ratio_loss": 0.6443883180618286,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.04413389414548874,
      "rewards/margins": 0.006931307725608349,
      "rewards/rejected": -0.05106520652770996,
      "sft_loss": 0.8826779127120972,
      "step": 320
    },
    {
      "epoch": 0.7918416316736653,
      "grad_norm": 1.6952516043840655,
      "learning_rate": 4.281282191348289e-06,
      "logits/chosen": 0.45927032828330994,
      "logits/rejected": 0.6593443751335144,
      "logps/chosen": -0.8378440141677856,
      "logps/rejected": -0.9682254791259766,
      "loss": 0.8995,
      "odds_ratio_loss": 0.6620376110076904,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.04189220070838928,
      "rewards/margins": 0.006519075483083725,
      "rewards/rejected": -0.04841126874089241,
      "sft_loss": 0.8378440141677856,
      "step": 330
    },
    {
      "epoch": 0.8158368326334733,
      "grad_norm": 2.4806806819218794,
      "learning_rate": 4.235827841157748e-06,
      "logits/chosen": 0.01970214769244194,
      "logits/rejected": 0.11670324951410294,
      "logps/chosen": -0.8856766819953918,
      "logps/rejected": -1.0817759037017822,
      "loss": 0.8834,
      "odds_ratio_loss": 0.6194185018539429,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.04428383335471153,
      "rewards/margins": 0.009804959408938885,
      "rewards/rejected": -0.054088789969682693,
      "sft_loss": 0.8856766819953918,
      "step": 340
    },
    {
      "epoch": 0.8398320335932813,
      "grad_norm": 1.5265892877639438,
      "learning_rate": 4.1892374709742186e-06,
      "logits/chosen": -0.7483745813369751,
      "logits/rejected": -0.42045336961746216,
      "logps/chosen": -0.7948485016822815,
      "logps/rejected": -0.9918915033340454,
      "loss": 0.9474,
      "odds_ratio_loss": 0.5842909812927246,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.03974242880940437,
      "rewards/margins": 0.009852146729826927,
      "rewards/rejected": -0.04959457367658615,
      "sft_loss": 0.7948485016822815,
      "step": 350
    },
    {
      "epoch": 0.8638272345530894,
      "grad_norm": 2.1051154185205543,
      "learning_rate": 4.141541572065762e-06,
      "logits/chosen": 0.41192498803138733,
      "logits/rejected": 0.5341157913208008,
      "logps/chosen": -0.7971394658088684,
      "logps/rejected": -0.9216561317443848,
      "loss": 0.8881,
      "odds_ratio_loss": 0.69920814037323,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.03985697776079178,
      "rewards/margins": 0.0062258280813694,
      "rewards/rejected": -0.04608280584216118,
      "sft_loss": 0.7971394658088684,
      "step": 360
    },
    {
      "epoch": 0.8878224355128974,
      "grad_norm": 2.049071087536336,
      "learning_rate": 4.092771359218462e-06,
      "logits/chosen": 0.2649831771850586,
      "logits/rejected": 0.45568495988845825,
      "logps/chosen": -0.8466150164604187,
      "logps/rejected": -1.0025365352630615,
      "loss": 0.9065,
      "odds_ratio_loss": 0.629971444606781,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.042330749332904816,
      "rewards/margins": 0.007796071469783783,
      "rewards/rejected": -0.0501268208026886,
      "sft_loss": 0.8466150164604187,
      "step": 370
    },
    {
      "epoch": 0.9118176364727054,
      "grad_norm": 3.597524104140319,
      "learning_rate": 4.04295875030778e-06,
      "logits/chosen": -0.18752217292785645,
      "logits/rejected": 0.15378537774085999,
      "logps/chosen": -0.8704308271408081,
      "logps/rejected": -0.9513336420059204,
      "loss": 0.9014,
      "odds_ratio_loss": 0.6948253512382507,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.043521542102098465,
      "rewards/margins": 0.004045139066874981,
      "rewards/rejected": -0.04756668210029602,
      "sft_loss": 0.8704308271408081,
      "step": 380
    },
    {
      "epoch": 0.9358128374325135,
      "grad_norm": 3.1405630532603395,
      "learning_rate": 3.992136345409765e-06,
      "logits/chosen": -0.1735876053571701,
      "logits/rejected": -0.20124337077140808,
      "logps/chosen": -0.9253339767456055,
      "logps/rejected": -1.0305973291397095,
      "loss": 0.9111,
      "odds_ratio_loss": 0.6636070013046265,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.04626670479774475,
      "rewards/margins": 0.005263164173811674,
      "rewards/rejected": -0.051529865711927414,
      "sft_loss": 0.9253339767456055,
      "step": 390
    },
    {
      "epoch": 0.9598080383923215,
      "grad_norm": 2.4716790122788983,
      "learning_rate": 3.940337405465786e-06,
      "logits/chosen": 0.26361703872680664,
      "logits/rejected": 0.44345617294311523,
      "logps/chosen": -0.8355854153633118,
      "logps/rejected": -1.0225704908370972,
      "loss": 0.9062,
      "odds_ratio_loss": 0.6545855402946472,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.04177927225828171,
      "rewards/margins": 0.009349259547889233,
      "rewards/rejected": -0.05112852901220322,
      "sft_loss": 0.8355854153633118,
      "step": 400
    },
    {
      "epoch": 0.9838032393521295,
      "grad_norm": 2.3985102639359406,
      "learning_rate": 3.887595830514775e-06,
      "logits/chosen": 0.21671700477600098,
      "logits/rejected": 0.29912179708480835,
      "logps/chosen": -0.809670090675354,
      "logps/rejected": -1.0107569694519043,
      "loss": 0.9029,
      "odds_ratio_loss": 0.6326887011528015,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.0404835119843483,
      "rewards/margins": 0.010054344311356544,
      "rewards/rejected": -0.05053785443305969,
      "sft_loss": 0.809670090675354,
      "step": 410
    },
    {
      "epoch": 1.0077984403119375,
      "grad_norm": 1.6971594247197401,
      "learning_rate": 3.833946137507195e-06,
      "logits/chosen": 0.4990086555480957,
      "logits/rejected": 0.616361141204834,
      "logps/chosen": -0.8005359768867493,
      "logps/rejected": -0.9603840708732605,
      "loss": 0.8398,
      "odds_ratio_loss": 0.6354148387908936,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.040026795119047165,
      "rewards/margins": 0.007992411032319069,
      "rewards/rejected": -0.04801920801401138,
      "sft_loss": 0.8005359768867493,
      "step": 420
    },
    {
      "epoch": 1.0317936412717457,
      "grad_norm": 2.2002987962167904,
      "learning_rate": 3.779423437715274e-06,
      "logits/chosen": 0.7601526975631714,
      "logits/rejected": 0.8180352449417114,
      "logps/chosen": -0.6671024560928345,
      "logps/rejected": -0.9577730298042297,
      "loss": 0.7742,
      "odds_ratio_loss": 0.5807942152023315,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.03335512429475784,
      "rewards/margins": 0.014533529989421368,
      "rewards/rejected": -0.047888655215501785,
      "sft_loss": 0.6671024560928345,
      "step": 430
    },
    {
      "epoch": 1.0557888422315538,
      "grad_norm": 1.5148819350515028,
      "learning_rate": 3.7240634137542864e-06,
      "logits/chosen": 0.19566980004310608,
      "logits/rejected": 0.3528198003768921,
      "logps/chosen": -0.6874720454216003,
      "logps/rejected": -1.0558958053588867,
      "loss": 0.7663,
      "odds_ratio_loss": 0.48211669921875,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.034373603761196136,
      "rewards/margins": 0.01842118799686432,
      "rewards/rejected": -0.052794791758060455,
      "sft_loss": 0.6874720454216003,
      "step": 440
    },
    {
      "epoch": 1.0797840431913617,
      "grad_norm": 1.6130353172110996,
      "learning_rate": 3.6679022962299054e-06,
      "logits/chosen": 0.8750432133674622,
      "logits/rejected": 0.8553866147994995,
      "logps/chosen": -0.7515122890472412,
      "logps/rejected": -0.9563247561454773,
      "loss": 0.7745,
      "odds_ratio_loss": 0.5920617580413818,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.037575613707304,
      "rewards/margins": 0.010240620002150536,
      "rewards/rejected": -0.047816235572099686,
      "sft_loss": 0.7515122890472412,
      "step": 450
    },
    {
      "epoch": 1.1037792441511698,
      "grad_norm": 1.8444047185661667,
      "learning_rate": 3.6109768400269336e-06,
      "logits/chosen": 0.21664266288280487,
      "logits/rejected": 0.3455556333065033,
      "logps/chosen": -0.7820109128952026,
      "logps/rejected": -1.1722263097763062,
      "loss": 0.7949,
      "odds_ratio_loss": 0.5249099731445312,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.03910055011510849,
      "rewards/margins": 0.019510772079229355,
      "rewards/rejected": -0.05861131474375725,
      "sft_loss": 0.7820109128952026,
      "step": 460
    },
    {
      "epoch": 1.127774445110978,
      "grad_norm": 1.923809039800638,
      "learning_rate": 3.5533243002549044e-06,
      "logits/chosen": -0.051299355924129486,
      "logits/rejected": 0.12599964439868927,
      "logps/chosen": -0.6766480803489685,
      "logps/rejected": -0.9556339979171753,
      "loss": 0.769,
      "odds_ratio_loss": 0.5771059989929199,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.03383240848779678,
      "rewards/margins": 0.013949294574558735,
      "rewards/rejected": -0.047781698405742645,
      "sft_loss": 0.6766480803489685,
      "step": 470
    },
    {
      "epoch": 1.1517696460707858,
      "grad_norm": 2.0416324249302593,
      "learning_rate": 3.4949824078663214e-06,
      "logits/chosen": 0.3260158598423004,
      "logits/rejected": 0.4627075791358948,
      "logps/chosen": -0.6955934762954712,
      "logps/rejected": -1.0405316352844238,
      "loss": 0.7744,
      "odds_ratio_loss": 0.5207543969154358,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.03477967530488968,
      "rewards/margins": 0.017246905714273453,
      "rewards/rejected": -0.05202658101916313,
      "sft_loss": 0.6955934762954712,
      "step": 480
    },
    {
      "epoch": 1.175764847030594,
      "grad_norm": 2.159701142475688,
      "learning_rate": 3.4359893449634713e-06,
      "logits/chosen": 0.10285909473896027,
      "logits/rejected": 0.18586108088493347,
      "logps/chosen": -0.7835036516189575,
      "logps/rejected": -0.9662873148918152,
      "loss": 0.7699,
      "odds_ratio_loss": 0.6257883310317993,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.03917517885565758,
      "rewards/margins": 0.009139184840023518,
      "rewards/rejected": -0.04831436648964882,
      "sft_loss": 0.7835036516189575,
      "step": 490
    },
    {
      "epoch": 1.1997600479904018,
      "grad_norm": 1.905386181833648,
      "learning_rate": 3.3763837198099807e-06,
      "logits/chosen": 0.2618166208267212,
      "logits/rejected": 0.403994083404541,
      "logps/chosen": -0.7472913861274719,
      "logps/rejected": -0.9723391532897949,
      "loss": 0.8034,
      "odds_ratio_loss": 0.5758217573165894,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.03736456483602524,
      "rewards/margins": 0.011252395808696747,
      "rewards/rejected": -0.048616960644721985,
      "sft_loss": 0.7472913861274719,
      "step": 500
    },
    {
      "epoch": 1.22375524895021,
      "grad_norm": 1.8483335773730425,
      "learning_rate": 3.3162045415634793e-06,
      "logits/chosen": -0.06936601549386978,
      "logits/rejected": 0.15932008624076843,
      "logps/chosen": -0.7298214435577393,
      "logps/rejected": -0.989848792552948,
      "loss": 0.764,
      "odds_ratio_loss": 0.5586143136024475,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.036491066217422485,
      "rewards/margins": 0.013001373037695885,
      "rewards/rejected": -0.04949244111776352,
      "sft_loss": 0.7298214435577393,
      "step": 510
    },
    {
      "epoch": 1.247750449910018,
      "grad_norm": 1.4105189905656275,
      "learning_rate": 3.255491194745878e-06,
      "logits/chosen": -0.0699717178940773,
      "logits/rejected": 0.11926586925983429,
      "logps/chosen": -0.7712666988372803,
      "logps/rejected": -1.0007984638214111,
      "loss": 0.7514,
      "odds_ratio_loss": 0.576269805431366,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.03856333717703819,
      "rewards/margins": 0.011476586572825909,
      "rewards/rejected": -0.050039924681186676,
      "sft_loss": 0.7712666988372803,
      "step": 520
    },
    {
      "epoch": 1.2717456508698262,
      "grad_norm": 1.5086406745902339,
      "learning_rate": 3.1942834134680123e-06,
      "logits/chosen": -0.4110763669013977,
      "logits/rejected": -0.197097510099411,
      "logps/chosen": -0.7337836027145386,
      "logps/rejected": -1.0581499338150024,
      "loss": 0.747,
      "odds_ratio_loss": 0.5731949806213379,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.03668918460607529,
      "rewards/margins": 0.016218315809965134,
      "rewards/rejected": -0.05290750414133072,
      "sft_loss": 0.7337836027145386,
      "step": 530
    },
    {
      "epoch": 1.295740851829634,
      "grad_norm": 2.007767969966132,
      "learning_rate": 3.13262125542547e-06,
      "logits/chosen": 0.24464428424835205,
      "logits/rejected": 0.42607539892196655,
      "logps/chosen": -0.8008230328559875,
      "logps/rejected": -1.019913911819458,
      "loss": 0.7839,
      "odds_ratio_loss": 0.5772299766540527,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.04004114866256714,
      "rewards/margins": 0.010954543016850948,
      "rewards/rejected": -0.05099569633603096,
      "sft_loss": 0.8008230328559875,
      "step": 540
    },
    {
      "epoch": 1.3197360527894422,
      "grad_norm": 2.031522996603775,
      "learning_rate": 3.0705450756826707e-06,
      "logits/chosen": -0.6761570572853088,
      "logits/rejected": -0.5336428880691528,
      "logps/chosen": -0.7791737914085388,
      "logps/rejected": -0.9758432507514954,
      "loss": 0.7734,
      "odds_ratio_loss": 0.5955380201339722,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.03895869478583336,
      "rewards/margins": 0.009833470918238163,
      "rewards/rejected": -0.04879216477274895,
      "sft_loss": 0.7791737914085388,
      "step": 550
    },
    {
      "epoch": 1.34373125374925,
      "grad_norm": 1.8127230145286217,
      "learning_rate": 3.00809550026231e-06,
      "logits/chosen": 0.7122937440872192,
      "logits/rejected": 0.8374090194702148,
      "logps/chosen": -0.7448546290397644,
      "logps/rejected": -1.0183660984039307,
      "loss": 0.7313,
      "odds_ratio_loss": 0.5605376362800598,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.03724273294210434,
      "rewards/margins": 0.01367556769400835,
      "rewards/rejected": -0.050918303430080414,
      "sft_loss": 0.7448546290397644,
      "step": 560
    },
    {
      "epoch": 1.3677264547090582,
      "grad_norm": 1.6102410365866324,
      "learning_rate": 2.9453133995574955e-06,
      "logits/chosen": 0.1695878505706787,
      "logits/rejected": 0.34987810254096985,
      "logps/chosen": -0.7041548490524292,
      "logps/rejected": -1.1295292377471924,
      "loss": 0.7529,
      "odds_ratio_loss": 0.5541011095046997,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.03520774096250534,
      "rewards/margins": 0.02126871421933174,
      "rewards/rejected": -0.05647646263241768,
      "sft_loss": 0.7041548490524292,
      "step": 570
    },
    {
      "epoch": 1.3917216556688663,
      "grad_norm": 2.0516481147792964,
      "learning_rate": 2.8822398615839337e-06,
      "logits/chosen": -0.15236589312553406,
      "logits/rejected": 0.005555987358093262,
      "logps/chosen": -0.7019264698028564,
      "logps/rejected": -0.9463084936141968,
      "loss": 0.7377,
      "odds_ratio_loss": 0.5546727180480957,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.03509632498025894,
      "rewards/margins": 0.012219103053212166,
      "rewards/rejected": -0.04731542617082596,
      "sft_loss": 0.7019264698028564,
      "step": 580
    },
    {
      "epoch": 1.4157168566286742,
      "grad_norm": 2.5703275268486463,
      "learning_rate": 2.8189161650897045e-06,
      "logits/chosen": 0.09915417432785034,
      "logits/rejected": 0.2876579761505127,
      "logps/chosen": -0.7416352033615112,
      "logps/rejected": -0.9542354345321655,
      "loss": 0.7748,
      "odds_ratio_loss": 0.5765627026557922,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.0370817631483078,
      "rewards/margins": 0.010630009695887566,
      "rewards/rejected": -0.04771176725625992,
      "sft_loss": 0.7416352033615112,
      "step": 590
    },
    {
      "epoch": 1.4397120575884823,
      "grad_norm": 1.6574957139548097,
      "learning_rate": 2.7553837525402095e-06,
      "logits/chosen": 0.14950448274612427,
      "logits/rejected": 0.14670611917972565,
      "logps/chosen": -0.7459922432899475,
      "logps/rejected": -0.9438718557357788,
      "loss": 0.764,
      "odds_ratio_loss": 0.6029990911483765,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.037299610674381256,
      "rewards/margins": 0.009893985465168953,
      "rewards/rejected": -0.04719359427690506,
      "sft_loss": 0.7459922432899475,
      "step": 600
    },
    {
      "epoch": 1.4637072585482904,
      "grad_norm": 1.5955732799355493,
      "learning_rate": 2.691684202995966e-06,
      "logits/chosen": 0.43530672788619995,
      "logits/rejected": 0.4994083344936371,
      "logps/chosen": -0.8142836689949036,
      "logps/rejected": -0.9706009030342102,
      "loss": 0.7559,
      "odds_ratio_loss": 0.7006958723068237,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.04071418568491936,
      "rewards/margins": 0.007815859280526638,
      "rewards/rejected": -0.04853004962205887,
      "sft_loss": 0.8142836689949036,
      "step": 610
    },
    {
      "epoch": 1.4877024595080983,
      "grad_norm": 1.9589861397245603,
      "learning_rate": 2.6278592049010204e-06,
      "logits/chosen": -0.19675548374652863,
      "logits/rejected": -0.004504656884819269,
      "logps/chosen": -0.7537368535995483,
      "logps/rejected": -1.0135046243667603,
      "loss": 0.7741,
      "odds_ratio_loss": 0.5691729187965393,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.03768684342503548,
      "rewards/margins": 0.012988388538360596,
      "rewards/rejected": -0.050675224512815475,
      "sft_loss": 0.7537368535995483,
      "step": 620
    },
    {
      "epoch": 1.5116976604679064,
      "grad_norm": 1.7255875955000524,
      "learning_rate": 2.5639505287997584e-06,
      "logits/chosen": 0.3145737051963806,
      "logits/rejected": 0.47394928336143494,
      "logps/chosen": -0.7314926385879517,
      "logps/rejected": -1.001952886581421,
      "loss": 0.7829,
      "odds_ratio_loss": 0.5629433393478394,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.03657463565468788,
      "rewards/margins": 0.013523015193641186,
      "rewards/rejected": -0.050097644329071045,
      "sft_loss": 0.7314926385879517,
      "step": 630
    },
    {
      "epoch": 1.5356928614277146,
      "grad_norm": 2.504847023988975,
      "learning_rate": 2.5e-06,
      "logits/chosen": 0.2320265769958496,
      "logits/rejected": 0.3284027874469757,
      "logps/chosen": -0.7656562924385071,
      "logps/rejected": -1.076923131942749,
      "loss": 0.7503,
      "odds_ratio_loss": 0.584337592124939,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.038282815366983414,
      "rewards/margins": 0.015563338994979858,
      "rewards/rejected": -0.053846150636672974,
      "sft_loss": 0.7656562924385071,
      "step": 640
    },
    {
      "epoch": 1.5596880623875224,
      "grad_norm": 1.4394266237384084,
      "learning_rate": 2.436049471200242e-06,
      "logits/chosen": -0.5206400156021118,
      "logits/rejected": -0.38631540536880493,
      "logps/chosen": -0.8094362020492554,
      "logps/rejected": -0.9923938512802124,
      "loss": 0.7752,
      "odds_ratio_loss": 0.5967071056365967,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.04047181457281113,
      "rewards/margins": 0.00914788618683815,
      "rewards/rejected": -0.04961969703435898,
      "sft_loss": 0.8094362020492554,
      "step": 650
    },
    {
      "epoch": 1.5836832633473306,
      "grad_norm": 1.7625452374002906,
      "learning_rate": 2.3721407950989804e-06,
      "logits/chosen": -0.24351301789283752,
      "logits/rejected": -0.07003232091665268,
      "logps/chosen": -0.6876959800720215,
      "logps/rejected": -0.9035342335700989,
      "loss": 0.7734,
      "odds_ratio_loss": 0.5917103290557861,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.034384798258543015,
      "rewards/margins": 0.010791914537549019,
      "rewards/rejected": -0.045176707208156586,
      "sft_loss": 0.6876959800720215,
      "step": 660
    },
    {
      "epoch": 1.6076784643071385,
      "grad_norm": 1.6046093499190943,
      "learning_rate": 2.3083157970040344e-06,
      "logits/chosen": 0.5633162260055542,
      "logits/rejected": 0.6462755799293518,
      "logps/chosen": -0.7524802684783936,
      "logps/rejected": -1.0558850765228271,
      "loss": 0.7563,
      "odds_ratio_loss": 0.552274227142334,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.03762401267886162,
      "rewards/margins": 0.015170246362686157,
      "rewards/rejected": -0.05279426649212837,
      "sft_loss": 0.7524802684783936,
      "step": 670
    },
    {
      "epoch": 1.6316736652669466,
      "grad_norm": 2.117352018263469,
      "learning_rate": 2.2446162474597913e-06,
      "logits/chosen": 0.43944865465164185,
      "logits/rejected": 0.5002392530441284,
      "logps/chosen": -0.7501770257949829,
      "logps/rejected": -0.9691005945205688,
      "loss": 0.7699,
      "odds_ratio_loss": 0.5791727304458618,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.037508852779865265,
      "rewards/margins": 0.010946177877485752,
      "rewards/rejected": -0.04845503345131874,
      "sft_loss": 0.7501770257949829,
      "step": 680
    },
    {
      "epoch": 1.6556688662267547,
      "grad_norm": 1.6685249776962552,
      "learning_rate": 2.1810838349102963e-06,
      "logits/chosen": 0.16153453290462494,
      "logits/rejected": 0.20878514647483826,
      "logps/chosen": -0.7516240477561951,
      "logps/rejected": -1.0250643491744995,
      "loss": 0.7666,
      "odds_ratio_loss": 0.5872852206230164,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.03758120536804199,
      "rewards/margins": 0.013672016561031342,
      "rewards/rejected": -0.051253218203783035,
      "sft_loss": 0.7516240477561951,
      "step": 690
    },
    {
      "epoch": 1.6796640671865628,
      "grad_norm": 2.782782057649718,
      "learning_rate": 2.117760138416067e-06,
      "logits/chosen": 0.24376201629638672,
      "logits/rejected": 0.44258540868759155,
      "logps/chosen": -0.6985687017440796,
      "logps/rejected": -1.0050299167633057,
      "loss": 0.7614,
      "odds_ratio_loss": 0.543103814125061,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.03492843732237816,
      "rewards/margins": 0.015323063358664513,
      "rewards/rejected": -0.05025150254368782,
      "sft_loss": 0.6985687017440796,
      "step": 700
    },
    {
      "epoch": 1.7036592681463707,
      "grad_norm": 1.5369658154698735,
      "learning_rate": 2.0546866004425053e-06,
      "logits/chosen": 0.3964254558086395,
      "logits/rejected": 0.4900701642036438,
      "logps/chosen": -0.7590494155883789,
      "logps/rejected": -1.2440413236618042,
      "loss": 0.7652,
      "odds_ratio_loss": 0.5372438430786133,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.037952471524477005,
      "rewards/margins": 0.024249596521258354,
      "rewards/rejected": -0.06220207363367081,
      "sft_loss": 0.7590494155883789,
      "step": 710
    },
    {
      "epoch": 1.7276544691061788,
      "grad_norm": 1.9970193945029362,
      "learning_rate": 1.9919044997376906e-06,
      "logits/chosen": 0.6031176447868347,
      "logits/rejected": 0.7783833742141724,
      "logps/chosen": -0.7290822267532349,
      "logps/rejected": -1.021554946899414,
      "loss": 0.7176,
      "odds_ratio_loss": 0.557815432548523,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.03645411133766174,
      "rewards/margins": 0.014623639173805714,
      "rewards/rejected": -0.051077745854854584,
      "sft_loss": 0.7290822267532349,
      "step": 720
    },
    {
      "epoch": 1.7516496700659867,
      "grad_norm": 2.558147455560064,
      "learning_rate": 1.9294549243173306e-06,
      "logits/chosen": -0.027294237166643143,
      "logits/rejected": 0.11035363376140594,
      "logps/chosen": -0.7765438556671143,
      "logps/rejected": -1.0300321578979492,
      "loss": 0.7771,
      "odds_ratio_loss": 0.5954040884971619,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.03882719203829765,
      "rewards/margins": 0.012674416415393353,
      "rewards/rejected": -0.05150160938501358,
      "sft_loss": 0.7765438556671143,
      "step": 730
    },
    {
      "epoch": 1.7756448710257948,
      "grad_norm": 2.346615273317464,
      "learning_rate": 1.8673787445745298e-06,
      "logits/chosen": -0.449845552444458,
      "logits/rejected": -0.3746832311153412,
      "logps/chosen": -0.7114017605781555,
      "logps/rejected": -0.928491473197937,
      "loss": 0.7699,
      "odds_ratio_loss": 0.5795110464096069,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.035570088773965836,
      "rewards/margins": 0.010854486376047134,
      "rewards/rejected": -0.04642457515001297,
      "sft_loss": 0.7114017605781555,
      "step": 740
    },
    {
      "epoch": 1.799640071985603,
      "grad_norm": 1.995371230537378,
      "learning_rate": 1.805716586531988e-06,
      "logits/chosen": -0.13443303108215332,
      "logits/rejected": 0.014731263741850853,
      "logps/chosen": -0.8079891204833984,
      "logps/rejected": -1.0810317993164062,
      "loss": 0.7825,
      "odds_ratio_loss": 0.6112096309661865,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.0403994545340538,
      "rewards/margins": 0.013652140274643898,
      "rewards/rejected": -0.05405158922076225,
      "sft_loss": 0.8079891204833984,
      "step": 750
    },
    {
      "epoch": 1.823635272945411,
      "grad_norm": 1.8742057389590454,
      "learning_rate": 1.7445088052541218e-06,
      "logits/chosen": 0.046121031045913696,
      "logits/rejected": 0.1955467015504837,
      "logps/chosen": -0.7093559503555298,
      "logps/rejected": -1.0484099388122559,
      "loss": 0.7617,
      "odds_ratio_loss": 0.5657014846801758,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.03546779602766037,
      "rewards/margins": 0.016952697187662125,
      "rewards/rejected": -0.05242049694061279,
      "sft_loss": 0.7093559503555298,
      "step": 760
    },
    {
      "epoch": 1.847630473905219,
      "grad_norm": 1.2680203881504901,
      "learning_rate": 1.6837954584365217e-06,
      "logits/chosen": 0.4459083080291748,
      "logits/rejected": 0.5636454224586487,
      "logps/chosen": -0.7526987195014954,
      "logps/rejected": -1.009804606437683,
      "loss": 0.7871,
      "odds_ratio_loss": 0.5556772947311401,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.03763493150472641,
      "rewards/margins": 0.012855296023190022,
      "rewards/rejected": -0.050490230321884155,
      "sft_loss": 0.7526987195014954,
      "step": 770
    },
    {
      "epoch": 1.8716256748650268,
      "grad_norm": 1.9254646582677224,
      "learning_rate": 1.6236162801900191e-06,
      "logits/chosen": -0.10451897233724594,
      "logits/rejected": 0.3060254156589508,
      "logps/chosen": -0.6585639715194702,
      "logps/rejected": -0.9869001507759094,
      "loss": 0.71,
      "odds_ratio_loss": 0.4942260682582855,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.03292820230126381,
      "rewards/margins": 0.016416804865002632,
      "rewards/rejected": -0.04934500530362129,
      "sft_loss": 0.6585639715194702,
      "step": 780
    },
    {
      "epoch": 1.895620875824835,
      "grad_norm": 1.9904836511656812,
      "learning_rate": 1.5640106550365298e-06,
      "logits/chosen": 0.11656351387500763,
      "logits/rejected": 0.29824742674827576,
      "logps/chosen": -0.7831540703773499,
      "logps/rejected": -1.0284688472747803,
      "loss": 0.7758,
      "odds_ratio_loss": 0.5839165449142456,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.03915770351886749,
      "rewards/margins": 0.01226573996245861,
      "rewards/rejected": -0.051423441618680954,
      "sft_loss": 0.7831540703773499,
      "step": 790
    },
    {
      "epoch": 1.919616076784643,
      "grad_norm": 1.7061927534288226,
      "learning_rate": 1.5050175921336797e-06,
      "logits/chosen": 0.14354857802391052,
      "logits/rejected": 0.27334246039390564,
      "logps/chosen": -0.7474446892738342,
      "logps/rejected": -0.9480558633804321,
      "loss": 0.7575,
      "odds_ratio_loss": 0.6441240310668945,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.03737223893404007,
      "rewards/margins": 0.010030550882220268,
      "rewards/rejected": -0.04740279167890549,
      "sft_loss": 0.7474446892738342,
      "step": 800
    },
    {
      "epoch": 1.9436112777444512,
      "grad_norm": 2.251879648695612,
      "learning_rate": 1.446675699745097e-06,
      "logits/chosen": 0.25183239579200745,
      "logits/rejected": 0.38326969742774963,
      "logps/chosen": -0.7823570966720581,
      "logps/rejected": -0.9946805238723755,
      "loss": 0.8037,
      "odds_ratio_loss": 0.6080455183982849,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.03911786153912544,
      "rewards/margins": 0.010616169311106205,
      "rewards/rejected": -0.049734026193618774,
      "sft_loss": 0.7823570966720581,
      "step": 810
    },
    {
      "epoch": 1.9676064787042593,
      "grad_norm": 1.9391362449031262,
      "learning_rate": 1.3890231599730674e-06,
      "logits/chosen": 0.31725913286209106,
      "logits/rejected": 0.5106421709060669,
      "logps/chosen": -0.7221857309341431,
      "logps/rejected": -0.9829575419425964,
      "loss": 0.7904,
      "odds_ratio_loss": 0.5538625121116638,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.03610928729176521,
      "rewards/margins": 0.013038587756454945,
      "rewards/rejected": -0.049147870391607285,
      "sft_loss": 0.7221857309341431,
      "step": 820
    },
    {
      "epoch": 1.9916016796640672,
      "grad_norm": 1.5457295502049215,
      "learning_rate": 1.3320977037700952e-06,
      "logits/chosen": 0.8291665315628052,
      "logits/rejected": 1.1122350692749023,
      "logps/chosen": -0.6864774227142334,
      "logps/rejected": -1.0247427225112915,
      "loss": 0.7452,
      "odds_ratio_loss": 0.49447354674339294,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.03432386741042137,
      "rewards/margins": 0.016913266852498055,
      "rewards/rejected": -0.051237136125564575,
      "sft_loss": 0.6864774227142334,
      "step": 830
    },
    {
      "epoch": 2.015596880623875,
      "grad_norm": 1.5016852289986733,
      "learning_rate": 1.2759365862457148e-06,
      "logits/chosen": -0.4956502318382263,
      "logits/rejected": -0.1621031016111374,
      "logps/chosen": -0.7308815717697144,
      "logps/rejected": -0.9828909039497375,
      "loss": 0.7173,
      "odds_ratio_loss": 0.5487710237503052,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.0365440808236599,
      "rewards/margins": 0.012600463815033436,
      "rewards/rejected": -0.049144547432661057,
      "sft_loss": 0.7308815717697144,
      "step": 840
    },
    {
      "epoch": 2.039592081583683,
      "grad_norm": 1.622924065562837,
      "learning_rate": 1.2205765622847273e-06,
      "logits/chosen": -0.12397761642932892,
      "logits/rejected": 0.08023932576179504,
      "logps/chosen": -0.6277745962142944,
      "logps/rejected": -1.0955206155776978,
      "loss": 0.6995,
      "odds_ratio_loss": 0.4475070536136627,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.03138873726129532,
      "rewards/margins": 0.023387301713228226,
      "rewards/rejected": -0.054776035249233246,
      "sft_loss": 0.6277745962142944,
      "step": 850
    },
    {
      "epoch": 2.0635872825434913,
      "grad_norm": 1.4741935497367946,
      "learning_rate": 1.1660538624928062e-06,
      "logits/chosen": -0.3639386296272278,
      "logits/rejected": -0.2011258602142334,
      "logps/chosen": -0.6642920970916748,
      "logps/rejected": -1.0270217657089233,
      "loss": 0.7019,
      "odds_ratio_loss": 0.4971997141838074,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.03321460261940956,
      "rewards/margins": 0.018136484548449516,
      "rewards/rejected": -0.05135108903050423,
      "sft_loss": 0.6642920970916748,
      "step": 860
    },
    {
      "epoch": 2.0875824835032994,
      "grad_norm": 1.7172174730539993,
      "learning_rate": 1.112404169485226e-06,
      "logits/chosen": -0.3923923075199127,
      "logits/rejected": -0.10327514261007309,
      "logps/chosen": -0.5645719766616821,
      "logps/rejected": -1.071115255355835,
      "loss": 0.6681,
      "odds_ratio_loss": 0.42052555084228516,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.028228599578142166,
      "rewards/margins": 0.025327179580926895,
      "rewards/rejected": -0.053555767983198166,
      "sft_loss": 0.5645719766616821,
      "step": 870
    },
    {
      "epoch": 2.1115776844631076,
      "grad_norm": 1.1474314844125568,
      "learning_rate": 1.0596625945342148e-06,
      "logits/chosen": -0.008033117279410362,
      "logits/rejected": 0.16419892013072968,
      "logps/chosen": -0.7100299000740051,
      "logps/rejected": -0.9733055233955383,
      "loss": 0.6813,
      "odds_ratio_loss": 0.5328400731086731,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.03550150245428085,
      "rewards/margins": 0.013163777068257332,
      "rewards/rejected": -0.048665277659893036,
      "sft_loss": 0.7100299000740051,
      "step": 880
    },
    {
      "epoch": 2.1355728854229152,
      "grad_norm": 2.1383619388719515,
      "learning_rate": 1.0078636545902363e-06,
      "logits/chosen": -0.4247666001319885,
      "logits/rejected": -0.17631380259990692,
      "logps/chosen": -0.6582883596420288,
      "logps/rejected": -1.0547147989273071,
      "loss": 0.6895,
      "odds_ratio_loss": 0.47398701310157776,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.0329144187271595,
      "rewards/margins": 0.019821325317025185,
      "rewards/rejected": -0.05273573845624924,
      "sft_loss": 0.6582883596420288,
      "step": 890
    },
    {
      "epoch": 2.1595680863827234,
      "grad_norm": 1.5320300236939732,
      "learning_rate": 9.570412496922198e-07,
      "logits/chosen": -0.27953624725341797,
      "logits/rejected": -0.08715387433767319,
      "logps/chosen": -0.5965186357498169,
      "logps/rejected": -1.154284119606018,
      "loss": 0.6738,
      "odds_ratio_loss": 0.4240815043449402,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.029825935140252113,
      "rewards/margins": 0.02788827195763588,
      "rewards/rejected": -0.05771421268582344,
      "sft_loss": 0.5965186357498169,
      "step": 900
    },
    {
      "epoch": 2.1835632873425315,
      "grad_norm": 1.6204787225170885,
      "learning_rate": 9.07228640781539e-07,
      "logits/chosen": 0.368365079164505,
      "logits/rejected": 0.6101259589195251,
      "logps/chosen": -0.6893322467803955,
      "logps/rejected": -1.0903311967849731,
      "loss": 0.6791,
      "odds_ratio_loss": 0.4818887710571289,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.03446660935878754,
      "rewards/margins": 0.02004995197057724,
      "rewards/rejected": -0.054516565054655075,
      "sft_loss": 0.6893322467803955,
      "step": 910
    },
    {
      "epoch": 2.2075584883023396,
      "grad_norm": 1.290844558254926,
      "learning_rate": 8.584584279342392e-07,
      "logits/chosen": -0.16083380579948425,
      "logits/rejected": -0.10739579051733017,
      "logps/chosen": -0.6938862800598145,
      "logps/rejected": -0.9513536691665649,
      "loss": 0.6888,
      "odds_ratio_loss": 0.5428452491760254,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.034694310277700424,
      "rewards/margins": 0.012873371131718159,
      "rewards/rejected": -0.047567687928676605,
      "sft_loss": 0.6938862800598145,
      "step": 920
    },
    {
      "epoch": 2.2315536892621477,
      "grad_norm": 1.5229766148545818,
      "learning_rate": 8.10762529025782e-07,
      "logits/chosen": -0.4659739136695862,
      "logits/rejected": -0.4786594808101654,
      "logps/chosen": -0.6584521532058716,
      "logps/rejected": -0.8917843699455261,
      "loss": 0.65,
      "odds_ratio_loss": 0.5486137866973877,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.03292260691523552,
      "rewards/margins": 0.011666612699627876,
      "rewards/rejected": -0.044589221477508545,
      "sft_loss": 0.6584521532058716,
      "step": 930
    },
    {
      "epoch": 2.255548890221956,
      "grad_norm": 1.7015940933867517,
      "learning_rate": 7.641721588422526e-07,
      "logits/chosen": -0.009342163801193237,
      "logits/rejected": 0.1280032843351364,
      "logps/chosen": -0.6387184262275696,
      "logps/rejected": -1.049140453338623,
      "loss": 0.687,
      "odds_ratio_loss": 0.4773840010166168,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.0319359228014946,
      "rewards/margins": 0.020521100610494614,
      "rewards/rejected": -0.05245702341198921,
      "sft_loss": 0.6387184262275696,
      "step": 940
    },
    {
      "epoch": 2.2795440911817635,
      "grad_norm": 1.4203319350991257,
      "learning_rate": 7.187178086517116e-07,
      "logits/chosen": 0.14468683302402496,
      "logits/rejected": 0.2608656883239746,
      "logps/chosen": -0.6514204144477844,
      "logps/rejected": -1.2591578960418701,
      "loss": 0.6695,
      "odds_ratio_loss": 0.455849826335907,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.03257102146744728,
      "rewards/margins": 0.03038688376545906,
      "rewards/rejected": -0.06295789778232574,
      "sft_loss": 0.6514204144477844,
      "step": 950
    },
    {
      "epoch": 2.3035392921415716,
      "grad_norm": 1.7783791010197938,
      "learning_rate": 6.74429226249049e-07,
      "logits/chosen": 0.09898465871810913,
      "logits/rejected": 0.21373791992664337,
      "logps/chosen": -0.6381307244300842,
      "logps/rejected": -0.9742431640625,
      "loss": 0.6712,
      "odds_ratio_loss": 0.49530988931655884,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.03190653771162033,
      "rewards/margins": 0.016805628314614296,
      "rewards/rejected": -0.04871216416358948,
      "sft_loss": 0.6381307244300842,
      "step": 960
    },
    {
      "epoch": 2.3275344931013797,
      "grad_norm": 1.6090454208525553,
      "learning_rate": 6.313353964874155e-07,
      "logits/chosen": 0.1333683431148529,
      "logits/rejected": 0.3417516350746155,
      "logps/chosen": -0.6887052655220032,
      "logps/rejected": -1.0016798973083496,
      "loss": 0.6673,
      "odds_ratio_loss": 0.5059822797775269,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.03443526476621628,
      "rewards/margins": 0.01564873196184635,
      "rewards/rejected": -0.05008399486541748,
      "sft_loss": 0.6887052655220032,
      "step": 970
    },
    {
      "epoch": 2.351529694061188,
      "grad_norm": 1.6382111002720514,
      "learning_rate": 5.894645223089584e-07,
      "logits/chosen": 0.7236309051513672,
      "logits/rejected": 0.8550646901130676,
      "logps/chosen": -0.6779772639274597,
      "logps/rejected": -1.2183148860931396,
      "loss": 0.6958,
      "odds_ratio_loss": 0.448292076587677,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.033898863941431046,
      "rewards/margins": 0.027016881853342056,
      "rewards/rejected": -0.0609157457947731,
      "sft_loss": 0.6779772639274597,
      "step": 980
    },
    {
      "epoch": 2.375524895020996,
      "grad_norm": 1.680992010239421,
      "learning_rate": 5.48844006287289e-07,
      "logits/chosen": 0.12925365567207336,
      "logits/rejected": 0.3167954981327057,
      "logps/chosen": -0.6692675352096558,
      "logps/rejected": -1.0140740871429443,
      "loss": 0.6691,
      "odds_ratio_loss": 0.4763975143432617,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.033463381230831146,
      "rewards/margins": 0.01724032498896122,
      "rewards/rejected": -0.050703711807727814,
      "sft_loss": 0.6692675352096558,
      "step": 990
    },
    {
      "epoch": 2.3995200959808036,
      "grad_norm": 1.544720546176764,
      "learning_rate": 5.095004326937445e-07,
      "logits/chosen": -0.4231066107749939,
      "logits/rejected": -0.20230142772197723,
      "logps/chosen": -0.6737790107727051,
      "logps/rejected": -1.0810075998306274,
      "loss": 0.6744,
      "odds_ratio_loss": 0.4769432544708252,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.033688947558403015,
      "rewards/margins": 0.02036142908036709,
      "rewards/rejected": -0.05405038595199585,
      "sft_loss": 0.6737790107727051,
      "step": 1000
    },
    {
      "epoch": 2.4235152969406117,
      "grad_norm": 1.7400382431256138,
      "learning_rate": 4.71459550099202e-07,
      "logits/chosen": 0.2943962812423706,
      "logits/rejected": 0.5343393087387085,
      "logps/chosen": -0.6686779856681824,
      "logps/rejected": -1.0820672512054443,
      "loss": 0.7078,
      "odds_ratio_loss": 0.5010559558868408,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.03343390300869942,
      "rewards/margins": 0.020669464021921158,
      "rewards/rejected": -0.054103363305330276,
      "sft_loss": 0.6686779856681824,
      "step": 1010
    },
    {
      "epoch": 2.44751049790042,
      "grad_norm": 1.548219424075948,
      "learning_rate": 4.347462545228134e-07,
      "logits/chosen": 0.13567771017551422,
      "logits/rejected": 0.31968480348587036,
      "logps/chosen": -0.6244124174118042,
      "logps/rejected": -1.05476975440979,
      "loss": 0.6563,
      "odds_ratio_loss": 0.4984089732170105,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.03122062422335148,
      "rewards/margins": 0.021517863497138023,
      "rewards/rejected": -0.052738480269908905,
      "sft_loss": 0.6244124174118042,
      "step": 1020
    },
    {
      "epoch": 2.471505698860228,
      "grad_norm": 1.4610216249122747,
      "learning_rate": 3.9938457313869914e-07,
      "logits/chosen": -0.08544759452342987,
      "logits/rejected": 0.07162941992282867,
      "logps/chosen": -0.7579829096794128,
      "logps/rejected": -1.1255767345428467,
      "loss": 0.6864,
      "odds_ratio_loss": 0.547897458076477,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.03789914771914482,
      "rewards/margins": 0.01837969198822975,
      "rewards/rejected": -0.05627884343266487,
      "sft_loss": 0.7579829096794128,
      "step": 1030
    },
    {
      "epoch": 2.495500899820036,
      "grad_norm": 1.6006797776983446,
      "learning_rate": 3.6539764855126224e-07,
      "logits/chosen": -0.23340921103954315,
      "logits/rejected": -0.1814245879650116,
      "logps/chosen": -0.6439553499221802,
      "logps/rejected": -1.0276587009429932,
      "loss": 0.6617,
      "odds_ratio_loss": 0.5049816370010376,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.03219776228070259,
      "rewards/margins": 0.019185172393918037,
      "rewards/rejected": -0.05138293653726578,
      "sft_loss": 0.6439553499221802,
      "step": 1040
    },
    {
      "epoch": 2.519496100779844,
      "grad_norm": 2.318524117790848,
      "learning_rate": 3.328077236494087e-07,
      "logits/chosen": -0.12850667536258698,
      "logits/rejected": 0.07032374292612076,
      "logps/chosen": -0.5922039747238159,
      "logps/rejected": -1.0730435848236084,
      "loss": 0.6694,
      "odds_ratio_loss": 0.43941235542297363,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.029610196128487587,
      "rewards/margins": 0.024041980504989624,
      "rewards/rejected": -0.05365217477083206,
      "sft_loss": 0.5922039747238159,
      "step": 1050
    },
    {
      "epoch": 2.5434913017396523,
      "grad_norm": 1.8087989245838814,
      "learning_rate": 3.0163612704959486e-07,
      "logits/chosen": -0.6611061692237854,
      "logits/rejected": -0.5293869376182556,
      "logps/chosen": -0.6281863451004028,
      "logps/rejected": -0.9944284558296204,
      "loss": 0.6705,
      "odds_ratio_loss": 0.47698038816452026,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.03140931576490402,
      "rewards/margins": 0.018312102183699608,
      "rewards/rejected": -0.04972142353653908,
      "sft_loss": 0.6281863451004028,
      "step": 1060
    },
    {
      "epoch": 2.56748650269946,
      "grad_norm": 1.5444353690364836,
      "learning_rate": 2.71903259137222e-07,
      "logits/chosen": 0.411745548248291,
      "logits/rejected": 0.4236873686313629,
      "logps/chosen": -0.611006498336792,
      "logps/rejected": -1.0047032833099365,
      "loss": 0.672,
      "odds_ratio_loss": 0.48614612221717834,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.03055032715201378,
      "rewards/margins": 0.019684839993715286,
      "rewards/rejected": -0.050235163420438766,
      "sft_loss": 0.611006498336792,
      "step": 1070
    },
    {
      "epoch": 2.591481703659268,
      "grad_norm": 2.593043127599419,
      "learning_rate": 2.436285787155185e-07,
      "logits/chosen": 0.316955029964447,
      "logits/rejected": 0.47285112738609314,
      "logps/chosen": -0.6786519885063171,
      "logps/rejected": -1.2019875049591064,
      "loss": 0.6881,
      "odds_ratio_loss": 0.4908427298069,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.03393259644508362,
      "rewards/margins": 0.026166772469878197,
      "rewards/rejected": -0.060099370777606964,
      "sft_loss": 0.6786519885063171,
      "step": 1080
    },
    {
      "epoch": 2.6154769046190762,
      "grad_norm": 2.2050381193088207,
      "learning_rate": 2.168305902706383e-07,
      "logits/chosen": -0.4541945457458496,
      "logits/rejected": -0.18702273070812225,
      "logps/chosen": -0.7026795148849487,
      "logps/rejected": -0.962356448173523,
      "loss": 0.6583,
      "odds_ratio_loss": 0.5365189909934998,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.035133976489305496,
      "rewards/margins": 0.012983846478164196,
      "rewards/rejected": -0.04811782017350197,
      "sft_loss": 0.7026795148849487,
      "step": 1090
    },
    {
      "epoch": 2.6394721055788843,
      "grad_norm": 1.6921175899136245,
      "learning_rate": 1.9152683186132476e-07,
      "logits/chosen": -0.4067768156528473,
      "logits/rejected": -0.3039708137512207,
      "logps/chosen": -0.6328436136245728,
      "logps/rejected": -1.12655770778656,
      "loss": 0.6919,
      "odds_ratio_loss": 0.4709090292453766,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.031642183661460876,
      "rewards/margins": 0.024685706943273544,
      "rewards/rejected": -0.05632789060473442,
      "sft_loss": 0.6328436136245728,
      "step": 1100
    },
    {
      "epoch": 2.663467306538692,
      "grad_norm": 1.5594348597838832,
      "learning_rate": 1.6773386364104972e-07,
      "logits/chosen": -0.1575368195772171,
      "logits/rejected": -0.003553843591362238,
      "logps/chosen": -0.6768941879272461,
      "logps/rejected": -1.032041072845459,
      "loss": 0.6913,
      "odds_ratio_loss": 0.50171959400177,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.033844709396362305,
      "rewards/margins": 0.017757344990968704,
      "rewards/rejected": -0.05160205811262131,
      "sft_loss": 0.6768941879272461,
      "step": 1110
    },
    {
      "epoch": 2.6874625074985,
      "grad_norm": 1.2735811398241894,
      "learning_rate": 1.4546725702015096e-07,
      "logits/chosen": 0.004650235176086426,
      "logits/rejected": 0.1661575585603714,
      "logps/chosen": -0.6541981101036072,
      "logps/rejected": -1.1094247102737427,
      "loss": 0.6669,
      "odds_ratio_loss": 0.4492813050746918,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.03270990774035454,
      "rewards/margins": 0.022761326283216476,
      "rewards/rejected": -0.055471230298280716,
      "sft_loss": 0.6541981101036072,
      "step": 1120
    },
    {
      "epoch": 2.7114577084583082,
      "grad_norm": 2.2135398834819715,
      "learning_rate": 1.24741584475056e-07,
      "logits/chosen": -0.07907108962535858,
      "logits/rejected": 0.08474680036306381,
      "logps/chosen": -0.6154497861862183,
      "logps/rejected": -1.0710924863815308,
      "loss": 0.6491,
      "odds_ratio_loss": 0.4509805142879486,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.030772492289543152,
      "rewards/margins": 0.022782133892178535,
      "rewards/rejected": -0.05355461686849594,
      "sft_loss": 0.6154497861862183,
      "step": 1130
    },
    {
      "epoch": 2.7354529094181164,
      "grad_norm": 1.5137426741255027,
      "learning_rate": 1.0557041001126145e-07,
      "logits/chosen": 0.3702402710914612,
      "logits/rejected": 0.6300150156021118,
      "logps/chosen": -0.5984182357788086,
      "logps/rejected": -1.115179419517517,
      "loss": 0.6191,
      "odds_ratio_loss": 0.41762223839759827,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.0299209114164114,
      "rewards/margins": 0.025838062167167664,
      "rewards/rejected": -0.05575897544622421,
      "sft_loss": 0.5984182357788086,
      "step": 1140
    },
    {
      "epoch": 2.7594481103779245,
      "grad_norm": 1.565522436867544,
      "learning_rate": 8.796628028631321e-08,
      "logits/chosen": 0.17880654335021973,
      "logits/rejected": 0.1116660013794899,
      "logps/chosen": -0.6091745495796204,
      "logps/rejected": -1.0210378170013428,
      "loss": 0.6583,
      "odds_ratio_loss": 0.4544963836669922,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.030458729714155197,
      "rewards/margins": 0.02059316076338291,
      "rewards/rejected": -0.05105189234018326,
      "sft_loss": 0.6091745495796204,
      "step": 1150
    },
    {
      "epoch": 2.7834433113377326,
      "grad_norm": 1.604017358081912,
      "learning_rate": 7.19407163985894e-08,
      "logits/chosen": -0.04378344863653183,
      "logits/rejected": 0.18321049213409424,
      "logps/chosen": -0.6626521348953247,
      "logps/rejected": -1.1215763092041016,
      "loss": 0.666,
      "odds_ratio_loss": 0.4741577208042145,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.033132605254650116,
      "rewards/margins": 0.022946210578083992,
      "rewards/rejected": -0.05607881397008896,
      "sft_loss": 0.6626521348953247,
      "step": 1160
    },
    {
      "epoch": 2.8074385122975407,
      "grad_norm": 1.4084206676302562,
      "learning_rate": 5.750420634727083e-08,
      "logits/chosen": -0.45710262656211853,
      "logits/rejected": -0.3050076961517334,
      "logps/chosen": -0.671418309211731,
      "logps/rejected": -1.1854102611541748,
      "loss": 0.6842,
      "odds_ratio_loss": 0.4368383288383484,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.03357091173529625,
      "rewards/margins": 0.02569960430264473,
      "rewards/rejected": -0.05927051231265068,
      "sft_loss": 0.671418309211731,
      "step": 1170
    },
    {
      "epoch": 2.8314337132573484,
      "grad_norm": 1.3507137389822068,
      "learning_rate": 4.4666198168422656e-08,
      "logits/chosen": 0.33376216888427734,
      "logits/rejected": 0.41172194480895996,
      "logps/chosen": -0.6510582566261292,
      "logps/rejected": -1.0800405740737915,
      "loss": 0.6747,
      "odds_ratio_loss": 0.5277644395828247,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.032552916556596756,
      "rewards/margins": 0.021449116989970207,
      "rewards/rejected": -0.054002027958631516,
      "sft_loss": 0.6510582566261292,
      "step": 1180
    },
    {
      "epoch": 2.8554289142171565,
      "grad_norm": 1.6874037821147798,
      "learning_rate": 3.343509375168863e-08,
      "logits/chosen": 0.20301933586597443,
      "logits/rejected": 0.32382094860076904,
      "logps/chosen": -0.6405006647109985,
      "logps/rejected": -1.0241023302078247,
      "loss": 0.6718,
      "odds_ratio_loss": 0.48166948556900024,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.03202503174543381,
      "rewards/margins": 0.019180091097950935,
      "rewards/rejected": -0.051205117255449295,
      "sft_loss": 0.6405006647109985,
      "step": 1190
    },
    {
      "epoch": 2.8794241151769646,
      "grad_norm": 1.6417139708130921,
      "learning_rate": 2.3818243341637293e-08,
      "logits/chosen": -0.3619822859764099,
      "logits/rejected": -0.15361133217811584,
      "logps/chosen": -0.6599988341331482,
      "logps/rejected": -1.098881483078003,
      "loss": 0.6565,
      "odds_ratio_loss": 0.456063449382782,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.03299994021654129,
      "rewards/margins": 0.021944135427474976,
      "rewards/rejected": -0.054944075644016266,
      "sft_loss": 0.6599988341331482,
      "step": 1200
    },
    {
      "epoch": 2.9034193161367727,
      "grad_norm": 1.648932215503252,
      "learning_rate": 1.5821940727361874e-08,
      "logits/chosen": -0.7362561821937561,
      "logits/rejected": -0.4996170997619629,
      "logps/chosen": -0.6824958920478821,
      "logps/rejected": -0.9969790577888489,
      "loss": 0.7067,
      "odds_ratio_loss": 0.5307115316390991,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.034124795347452164,
      "rewards/margins": 0.01572415977716446,
      "rewards/rejected": -0.049848951399326324,
      "sft_loss": 0.6824958920478821,
      "step": 1210
    },
    {
      "epoch": 2.927414517096581,
      "grad_norm": 1.7678674281978446,
      "learning_rate": 9.451419123484573e-09,
      "logits/chosen": -0.15318191051483154,
      "logits/rejected": 0.047946538776159286,
      "logps/chosen": -0.6560810804367065,
      "logps/rejected": -1.0658347606658936,
      "loss": 0.6692,
      "odds_ratio_loss": 0.5046226382255554,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.032804060727357864,
      "rewards/margins": 0.02048768661916256,
      "rewards/rejected": -0.053291745483875275,
      "sft_loss": 0.6560810804367065,
      "step": 1220
    },
    {
      "epoch": 2.9514097180563885,
      "grad_norm": 1.4413325593301094,
      "learning_rate": 4.710847745256209e-09,
      "logits/chosen": 0.12647075951099396,
      "logits/rejected": 0.2795228958129883,
      "logps/chosen": -0.6180914640426636,
      "logps/rejected": -1.0847346782684326,
      "loss": 0.6722,
      "odds_ratio_loss": 0.41623228788375854,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.030904576182365417,
      "rewards/margins": 0.02333216182887554,
      "rewards/rejected": -0.05423673242330551,
      "sft_loss": 0.6180914640426636,
      "step": 1230
    },
    {
      "epoch": 2.9754049190161966,
      "grad_norm": 1.5296676400661524,
      "learning_rate": 1.603329079994942e-09,
      "logits/chosen": -0.3425149619579315,
      "logits/rejected": -0.06856220215559006,
      "logps/chosen": -0.6569226980209351,
      "logps/rejected": -1.1020539999008179,
      "loss": 0.6649,
      "odds_ratio_loss": 0.4642546772956848,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.03284613788127899,
      "rewards/margins": 0.02225656434893608,
      "rewards/rejected": -0.055102698504924774,
      "sft_loss": 0.6569226980209351,
      "step": 1240
    },
    {
      "epoch": 2.994601079784043,
      "step": 1248,
      "total_flos": 132590267662336.0,
      "train_loss": 0.7937506708579186,
      "train_runtime": 49781.9259,
      "train_samples_per_second": 1.205,
      "train_steps_per_second": 0.025
    }
  ],
  "logging_steps": 10,
  "max_steps": 1248,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 3,
  "save_steps": 100.0,
  "total_flos": 132590267662336.0,
  "train_batch_size": 1,
  "trial_name": null,
  "trial_params": null
}