{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 3.0,
  "eval_steps": 100,
  "global_step": 1434,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.0,
      "grad_norm": 46.495066480188626,
      "learning_rate": 3.4722222222222217e-09,
      "logits/chosen": -2.7660439014434814,
      "logits/rejected": -2.717564582824707,
      "logps/chosen": -269.8568420410156,
      "logps/rejected": -360.52459716796875,
      "loss": 0.6931,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 1
    },
    {
      "epoch": 0.02,
      "grad_norm": 45.48250079754631,
      "learning_rate": 3.472222222222222e-08,
      "logits/chosen": -2.592761516571045,
      "logits/rejected": -2.5630030632019043,
      "logps/chosen": -264.7732238769531,
      "logps/rejected": -251.50889587402344,
      "loss": 0.6929,
      "rewards/accuracies": 0.3958333432674408,
      "rewards/chosen": 0.0012306140270084143,
      "rewards/margins": -0.0009704786934889853,
      "rewards/rejected": 0.0022010933607816696,
      "step": 10
    },
    {
      "epoch": 0.04,
      "grad_norm": 41.93789081750966,
      "learning_rate": 6.944444444444444e-08,
      "logits/chosen": -2.6552841663360596,
      "logits/rejected": -2.6074695587158203,
      "logps/chosen": -281.4732360839844,
      "logps/rejected": -296.79010009765625,
      "loss": 0.6925,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": 0.004006093833595514,
      "rewards/margins": 0.003165980102494359,
      "rewards/rejected": 0.0008401140803471208,
      "step": 20
    },
    {
      "epoch": 0.06,
      "grad_norm": 41.15398691411152,
      "learning_rate": 1.0416666666666667e-07,
      "logits/chosen": -2.67468523979187,
      "logits/rejected": -2.6028592586517334,
      "logps/chosen": -300.5101013183594,
      "logps/rejected": -263.88922119140625,
      "loss": 0.6904,
      "rewards/accuracies": 0.518750011920929,
      "rewards/chosen": 0.008085623383522034,
      "rewards/margins": 0.00503805186599493,
      "rewards/rejected": 0.003047570353373885,
      "step": 30
    },
    {
      "epoch": 0.08,
      "grad_norm": 39.57195743724939,
      "learning_rate": 1.3888888888888888e-07,
      "logits/chosen": -2.5917325019836426,
      "logits/rejected": -2.559333562850952,
      "logps/chosen": -266.6319274902344,
      "logps/rejected": -244.5043487548828,
      "loss": 0.6826,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": 0.024673160165548325,
      "rewards/margins": 0.030291978269815445,
      "rewards/rejected": -0.005618819035589695,
      "step": 40
    },
    {
      "epoch": 0.1,
      "grad_norm": 38.787652763321745,
      "learning_rate": 1.736111111111111e-07,
      "logits/chosen": -2.5633091926574707,
      "logits/rejected": -2.527255058288574,
      "logps/chosen": -278.7782287597656,
      "logps/rejected": -292.60064697265625,
      "loss": 0.6686,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.0476832240819931,
      "rewards/margins": 0.03851194307208061,
      "rewards/rejected": 0.009171287529170513,
      "step": 50
    },
    {
      "epoch": 0.13,
      "grad_norm": 42.23287032668302,
      "learning_rate": 2.0833333333333333e-07,
      "logits/chosen": -2.6264617443084717,
      "logits/rejected": -2.5678651332855225,
      "logps/chosen": -268.39715576171875,
      "logps/rejected": -296.73492431640625,
      "loss": 0.6517,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": 0.1604687124490738,
      "rewards/margins": 0.09314907342195511,
      "rewards/rejected": 0.06731964647769928,
      "step": 60
    },
    {
      "epoch": 0.15,
      "grad_norm": 37.074057156615645,
      "learning_rate": 2.4305555555555555e-07,
      "logits/chosen": -2.529860734939575,
      "logits/rejected": -2.5335025787353516,
      "logps/chosen": -261.59197998046875,
      "logps/rejected": -255.90951538085938,
      "loss": 0.6244,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": 0.20122408866882324,
      "rewards/margins": 0.160838782787323,
      "rewards/rejected": 0.04038532078266144,
      "step": 70
    },
    {
      "epoch": 0.17,
      "grad_norm": 53.60948399217281,
      "learning_rate": 2.7777777777777776e-07,
      "logits/chosen": -2.6630194187164307,
      "logits/rejected": -2.5986924171447754,
      "logps/chosen": -319.0962829589844,
      "logps/rejected": -265.9037170410156,
      "loss": 0.6056,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": 0.1860823631286621,
      "rewards/margins": 0.2783365547657013,
      "rewards/rejected": -0.09225417673587799,
      "step": 80
    },
    {
      "epoch": 0.19,
      "grad_norm": 39.28479543071564,
      "learning_rate": 3.1249999999999997e-07,
      "logits/chosen": -2.5524282455444336,
      "logits/rejected": -2.505441188812256,
      "logps/chosen": -258.28790283203125,
      "logps/rejected": -260.45709228515625,
      "loss": 0.5784,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": 0.04117143899202347,
      "rewards/margins": 0.4441215991973877,
      "rewards/rejected": -0.40295013785362244,
      "step": 90
    },
    {
      "epoch": 0.21,
      "grad_norm": 37.33085523160704,
      "learning_rate": 3.472222222222222e-07,
      "logits/chosen": -2.5317625999450684,
      "logits/rejected": -2.5069174766540527,
      "logps/chosen": -255.98025512695312,
      "logps/rejected": -263.0767517089844,
      "loss": 0.582,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.21891236305236816,
      "rewards/margins": 0.5213271975517273,
      "rewards/rejected": -0.3024148643016815,
      "step": 100
    },
    {
      "epoch": 0.21,
      "eval_logits/chosen": -2.581188917160034,
      "eval_logits/rejected": -2.5431487560272217,
      "eval_logps/chosen": -254.23861694335938,
      "eval_logps/rejected": -263.28759765625,
      "eval_loss": 0.5878116488456726,
      "eval_rewards/accuracies": 0.71875,
      "eval_rewards/chosen": 0.41774246096611023,
      "eval_rewards/margins": 0.44876235723495483,
      "eval_rewards/rejected": -0.031019899994134903,
      "eval_runtime": 97.2965,
      "eval_samples_per_second": 20.556,
      "eval_steps_per_second": 0.329,
      "step": 100
    },
    {
      "epoch": 0.23,
      "grad_norm": 38.91387415080278,
      "learning_rate": 3.819444444444444e-07,
      "logits/chosen": -2.580745220184326,
      "logits/rejected": -2.495126724243164,
      "logps/chosen": -298.351806640625,
      "logps/rejected": -262.37762451171875,
      "loss": 0.5835,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": 0.31141430139541626,
      "rewards/margins": 0.5385011434555054,
      "rewards/rejected": -0.2270868569612503,
      "step": 110
    },
    {
      "epoch": 0.25,
      "grad_norm": 38.5197424217738,
      "learning_rate": 4.1666666666666667e-07,
      "logits/chosen": -2.473219633102417,
      "logits/rejected": -2.4464592933654785,
      "logps/chosen": -286.4003601074219,
      "logps/rejected": -275.6720886230469,
      "loss": 0.5583,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.2616604268550873,
      "rewards/margins": 0.5291147828102112,
      "rewards/rejected": -0.7907751798629761,
      "step": 120
    },
    {
      "epoch": 0.27,
      "grad_norm": 37.759243136528056,
      "learning_rate": 4.513888888888889e-07,
      "logits/chosen": -2.463078498840332,
      "logits/rejected": -2.4377052783966064,
      "logps/chosen": -277.6775817871094,
      "logps/rejected": -277.7251281738281,
      "loss": 0.53,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": 0.2162761688232422,
      "rewards/margins": 0.6807397603988647,
      "rewards/rejected": -0.46446362137794495,
      "step": 130
    },
    {
      "epoch": 0.29,
      "grad_norm": 43.5485570360565,
      "learning_rate": 4.861111111111111e-07,
      "logits/chosen": -2.488098621368408,
      "logits/rejected": -2.4522647857666016,
      "logps/chosen": -313.66253662109375,
      "logps/rejected": -313.8337707519531,
      "loss": 0.5305,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": 0.1151454821228981,
      "rewards/margins": 0.7729904651641846,
      "rewards/rejected": -0.6578450202941895,
      "step": 140
    },
    {
      "epoch": 0.31,
      "grad_norm": 42.70051446601528,
      "learning_rate": 4.999733114418725e-07,
      "logits/chosen": -2.4509975910186768,
      "logits/rejected": -2.384775161743164,
      "logps/chosen": -286.3575744628906,
      "logps/rejected": -292.4871520996094,
      "loss": 0.5401,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.15666823089122772,
      "rewards/margins": 0.6572391390800476,
      "rewards/rejected": -0.8139073252677917,
      "step": 150
    },
    {
      "epoch": 0.33,
      "grad_norm": 53.334691841946395,
      "learning_rate": 4.998102353328799e-07,
      "logits/chosen": -2.4482672214508057,
      "logits/rejected": -2.358886957168579,
      "logps/chosen": -269.8271789550781,
      "logps/rejected": -270.34710693359375,
      "loss": 0.57,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.16204610466957092,
      "rewards/margins": 0.808205246925354,
      "rewards/rejected": -0.6461590528488159,
      "step": 160
    },
    {
      "epoch": 0.36,
      "grad_norm": 44.14727072303912,
      "learning_rate": 4.994990066883491e-07,
      "logits/chosen": -2.3647940158843994,
      "logits/rejected": -2.2960338592529297,
      "logps/chosen": -259.77276611328125,
      "logps/rejected": -255.1217498779297,
      "loss": 0.5382,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.5218764543533325,
      "rewards/margins": 0.6779727339744568,
      "rewards/rejected": -1.1998491287231445,
      "step": 170
    },
    {
      "epoch": 0.38,
      "grad_norm": 86.93750039599435,
      "learning_rate": 4.990398100856366e-07,
      "logits/chosen": -2.234858989715576,
      "logits/rejected": -2.1829886436462402,
      "logps/chosen": -262.09600830078125,
      "logps/rejected": -243.8761444091797,
      "loss": 0.529,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.2868492603302002,
      "rewards/margins": 0.5284022092819214,
      "rewards/rejected": -0.8152514696121216,
      "step": 180
    },
    {
      "epoch": 0.4,
      "grad_norm": 42.637015477998524,
      "learning_rate": 4.984329178560219e-07,
      "logits/chosen": -2.275543689727783,
      "logits/rejected": -2.2290995121002197,
      "logps/chosen": -273.13580322265625,
      "logps/rejected": -277.8387756347656,
      "loss": 0.5193,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.09464643895626068,
      "rewards/margins": 0.8095524907112122,
      "rewards/rejected": -0.9041990041732788,
      "step": 190
    },
    {
      "epoch": 0.42,
      "grad_norm": 224.49804573493958,
      "learning_rate": 4.976786899231985e-07,
      "logits/chosen": -2.2373015880584717,
      "logits/rejected": -2.2193970680236816,
      "logps/chosen": -264.38330078125,
      "logps/rejected": -271.14654541015625,
      "loss": 0.558,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.22433169186115265,
      "rewards/margins": 0.7305465936660767,
      "rewards/rejected": -0.9548781514167786,
      "step": 200
    },
    {
      "epoch": 0.42,
      "eval_logits/chosen": -2.38926362991333,
      "eval_logits/rejected": -2.3397982120513916,
      "eval_logps/chosen": -261.4733581542969,
      "eval_logps/rejected": -280.4190673828125,
      "eval_loss": 0.5195851922035217,
      "eval_rewards/accuracies": 0.77734375,
      "eval_rewards/chosen": 0.05600578337907791,
      "eval_rewards/margins": 0.9436004757881165,
      "eval_rewards/rejected": -0.8875946998596191,
      "eval_runtime": 96.7112,
      "eval_samples_per_second": 20.68,
      "eval_steps_per_second": 0.331,
      "step": 200
    },
    {
      "epoch": 0.44,
      "grad_norm": 57.978840386902434,
      "learning_rate": 4.967775735898179e-07,
      "logits/chosen": -2.3195013999938965,
      "logits/rejected": -2.2525343894958496,
      "logps/chosen": -293.5995178222656,
      "logps/rejected": -273.7899475097656,
      "loss": 0.5321,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.1953679621219635,
      "rewards/margins": 0.9317470788955688,
      "rewards/rejected": -1.12711501121521,
      "step": 210
    },
    {
      "epoch": 0.46,
      "grad_norm": 45.042317188060295,
      "learning_rate": 4.957301032722118e-07,
      "logits/chosen": -2.320981502532959,
      "logits/rejected": -2.2780816555023193,
      "logps/chosen": -275.7085266113281,
      "logps/rejected": -272.6601257324219,
      "loss": 0.5272,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.5165706872940063,
      "rewards/margins": 0.8332468271255493,
      "rewards/rejected": -1.3498175144195557,
      "step": 220
    },
    {
      "epoch": 0.48,
      "grad_norm": 42.64386380595179,
      "learning_rate": 4.945369001834514e-07,
      "logits/chosen": -2.335122585296631,
      "logits/rejected": -2.2802178859710693,
      "logps/chosen": -251.1619415283203,
      "logps/rejected": -268.8333740234375,
      "loss": 0.5242,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.1148248165845871,
      "rewards/margins": 0.939952552318573,
      "rewards/rejected": -1.0547773838043213,
      "step": 230
    },
    {
      "epoch": 0.5,
      "grad_norm": 43.7138056070312,
      "learning_rate": 4.931986719649298e-07,
      "logits/chosen": -2.372040271759033,
      "logits/rejected": -2.3250112533569336,
      "logps/chosen": -267.7872619628906,
      "logps/rejected": -270.08612060546875,
      "loss": 0.4886,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.20435211062431335,
      "rewards/margins": 0.8465341329574585,
      "rewards/rejected": -1.0508863925933838,
      "step": 240
    },
    {
      "epoch": 0.52,
      "grad_norm": 46.84966459893474,
      "learning_rate": 4.91716212266689e-07,
      "logits/chosen": -2.2988550662994385,
      "logits/rejected": -2.2003023624420166,
      "logps/chosen": -298.1326904296875,
      "logps/rejected": -279.72857666015625,
      "loss": 0.4983,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.0028904906939715147,
      "rewards/margins": 1.0131902694702148,
      "rewards/rejected": -1.0102999210357666,
      "step": 250
    },
    {
      "epoch": 0.54,
      "grad_norm": 45.60631693718858,
      "learning_rate": 4.900904002767367e-07,
      "logits/chosen": -2.359170913696289,
      "logits/rejected": -2.298555850982666,
      "logps/chosen": -298.26898193359375,
      "logps/rejected": -277.553955078125,
      "loss": 0.4979,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.1978708803653717,
      "rewards/margins": 0.9808057546615601,
      "rewards/rejected": -1.178676724433899,
      "step": 260
    },
    {
      "epoch": 0.56,
      "grad_norm": 43.06547616205831,
      "learning_rate": 4.883222001996351e-07,
      "logits/chosen": -2.3876845836639404,
      "logits/rejected": -2.324982166290283,
      "logps/chosen": -285.72674560546875,
      "logps/rejected": -283.54534912109375,
      "loss": 0.529,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.43653813004493713,
      "rewards/margins": 0.8585146069526672,
      "rewards/rejected": -1.2950527667999268,
      "step": 270
    },
    {
      "epoch": 0.59,
      "grad_norm": 47.726430451771456,
      "learning_rate": 4.864126606846696e-07,
      "logits/chosen": -2.3547611236572266,
      "logits/rejected": -2.261406898498535,
      "logps/chosen": -271.0657043457031,
      "logps/rejected": -269.5400390625,
      "loss": 0.5339,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.4694308340549469,
      "rewards/margins": 0.8982539176940918,
      "rewards/rejected": -1.3676847219467163,
      "step": 280
    },
    {
      "epoch": 0.61,
      "grad_norm": 43.598738742809324,
      "learning_rate": 4.843629142039366e-07,
      "logits/chosen": -2.398907423019409,
      "logits/rejected": -2.3398590087890625,
      "logps/chosen": -260.32208251953125,
      "logps/rejected": -267.5470886230469,
      "loss": 0.5116,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.4617377817630768,
      "rewards/margins": 0.8178228139877319,
      "rewards/rejected": -1.2795606851577759,
      "step": 290
    },
    {
      "epoch": 0.63,
      "grad_norm": 53.461838586248135,
      "learning_rate": 4.821741763807186e-07,
      "logits/chosen": -2.358552932739258,
      "logits/rejected": -2.2632975578308105,
      "logps/chosen": -306.07708740234375,
      "logps/rejected": -284.6916198730469,
      "loss": 0.4914,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.254052072763443,
      "rewards/margins": 0.9752548336982727,
      "rewards/rejected": -1.2293068170547485,
      "step": 300
    },
    {
      "epoch": 0.63,
      "eval_logits/chosen": -2.3652639389038086,
      "eval_logits/rejected": -2.3039121627807617,
      "eval_logps/chosen": -264.29364013671875,
      "eval_logps/rejected": -286.6200866699219,
      "eval_loss": 0.5109529495239258,
      "eval_rewards/accuracies": 0.765625,
      "eval_rewards/chosen": -0.08500880002975464,
      "eval_rewards/margins": 1.1126374006271362,
      "eval_rewards/rejected": -1.197646141052246,
      "eval_runtime": 96.7466,
      "eval_samples_per_second": 20.673,
      "eval_steps_per_second": 0.331,
      "step": 300
    },
    {
      "epoch": 0.65,
      "grad_norm": 39.546683435658494,
      "learning_rate": 4.798477452685468e-07,
      "logits/chosen": -2.268113613128662,
      "logits/rejected": -2.2365353107452393,
      "logps/chosen": -298.82098388671875,
      "logps/rejected": -280.2306823730469,
      "loss": 0.4878,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.17425382137298584,
      "rewards/margins": 0.9114526510238647,
      "rewards/rejected": -1.0857064723968506,
      "step": 310
    },
    {
      "epoch": 0.67,
      "grad_norm": 46.66091222824393,
      "learning_rate": 4.773850005813776e-07,
      "logits/chosen": -2.2866523265838623,
      "logits/rejected": -2.221771240234375,
      "logps/chosen": -301.2940368652344,
      "logps/rejected": -271.4169921875,
      "loss": 0.4739,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.3811998963356018,
      "rewards/margins": 0.9484872817993164,
      "rewards/rejected": -1.329687237739563,
      "step": 320
    },
    {
      "epoch": 0.69,
      "grad_norm": 38.86653009667017,
      "learning_rate": 4.747874028753375e-07,
      "logits/chosen": -2.278787136077881,
      "logits/rejected": -2.1933655738830566,
      "logps/chosen": -265.38922119140625,
      "logps/rejected": -282.16143798828125,
      "loss": 0.4954,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.1589515209197998,
      "rewards/margins": 0.9741891026496887,
      "rewards/rejected": -1.1331405639648438,
      "step": 330
    },
    {
      "epoch": 0.71,
      "grad_norm": 52.563625371137256,
      "learning_rate": 4.720564926825267e-07,
      "logits/chosen": -2.261157751083374,
      "logits/rejected": -2.1779565811157227,
      "logps/chosen": -300.0699462890625,
      "logps/rejected": -313.32012939453125,
      "loss": 0.4961,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.5037744641304016,
      "rewards/margins": 1.1601417064666748,
      "rewards/rejected": -1.663915991783142,
      "step": 340
    },
    {
      "epoch": 0.73,
      "grad_norm": 44.99037304721281,
      "learning_rate": 4.6919388959739e-07,
      "logits/chosen": -2.2558467388153076,
      "logits/rejected": -2.163471221923828,
      "logps/chosen": -268.26397705078125,
      "logps/rejected": -272.03729248046875,
      "loss": 0.465,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.5026377439498901,
      "rewards/margins": 1.063787579536438,
      "rewards/rejected": -1.566425085067749,
      "step": 350
    },
    {
      "epoch": 0.75,
      "grad_norm": 62.16268030811922,
      "learning_rate": 4.662012913161997e-07,
      "logits/chosen": -2.2976787090301514,
      "logits/rejected": -2.1752614974975586,
      "logps/chosen": -306.94708251953125,
      "logps/rejected": -288.148193359375,
      "loss": 0.52,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.212753027677536,
      "rewards/margins": 1.1294101476669312,
      "rewards/rejected": -1.3421632051467896,
      "step": 360
    },
    {
      "epoch": 0.77,
      "grad_norm": 48.68068839220641,
      "learning_rate": 4.6308047263021925e-07,
      "logits/chosen": -2.221646785736084,
      "logits/rejected": -2.1589419841766357,
      "logps/chosen": -286.72344970703125,
      "logps/rejected": -267.8009033203125,
      "loss": 0.5138,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.5695220232009888,
      "rewards/margins": 0.9088203310966492,
      "rewards/rejected": -1.4783422946929932,
      "step": 370
    },
    {
      "epoch": 0.79,
      "grad_norm": 40.16847273705918,
      "learning_rate": 4.5983328437314523e-07,
      "logits/chosen": -2.1364545822143555,
      "logits/rejected": -2.1142001152038574,
      "logps/chosen": -257.01513671875,
      "logps/rejected": -296.47222900390625,
      "loss": 0.4949,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.8765251040458679,
      "rewards/margins": 1.0112650394439697,
      "rewards/rejected": -1.8877900838851929,
      "step": 380
    },
    {
      "epoch": 0.82,
      "grad_norm": 51.78049342719275,
      "learning_rate": 4.5646165232345103e-07,
      "logits/chosen": -2.223437547683716,
      "logits/rejected": -2.1807045936584473,
      "logps/chosen": -312.63714599609375,
      "logps/rejected": -308.8570861816406,
      "loss": 0.4921,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.3500111699104309,
      "rewards/margins": 0.8943204879760742,
      "rewards/rejected": -1.2443315982818604,
      "step": 390
    },
    {
      "epoch": 0.84,
      "grad_norm": 42.79568140865266,
      "learning_rate": 4.529675760622843e-07,
      "logits/chosen": -2.2350916862487793,
      "logits/rejected": -2.148932695388794,
      "logps/chosen": -276.3843078613281,
      "logps/rejected": -295.5137939453125,
      "loss": 0.4922,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.29245343804359436,
      "rewards/margins": 1.1326130628585815,
      "rewards/rejected": -1.425066590309143,
      "step": 400
    },
    {
      "epoch": 0.84,
      "eval_logits/chosen": -2.3144989013671875,
      "eval_logits/rejected": -2.2569639682769775,
      "eval_logps/chosen": -263.2853698730469,
      "eval_logps/rejected": -285.4248352050781,
      "eval_loss": 0.5095034837722778,
      "eval_rewards/accuracies": 0.78515625,
      "eval_rewards/chosen": -0.0345957949757576,
      "eval_rewards/margins": 1.1032863855361938,
      "eval_rewards/rejected": -1.137882113456726,
      "eval_runtime": 96.7428,
      "eval_samples_per_second": 20.673,
      "eval_steps_per_second": 0.331,
      "step": 400
    },
    {
      "epoch": 0.86,
      "grad_norm": 41.17913551343322,
      "learning_rate": 4.493531277875948e-07,
      "logits/chosen": -2.300753116607666,
      "logits/rejected": -2.2138209342956543,
      "logps/chosen": -300.85986328125,
      "logps/rejected": -293.10089111328125,
      "loss": 0.4939,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.09711817651987076,
      "rewards/margins": 1.1170685291290283,
      "rewards/rejected": -1.214186668395996,
      "step": 410
    },
    {
      "epoch": 0.88,
      "grad_norm": 44.7817002923167,
      "learning_rate": 4.456204510851956e-07,
      "logits/chosen": -2.296912670135498,
      "logits/rejected": -2.2389960289001465,
      "logps/chosen": -304.0657043457031,
      "logps/rejected": -303.19512939453125,
      "loss": 0.4853,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.34959596395492554,
      "rewards/margins": 1.0870481729507446,
      "rewards/rejected": -1.4366440773010254,
      "step": 420
    },
    {
      "epoch": 0.9,
      "grad_norm": 42.837722286044084,
      "learning_rate": 4.4177175965748804e-07,
      "logits/chosen": -2.190886974334717,
      "logits/rejected": -2.1311051845550537,
      "logps/chosen": -289.19403076171875,
      "logps/rejected": -291.84967041015625,
      "loss": 0.4704,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.5077947378158569,
      "rewards/margins": 1.1956888437271118,
      "rewards/rejected": -1.7034835815429688,
      "step": 430
    },
    {
      "epoch": 0.92,
      "grad_norm": 50.46876260755674,
      "learning_rate": 4.378093360106022e-07,
      "logits/chosen": -2.275932788848877,
      "logits/rejected": -2.206188678741455,
      "logps/chosen": -291.9374694824219,
      "logps/rejected": -296.0220947265625,
      "loss": 0.4947,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.3800189793109894,
      "rewards/margins": 0.9654985666275024,
      "rewards/rejected": -1.3455175161361694,
      "step": 440
    },
    {
      "epoch": 0.94,
      "grad_norm": 49.16582597289157,
      "learning_rate": 4.337355301007335e-07,
      "logits/chosen": -2.2473514080047607,
      "logits/rejected": -2.212566375732422,
      "logps/chosen": -304.7669372558594,
      "logps/rejected": -337.439453125,
      "loss": 0.4809,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.5541377663612366,
      "rewards/margins": 0.9641104936599731,
      "rewards/rejected": -1.5182483196258545,
      "step": 450
    },
    {
      "epoch": 0.96,
      "grad_norm": 45.063096434401295,
      "learning_rate": 4.2955275794047627e-07,
      "logits/chosen": -2.255167007446289,
      "logits/rejected": -2.2048747539520264,
      "logps/chosen": -277.2271423339844,
      "logps/rejected": -268.5289001464844,
      "loss": 0.4941,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.26313039660453796,
      "rewards/margins": 1.0289232730865479,
      "rewards/rejected": -1.2920535802841187,
      "step": 460
    },
    {
      "epoch": 0.98,
      "grad_norm": 44.06520192137465,
      "learning_rate": 4.252635001659837e-07,
      "logits/chosen": -2.2488787174224854,
      "logits/rejected": -2.1796398162841797,
      "logps/chosen": -284.37713623046875,
      "logps/rejected": -309.53546142578125,
      "loss": 0.4733,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.28392332792282104,
      "rewards/margins": 1.0752943754196167,
      "rewards/rejected": -1.3592177629470825,
      "step": 470
    },
    {
      "epoch": 1.0,
      "grad_norm": 20.757199114043438,
      "learning_rate": 4.2087030056579986e-07,
      "logits/chosen": -2.2137668132781982,
      "logits/rejected": -2.1583967208862305,
      "logps/chosen": -291.67376708984375,
      "logps/rejected": -288.5511169433594,
      "loss": 0.417,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.14164048433303833,
      "rewards/margins": 1.205784559249878,
      "rewards/rejected": -1.347425103187561,
      "step": 480
    },
    {
      "epoch": 1.03,
      "grad_norm": 30.758787856873905,
      "learning_rate": 4.163757645722403e-07,
      "logits/chosen": -2.273852825164795,
      "logits/rejected": -2.1564927101135254,
      "logps/chosen": -292.22467041015625,
      "logps/rejected": -292.77349853515625,
      "loss": 0.1895,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": 0.2319210022687912,
      "rewards/margins": 2.500307559967041,
      "rewards/rejected": -2.2683863639831543,
      "step": 490
    },
    {
      "epoch": 1.05,
      "grad_norm": 30.542382675770316,
      "learning_rate": 4.117825577162134e-07,
      "logits/chosen": -2.2761318683624268,
      "logits/rejected": -2.1567749977111816,
      "logps/chosen": -279.54486083984375,
      "logps/rejected": -309.9883728027344,
      "loss": 0.1908,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.061248529702425,
      "rewards/margins": 2.6951680183410645,
      "rewards/rejected": -2.6339192390441895,
      "step": 500
    },
    {
      "epoch": 1.05,
      "eval_logits/chosen": -2.2441868782043457,
      "eval_logits/rejected": -2.165961980819702,
      "eval_logps/chosen": -269.8425598144531,
      "eval_logps/rejected": -300.6473693847656,
      "eval_loss": 0.5178976058959961,
      "eval_rewards/accuracies": 0.78515625,
      "eval_rewards/chosen": -0.36245518922805786,
      "eval_rewards/margins": 1.5365545749664307,
      "eval_rewards/rejected": -1.8990097045898438,
      "eval_runtime": 96.6241,
      "eval_samples_per_second": 20.699,
      "eval_steps_per_second": 0.331,
      "step": 500
    },
    {
      "epoch": 1.07,
      "grad_norm": 18.232913631558603,
      "learning_rate": 4.070934040463998e-07,
      "logits/chosen": -2.1585049629211426,
      "logits/rejected": -2.0768818855285645,
      "logps/chosen": -279.84527587890625,
      "logps/rejected": -321.41375732421875,
      "loss": 0.1751,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.1159655898809433,
      "rewards/margins": 2.8791768550872803,
      "rewards/rejected": -2.763211250305176,
      "step": 510
    },
    {
      "epoch": 1.09,
      "grad_norm": 28.960170029042352,
      "learning_rate": 4.023110845137273e-07,
      "logits/chosen": -2.215406894683838,
      "logits/rejected": -2.0539145469665527,
      "logps/chosen": -276.207763671875,
      "logps/rejected": -310.82513427734375,
      "loss": 0.1755,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -0.07207924127578735,
      "rewards/margins": 2.9146130084991455,
      "rewards/rejected": -2.986691951751709,
      "step": 520
    },
    {
      "epoch": 1.11,
      "grad_norm": 35.158803563669196,
      "learning_rate": 3.974384353220992e-07,
      "logits/chosen": -2.187908172607422,
      "logits/rejected": -2.049764394760132,
      "logps/chosen": -256.0184326171875,
      "logps/rejected": -290.10821533203125,
      "loss": 0.167,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -0.13029596209526062,
      "rewards/margins": 2.844452381134033,
      "rewards/rejected": -2.974748134613037,
      "step": 530
    },
    {
      "epoch": 1.13,
      "grad_norm": 38.75397385029711,
      "learning_rate": 3.9247834624635404e-07,
      "logits/chosen": -2.1720824241638184,
      "logits/rejected": -2.047842502593994,
      "logps/chosen": -267.35888671875,
      "logps/rejected": -331.7341003417969,
      "loss": 0.1939,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -0.3224119246006012,
      "rewards/margins": 3.103816509246826,
      "rewards/rejected": -3.4262282848358154,
      "step": 540
    },
    {
      "epoch": 1.15,
      "grad_norm": 21.6571458368692,
      "learning_rate": 3.8743375891845556e-07,
      "logits/chosen": -2.2194952964782715,
      "logits/rejected": -2.088972806930542,
      "logps/chosen": -301.56414794921875,
      "logps/rejected": -329.8552551269531,
      "loss": 0.1541,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -0.44850125908851624,
      "rewards/margins": 2.9901504516601562,
      "rewards/rejected": -3.4386515617370605,
      "step": 550
    },
    {
      "epoch": 1.17,
      "grad_norm": 41.5671178301598,
      "learning_rate": 3.823076650829267e-07,
      "logits/chosen": -2.198901891708374,
      "logits/rejected": -2.0278093814849854,
      "logps/chosen": -261.13189697265625,
      "logps/rejected": -295.24127197265625,
      "loss": 0.1609,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.0586366169154644,
      "rewards/margins": 2.883596897125244,
      "rewards/rejected": -2.9422335624694824,
      "step": 560
    },
    {
      "epoch": 1.19,
      "grad_norm": 22.403810834900185,
      "learning_rate": 3.7710310482256523e-07,
      "logits/chosen": -2.10243558883667,
      "logits/rejected": -1.9939591884613037,
      "logps/chosen": -281.2176208496094,
      "logps/rejected": -338.7304382324219,
      "loss": 0.1581,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -0.1340499371290207,
      "rewards/margins": 3.009289503097534,
      "rewards/rejected": -3.1433396339416504,
      "step": 570
    },
    {
      "epoch": 1.21,
      "grad_norm": 41.02190574631486,
      "learning_rate": 3.718231647554911e-07,
      "logits/chosen": -2.1406311988830566,
      "logits/rejected": -1.9555755853652954,
      "logps/chosen": -292.01739501953125,
      "logps/rejected": -329.15045166015625,
      "loss": 0.1561,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -0.6274134516716003,
      "rewards/margins": 3.1701507568359375,
      "rewards/rejected": -3.7975640296936035,
      "step": 580
    },
    {
      "epoch": 1.23,
      "grad_norm": 24.855938354465067,
      "learning_rate": 3.664709762045961e-07,
      "logits/chosen": -2.1577706336975098,
      "logits/rejected": -2.070456027984619,
      "logps/chosen": -284.9651794433594,
      "logps/rejected": -321.62994384765625,
      "loss": 0.1622,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -0.7565070390701294,
      "rewards/margins": 2.949742078781128,
      "rewards/rejected": -3.706249237060547,
      "step": 590
    },
    {
      "epoch": 1.26,
      "grad_norm": 34.33138656502216,
      "learning_rate": 3.610497133404795e-07,
      "logits/chosen": -2.268091917037964,
      "logits/rejected": -2.1013598442077637,
      "logps/chosen": -285.04327392578125,
      "logps/rejected": -315.2426452636719,
      "loss": 0.1675,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -0.31151503324508667,
      "rewards/margins": 3.0706770420074463,
      "rewards/rejected": -3.3821918964385986,
      "step": 600
    },
    {
      "epoch": 1.26,
      "eval_logits/chosen": -2.2219574451446533,
      "eval_logits/rejected": -2.1248531341552734,
      "eval_logps/chosen": -287.2300109863281,
      "eval_logps/rejected": -324.08123779296875,
      "eval_loss": 0.5377076268196106,
      "eval_rewards/accuracies": 0.80078125,
      "eval_rewards/chosen": -1.2318270206451416,
      "eval_rewards/margins": 1.8388763666152954,
      "eval_rewards/rejected": -3.0707035064697266,
      "eval_runtime": 96.6082,
      "eval_samples_per_second": 20.702,
      "eval_steps_per_second": 0.331,
      "step": 600
    },
    {
      "epoch": 1.28,
      "grad_norm": 24.42926394663865,
      "learning_rate": 3.555625912989747e-07,
      "logits/chosen": -2.2549402713775635,
      "logits/rejected": -2.0435824394226074,
      "logps/chosen": -345.69683837890625,
      "logps/rejected": -355.43841552734375,
      "loss": 0.1641,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -0.57981938123703,
      "rewards/margins": 3.4938979148864746,
      "rewards/rejected": -4.0737175941467285,
      "step": 610
    },
    {
      "epoch": 1.3,
      "grad_norm": 40.76003061063018,
      "learning_rate": 3.500128642743793e-07,
      "logits/chosen": -1.9793628454208374,
      "logits/rejected": -1.8766018152236938,
      "logps/chosen": -275.6058654785156,
      "logps/rejected": -325.64532470703125,
      "loss": 0.1676,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -0.4727633595466614,
      "rewards/margins": 3.388253688812256,
      "rewards/rejected": -3.8610172271728516,
      "step": 620
    },
    {
      "epoch": 1.32,
      "grad_norm": 27.005984360116628,
      "learning_rate": 3.4440382358952115e-07,
      "logits/chosen": -2.0091536045074463,
      "logits/rejected": -1.8107984066009521,
      "logps/chosen": -286.50762939453125,
      "logps/rejected": -327.90350341796875,
      "loss": 0.1604,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -0.3183102309703827,
      "rewards/margins": 3.1906726360321045,
      "rewards/rejected": -3.5089828968048096,
      "step": 630
    },
    {
      "epoch": 1.34,
      "grad_norm": 23.606941374377882,
      "learning_rate": 3.387387957438061e-07,
      "logits/chosen": -1.9466636180877686,
      "logits/rejected": -1.7844781875610352,
      "logps/chosen": -283.4999084472656,
      "logps/rejected": -322.8622131347656,
      "loss": 0.1551,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -0.5999014973640442,
      "rewards/margins": 3.3343536853790283,
      "rewards/rejected": -3.9342548847198486,
      "step": 640
    },
    {
      "epoch": 1.36,
      "grad_norm": 35.62460431561092,
      "learning_rate": 3.33021140440403e-07,
      "logits/chosen": -2.041600465774536,
      "logits/rejected": -1.8977054357528687,
      "logps/chosen": -282.59033203125,
      "logps/rejected": -359.6671447753906,
      "loss": 0.1662,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -0.652147650718689,
      "rewards/margins": 3.4888508319854736,
      "rewards/rejected": -4.140998363494873,
      "step": 650
    },
    {
      "epoch": 1.38,
      "grad_norm": 40.45521004861816,
      "learning_rate": 3.272542485937368e-07,
      "logits/chosen": -2.2044432163238525,
      "logits/rejected": -1.995490312576294,
      "logps/chosen": -281.81591796875,
      "logps/rejected": -305.563720703125,
      "loss": 0.1562,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -0.9047712087631226,
      "rewards/margins": 3.0545411109924316,
      "rewards/rejected": -3.9593119621276855,
      "step": 660
    },
    {
      "epoch": 1.4,
      "grad_norm": 25.83353768539722,
      "learning_rate": 3.214415403184725e-07,
      "logits/chosen": -2.136054515838623,
      "logits/rejected": -2.0022921562194824,
      "logps/chosen": -287.70721435546875,
      "logps/rejected": -347.98626708984375,
      "loss": 0.1527,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -0.8447133898735046,
      "rewards/margins": 3.195286512374878,
      "rewards/rejected": -4.039999961853027,
      "step": 670
    },
    {
      "epoch": 1.42,
      "grad_norm": 33.98889011519535,
      "learning_rate": 3.155864629011798e-07,
      "logits/chosen": -2.006204605102539,
      "logits/rejected": -1.8692758083343506,
      "logps/chosen": -266.548583984375,
      "logps/rejected": -349.1086730957031,
      "loss": 0.1618,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -0.7886885404586792,
      "rewards/margins": 3.2961840629577637,
      "rewards/rejected": -4.084872722625732,
      "step": 680
    },
    {
      "epoch": 1.44,
      "grad_norm": 23.86841822029929,
      "learning_rate": 3.096924887558854e-07,
      "logits/chosen": -2.0170860290527344,
      "logits/rejected": -1.8624064922332764,
      "logps/chosen": -289.9138488769531,
      "logps/rejected": -340.1861877441406,
      "loss": 0.1705,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.879202663898468,
      "rewards/margins": 3.349952220916748,
      "rewards/rejected": -4.22915506362915,
      "step": 690
    },
    {
      "epoch": 1.46,
      "grad_norm": 30.325436164549153,
      "learning_rate": 3.0376311336472157e-07,
      "logits/chosen": -1.9222033023834229,
      "logits/rejected": -1.7393171787261963,
      "logps/chosen": -308.6827087402344,
      "logps/rejected": -357.66937255859375,
      "loss": 0.1567,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -1.2114474773406982,
      "rewards/margins": 3.2933521270751953,
      "rewards/rejected": -4.504799842834473,
      "step": 700
    },
    {
      "epoch": 1.46,
      "eval_logits/chosen": -2.045348882675171,
      "eval_logits/rejected": -1.9284720420837402,
      "eval_logps/chosen": -298.781982421875,
      "eval_logps/rejected": -333.33538818359375,
      "eval_loss": 0.5347517132759094,
      "eval_rewards/accuracies": 0.7890625,
      "eval_rewards/chosen": -1.8094260692596436,
      "eval_rewards/margins": 1.7239831686019897,
      "eval_rewards/rejected": -3.5334088802337646,
      "eval_runtime": 96.8349,
      "eval_samples_per_second": 20.654,
      "eval_steps_per_second": 0.33,
      "step": 700
    },
    {
      "epoch": 1.49,
      "grad_norm": 44.30207392759187,
      "learning_rate": 2.9780185320489397e-07,
      "logits/chosen": -1.9471759796142578,
      "logits/rejected": -1.777573585510254,
      "logps/chosen": -307.9242858886719,
      "logps/rejected": -352.77618408203125,
      "loss": 0.1417,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -0.9882961511611938,
      "rewards/margins": 3.599376678466797,
      "rewards/rejected": -4.587672233581543,
      "step": 710
    },
    {
      "epoch": 1.51,
      "grad_norm": 37.177307769645225,
      "learning_rate": 2.9181224366319943e-07,
      "logits/chosen": -1.9124629497528076,
      "logits/rejected": -1.7028827667236328,
      "logps/chosen": -315.4266662597656,
      "logps/rejected": -355.34857177734375,
      "loss": 0.166,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.7210714817047119,
      "rewards/margins": 3.6800003051757812,
      "rewards/rejected": -4.401071071624756,
      "step": 720
    },
    {
      "epoch": 1.53,
      "grad_norm": 39.29672592174553,
      "learning_rate": 2.857978369393279e-07,
      "logits/chosen": -2.243919849395752,
      "logits/rejected": -2.1087582111358643,
      "logps/chosen": -308.04412841796875,
      "logps/rejected": -345.15594482421875,
      "loss": 0.152,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.5387176871299744,
      "rewards/margins": 3.3322176933288574,
      "rewards/rejected": -3.8709354400634766,
      "step": 730
    },
    {
      "epoch": 1.55,
      "grad_norm": 24.15001856800991,
      "learning_rate": 2.797621999391938e-07,
      "logits/chosen": -2.3955087661743164,
      "logits/rejected": -2.282515048980713,
      "logps/chosen": -296.63604736328125,
      "logps/rejected": -331.5033264160156,
      "loss": 0.1584,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -1.0399105548858643,
      "rewards/margins": 3.2611820697784424,
      "rewards/rejected": -4.301093101501465,
      "step": 740
    },
    {
      "epoch": 1.57,
      "grad_norm": 28.800655620715126,
      "learning_rate": 2.7370891215954565e-07,
      "logits/chosen": -2.291198253631592,
      "logits/rejected": -2.1874325275421143,
      "logps/chosen": -294.86126708984375,
      "logps/rejected": -342.1693420410156,
      "loss": 0.1327,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -0.5225303769111633,
      "rewards/margins": 3.4790916442871094,
      "rewards/rejected": -4.001622200012207,
      "step": 750
    },
    {
      "epoch": 1.59,
      "grad_norm": 36.52736639617503,
      "learning_rate": 2.676415635651091e-07,
      "logits/chosen": -2.2511188983917236,
      "logits/rejected": -2.1171295642852783,
      "logps/chosen": -314.3142395019531,
      "logps/rejected": -401.13018798828125,
      "loss": 0.1373,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -0.4879869818687439,
      "rewards/margins": 4.032652854919434,
      "rewards/rejected": -4.520639896392822,
      "step": 760
    },
    {
      "epoch": 1.61,
      "grad_norm": 22.81336311327227,
      "learning_rate": 2.615637524595207e-07,
      "logits/chosen": -2.2899999618530273,
      "logits/rejected": -2.117185115814209,
      "logps/chosen": -290.29815673828125,
      "logps/rejected": -304.9264831542969,
      "loss": 0.137,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.9585368037223816,
      "rewards/margins": 3.114755153656006,
      "rewards/rejected": -4.0732927322387695,
      "step": 770
    },
    {
      "epoch": 1.63,
      "grad_norm": 25.499517087179957,
      "learning_rate": 2.55479083351317e-07,
      "logits/chosen": -2.2676925659179688,
      "logits/rejected": -2.083047866821289,
      "logps/chosen": -301.6076965332031,
      "logps/rejected": -364.6853942871094,
      "loss": 0.1536,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -0.8435381054878235,
      "rewards/margins": 3.4508233070373535,
      "rewards/rejected": -4.294361114501953,
      "step": 780
    },
    {
      "epoch": 1.65,
      "grad_norm": 27.418147190397534,
      "learning_rate": 2.4939116481624407e-07,
      "logits/chosen": -2.220470666885376,
      "logits/rejected": -1.959284782409668,
      "logps/chosen": -299.3353576660156,
      "logps/rejected": -319.30914306640625,
      "loss": 0.1568,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -0.8999794125556946,
      "rewards/margins": 3.3365378379821777,
      "rewards/rejected": -4.236516952514648,
      "step": 790
    },
    {
      "epoch": 1.67,
      "grad_norm": 22.42449451210148,
      "learning_rate": 2.4330360735715374e-07,
      "logits/chosen": -2.2539217472076416,
      "logits/rejected": -2.0446996688842773,
      "logps/chosen": -333.44366455078125,
      "logps/rejected": -357.18841552734375,
      "loss": 0.1475,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -1.156060814857483,
      "rewards/margins": 3.485567092895508,
      "rewards/rejected": -4.641627311706543,
      "step": 800
    },
    {
      "epoch": 1.67,
      "eval_logits/chosen": -2.2409238815307617,
      "eval_logits/rejected": -2.1201870441436768,
      "eval_logps/chosen": -296.3533020019531,
      "eval_logps/rejected": -332.4950866699219,
      "eval_loss": 0.538158118724823,
      "eval_rewards/accuracies": 0.80078125,
      "eval_rewards/chosen": -1.6879926919937134,
      "eval_rewards/margins": 1.8034026622772217,
      "eval_rewards/rejected": -3.4913952350616455,
      "eval_runtime": 96.753,
      "eval_samples_per_second": 20.671,
      "eval_steps_per_second": 0.331,
      "step": 800
    },
    {
      "epoch": 1.69,
      "grad_norm": 27.648844978471217,
      "learning_rate": 2.3722002126275822e-07,
      "logits/chosen": -2.118046998977661,
      "logits/rejected": -1.9478477239608765,
      "logps/chosen": -295.06268310546875,
      "logps/rejected": -350.56829833984375,
      "loss": 0.1534,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.974108099937439,
      "rewards/margins": 3.4369990825653076,
      "rewards/rejected": -4.411107540130615,
      "step": 810
    },
    {
      "epoch": 1.72,
      "grad_norm": 33.288825778542474,
      "learning_rate": 2.311440144665108e-07,
      "logits/chosen": -2.166576623916626,
      "logits/rejected": -2.005044460296631,
      "logps/chosen": -285.5037841796875,
      "logps/rejected": -354.74578857421875,
      "loss": 0.1616,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -0.7247167229652405,
      "rewards/margins": 3.4520015716552734,
      "rewards/rejected": -4.176717758178711,
      "step": 820
    },
    {
      "epoch": 1.74,
      "grad_norm": 28.54134777409646,
      "learning_rate": 2.2507919040688398e-07,
      "logits/chosen": -2.035219669342041,
      "logits/rejected": -1.9823194742202759,
      "logps/chosen": -302.2669677734375,
      "logps/rejected": -337.7466125488281,
      "loss": 0.1361,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -0.7065524458885193,
      "rewards/margins": 3.6438660621643066,
      "rewards/rejected": -4.350418567657471,
      "step": 830
    },
    {
      "epoch": 1.76,
      "grad_norm": 29.525737237536532,
      "learning_rate": 2.19029145890313e-07,
      "logits/chosen": -2.0496773719787598,
      "logits/rejected": -1.8104021549224854,
      "logps/chosen": -295.2398376464844,
      "logps/rejected": -348.8295593261719,
      "loss": 0.137,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -1.180415391921997,
      "rewards/margins": 4.056873798370361,
      "rewards/rejected": -5.237288951873779,
      "step": 840
    },
    {
      "epoch": 1.78,
      "grad_norm": 29.418652242465264,
      "learning_rate": 2.1299746895807268e-07,
      "logits/chosen": -2.0320029258728027,
      "logits/rejected": -1.8159306049346924,
      "logps/chosen": -298.8084716796875,
      "logps/rejected": -351.4793395996094,
      "loss": 0.1426,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -0.8422020077705383,
      "rewards/margins": 3.6892521381378174,
      "rewards/rejected": -4.531454563140869,
      "step": 850
    },
    {
      "epoch": 1.8,
      "grad_norm": 28.75219392191004,
      "learning_rate": 2.0698773675835246e-07,
      "logits/chosen": -1.9433095455169678,
      "logits/rejected": -1.7687761783599854,
      "logps/chosen": -307.7235107421875,
      "logps/rejected": -342.0555114746094,
      "loss": 0.1486,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.4170432090759277,
      "rewards/margins": 3.423736095428467,
      "rewards/rejected": -4.8407793045043945,
      "step": 860
    },
    {
      "epoch": 1.82,
      "grad_norm": 36.42272712771708,
      "learning_rate": 2.0100351342479216e-07,
      "logits/chosen": -1.9862619638442993,
      "logits/rejected": -1.750312089920044,
      "logps/chosen": -312.4382019042969,
      "logps/rejected": -355.03045654296875,
      "loss": 0.1468,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -1.4696812629699707,
      "rewards/margins": 3.662865400314331,
      "rewards/rejected": -5.132546901702881,
      "step": 870
    },
    {
      "epoch": 1.84,
      "grad_norm": 29.128706338046875,
      "learning_rate": 1.9504834796273545e-07,
      "logits/chosen": -2.0059654712677,
      "logits/rejected": -1.7494417428970337,
      "logps/chosen": -284.04669189453125,
      "logps/rejected": -307.86334228515625,
      "loss": 0.1514,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -1.4539439678192139,
      "rewards/margins": 3.2319369316101074,
      "rewards/rejected": -4.6858811378479,
      "step": 880
    },
    {
      "epoch": 1.86,
      "grad_norm": 54.834861913718775,
      "learning_rate": 1.8912577214445558e-07,
      "logits/chosen": -2.125431537628174,
      "logits/rejected": -1.975856065750122,
      "logps/chosen": -302.9638671875,
      "logps/rejected": -345.03851318359375,
      "loss": 0.1219,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -0.8938841819763184,
      "rewards/margins": 3.579861879348755,
      "rewards/rejected": -4.473746299743652,
      "step": 890
    },
    {
      "epoch": 1.88,
      "grad_norm": 36.4163607482829,
      "learning_rate": 1.8323929841460178e-07,
      "logits/chosen": -2.0952582359313965,
      "logits/rejected": -1.9091037511825562,
      "logps/chosen": -300.1233215332031,
      "logps/rejected": -345.5321960449219,
      "loss": 0.1422,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -1.1558645963668823,
      "rewards/margins": 3.8015289306640625,
      "rewards/rejected": -4.957393169403076,
      "step": 900
    },
    {
      "epoch": 1.88,
      "eval_logits/chosen": -2.198045492172241,
      "eval_logits/rejected": -2.0630321502685547,
      "eval_logps/chosen": -296.03240966796875,
      "eval_logps/rejected": -335.6015625,
      "eval_loss": 0.5517675280570984,
      "eval_rewards/accuracies": 0.78515625,
      "eval_rewards/chosen": -1.67194664478302,
      "eval_rewards/margins": 1.9747719764709473,
      "eval_rewards/rejected": -3.6467185020446777,
      "eval_runtime": 96.8149,
      "eval_samples_per_second": 20.658,
      "eval_steps_per_second": 0.331,
      "step": 900
    },
    {
      "epoch": 1.9,
      "grad_norm": 32.93263141944983,
      "learning_rate": 1.7739241780710745e-07,
      "logits/chosen": -2.067739963531494,
      "logits/rejected": -1.879922866821289,
      "logps/chosen": -285.4407958984375,
      "logps/rejected": -354.6173400878906,
      "loss": 0.1572,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -1.3456696271896362,
      "rewards/margins": 3.62982439994812,
      "rewards/rejected": -4.975494384765625,
      "step": 910
    },
    {
      "epoch": 1.92,
      "grad_norm": 30.163608722340616,
      "learning_rate": 1.7158859787479653e-07,
      "logits/chosen": -1.945054054260254,
      "logits/rejected": -1.7525079250335693,
      "logps/chosen": -271.43670654296875,
      "logps/rejected": -355.9076232910156,
      "loss": 0.1457,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -1.121407151222229,
      "rewards/margins": 3.6710968017578125,
      "rewards/rejected": -4.79250431060791,
      "step": 920
    },
    {
      "epoch": 1.95,
      "grad_norm": 38.64417902485811,
      "learning_rate": 1.6583128063291573e-07,
      "logits/chosen": -2.06476092338562,
      "logits/rejected": -1.8927568197250366,
      "logps/chosen": -288.0918884277344,
      "logps/rejected": -331.8973388671875,
      "loss": 0.1544,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -1.375701904296875,
      "rewards/margins": 3.178281784057617,
      "rewards/rejected": -4.55398416519165,
      "step": 930
    },
    {
      "epoch": 1.97,
      "grad_norm": 33.76607136138911,
      "learning_rate": 1.6012388051781152e-07,
      "logits/chosen": -2.157871961593628,
      "logits/rejected": -1.855956792831421,
      "logps/chosen": -337.51568603515625,
      "logps/rejected": -342.9998779296875,
      "loss": 0.1424,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -1.327646255493164,
      "rewards/margins": 3.696226119995117,
      "rewards/rejected": -5.0238728523254395,
      "step": 940
    },
    {
      "epoch": 1.99,
      "grad_norm": 34.755846610400575,
      "learning_rate": 1.54469782361964e-07,
      "logits/chosen": -1.9773037433624268,
      "logits/rejected": -1.8121259212493896,
      "logps/chosen": -310.9110107421875,
      "logps/rejected": -364.62359619140625,
      "loss": 0.145,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -1.399800181388855,
      "rewards/margins": 3.679616928100586,
      "rewards/rejected": -5.0794172286987305,
      "step": 950
    },
    {
      "epoch": 2.01,
      "grad_norm": 9.487017101643959,
      "learning_rate": 1.488723393865766e-07,
      "logits/chosen": -1.9744060039520264,
      "logits/rejected": -1.662217140197754,
      "logps/chosen": -328.89556884765625,
      "logps/rejected": -356.9366149902344,
      "loss": 0.0972,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -1.0852992534637451,
      "rewards/margins": 4.1752800941467285,
      "rewards/rejected": -5.260579586029053,
      "step": 960
    },
    {
      "epoch": 2.03,
      "grad_norm": 6.928449852103405,
      "learning_rate": 1.4333487121291395e-07,
      "logits/chosen": -1.9290939569473267,
      "logits/rejected": -1.646998643875122,
      "logps/chosen": -290.2763977050781,
      "logps/rejected": -359.6099548339844,
      "loss": 0.0377,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -0.9613211750984192,
      "rewards/margins": 4.470429420471191,
      "rewards/rejected": -5.431751251220703,
      "step": 970
    },
    {
      "epoch": 2.05,
      "grad_norm": 12.357238308252128,
      "learning_rate": 1.3786066189356627e-07,
      "logits/chosen": -1.840659499168396,
      "logits/rejected": -1.3801103830337524,
      "logps/chosen": -323.639892578125,
      "logps/rejected": -382.70281982421875,
      "loss": 0.0412,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.1784292459487915,
      "rewards/margins": 4.754664897918701,
      "rewards/rejected": -5.933094501495361,
      "step": 980
    },
    {
      "epoch": 2.07,
      "grad_norm": 12.765885065958322,
      "learning_rate": 1.3245295796480788e-07,
      "logits/chosen": -1.6344573497772217,
      "logits/rejected": -1.3037782907485962,
      "logps/chosen": -280.02520751953125,
      "logps/rejected": -373.9437561035156,
      "loss": 0.0483,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -1.5978093147277832,
      "rewards/margins": 4.701867580413818,
      "rewards/rejected": -6.299676895141602,
      "step": 990
    },
    {
      "epoch": 2.09,
      "grad_norm": 10.639657237631194,
      "learning_rate": 1.2711496652120578e-07,
      "logits/chosen": -1.6058915853500366,
      "logits/rejected": -1.2118239402770996,
      "logps/chosen": -295.133056640625,
      "logps/rejected": -380.3929748535156,
      "loss": 0.044,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -1.539745807647705,
      "rewards/margins": 5.079216957092285,
      "rewards/rejected": -6.61896276473999,
      "step": 1000
    },
    {
      "epoch": 2.09,
      "eval_logits/chosen": -1.7406296730041504,
      "eval_logits/rejected": -1.4629344940185547,
      "eval_logps/chosen": -316.4520263671875,
      "eval_logps/rejected": -365.49591064453125,
      "eval_loss": 0.6058282852172852,
      "eval_rewards/accuracies": 0.7890625,
      "eval_rewards/chosen": -2.6929280757904053,
      "eval_rewards/margins": 2.4485080242156982,
      "eval_rewards/rejected": -5.1414361000061035,
      "eval_runtime": 96.8107,
      "eval_samples_per_second": 20.659,
      "eval_steps_per_second": 0.331,
      "step": 1000
    },
    {
      "epoch": 2.11,
      "grad_norm": 8.532531818207984,
      "learning_rate": 1.2184985331361878e-07,
      "logits/chosen": -1.5615358352661133,
      "logits/rejected": -1.3172276020050049,
      "logps/chosen": -293.0592041015625,
      "logps/rejected": -389.7725830078125,
      "loss": 0.0365,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -1.6128288507461548,
      "rewards/margins": 4.956469535827637,
      "rewards/rejected": -6.56929874420166,
      "step": 1010
    },
    {
      "epoch": 2.13,
      "grad_norm": 8.404017113839789,
      "learning_rate": 1.1666074087171627e-07,
      "logits/chosen": -1.585607886314392,
      "logits/rejected": -1.2554726600646973,
      "logps/chosen": -287.69989013671875,
      "logps/rejected": -414.93267822265625,
      "loss": 0.0337,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -1.7983949184417725,
      "rewards/margins": 5.210457801818848,
      "rewards/rejected": -7.008852481842041,
      "step": 1020
    },
    {
      "epoch": 2.15,
      "grad_norm": 10.846070292764287,
      "learning_rate": 1.115507066521304e-07,
      "logits/chosen": -1.5471771955490112,
      "logits/rejected": -1.1083250045776367,
      "logps/chosen": -304.71533203125,
      "logps/rejected": -398.4665222167969,
      "loss": 0.0418,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -1.608544111251831,
      "rewards/margins": 5.360156536102295,
      "rewards/rejected": -6.968701362609863,
      "step": 1030
    },
    {
      "epoch": 2.18,
      "grad_norm": 11.175167320601789,
      "learning_rate": 1.065227812133381e-07,
      "logits/chosen": -1.481093168258667,
      "logits/rejected": -0.8541752696037292,
      "logps/chosen": -340.7627868652344,
      "logps/rejected": -378.1071472167969,
      "loss": 0.0369,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -2.031552791595459,
      "rewards/margins": 5.121342658996582,
      "rewards/rejected": -7.152895927429199,
      "step": 1040
    },
    {
      "epoch": 2.2,
      "grad_norm": 9.191567382424724,
      "learning_rate": 1.0157994641835734e-07,
      "logits/chosen": -1.3451311588287354,
      "logits/rejected": -0.851954460144043,
      "logps/chosen": -312.46246337890625,
      "logps/rejected": -443.61480712890625,
      "loss": 0.0366,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.4045233726501465,
      "rewards/margins": 5.692896842956543,
      "rewards/rejected": -8.097419738769531,
      "step": 1050
    },
    {
      "epoch": 2.22,
      "grad_norm": 10.368281313443552,
      "learning_rate": 9.672513366632259e-08,
      "logits/chosen": -1.4387218952178955,
      "logits/rejected": -0.9104587435722351,
      "logps/chosen": -313.5397644042969,
      "logps/rejected": -392.3639221191406,
      "loss": 0.0335,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -2.1655592918395996,
      "rewards/margins": 5.339682579040527,
      "rewards/rejected": -7.505242347717285,
      "step": 1060
    },
    {
      "epoch": 2.24,
      "grad_norm": 9.399192097048465,
      "learning_rate": 9.196122215398824e-08,
      "logits/chosen": -1.5347858667373657,
      "logits/rejected": -1.1239128112792969,
      "logps/chosen": -345.493408203125,
      "logps/rejected": -442.33880615234375,
      "loss": 0.0335,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -2.0253329277038574,
      "rewards/margins": 5.696555137634277,
      "rewards/rejected": -7.721887111663818,
      "step": 1070
    },
    {
      "epoch": 2.26,
      "grad_norm": 13.854211150002678,
      "learning_rate": 8.729103716819111e-08,
      "logits/chosen": -1.4636061191558838,
      "logits/rejected": -0.9430959820747375,
      "logps/chosen": -348.165771484375,
      "logps/rejected": -437.73150634765625,
      "loss": 0.0324,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -1.851531982421875,
      "rewards/margins": 5.859683990478516,
      "rewards/rejected": -7.711215972900391,
      "step": 1080
    },
    {
      "epoch": 2.28,
      "grad_norm": 13.012519621486515,
      "learning_rate": 8.271734841028552e-08,
      "logits/chosen": -1.4426562786102295,
      "logits/rejected": -0.9428352117538452,
      "logps/chosen": -357.9177551269531,
      "logps/rejected": -440.7193298339844,
      "loss": 0.0276,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -2.550102710723877,
      "rewards/margins": 5.568441390991211,
      "rewards/rejected": -8.11854362487793,
      "step": 1090
    },
    {
      "epoch": 2.3,
      "grad_norm": 7.343017253233073,
      "learning_rate": 7.824286835354262e-08,
      "logits/chosen": -1.2438175678253174,
      "logits/rejected": -0.7905367612838745,
      "logps/chosen": -294.59564208984375,
      "logps/rejected": -394.0030517578125,
      "loss": 0.0307,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -1.902488350868225,
      "rewards/margins": 5.68535041809082,
      "rewards/rejected": -7.587839603424072,
      "step": 1100
    },
    {
      "epoch": 2.3,
      "eval_logits/chosen": -1.330996036529541,
      "eval_logits/rejected": -0.9162449240684509,
      "eval_logps/chosen": -337.038330078125,
      "eval_logps/rejected": -397.1616516113281,
      "eval_loss": 0.6699962019920349,
      "eval_rewards/accuracies": 0.76953125,
      "eval_rewards/chosen": -3.722243070602417,
      "eval_rewards/margins": 3.002480983734131,
      "eval_rewards/rejected": -6.724723815917969,
      "eval_runtime": 96.8573,
      "eval_samples_per_second": 20.649,
      "eval_steps_per_second": 0.33,
      "step": 1100
    },
    {
      "epoch": 2.32,
      "grad_norm": 22.14619972377515,
      "learning_rate": 7.387025063449081e-08,
      "logits/chosen": -1.2601680755615234,
      "logits/rejected": -0.4739012122154236,
      "logps/chosen": -334.8611755371094,
      "logps/rejected": -403.3932800292969,
      "loss": 0.0294,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -2.9858272075653076,
      "rewards/margins": 5.638393878936768,
      "rewards/rejected": -8.624221801757812,
      "step": 1110
    },
    {
      "epoch": 2.34,
      "grad_norm": 22.797319211696394,
      "learning_rate": 6.960208847914884e-08,
      "logits/chosen": -1.1370799541473389,
      "logits/rejected": -0.7852309942245483,
      "logps/chosen": -302.19561767578125,
      "logps/rejected": -427.47283935546875,
      "loss": 0.0337,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -2.330061912536621,
      "rewards/margins": 6.187905311584473,
      "rewards/rejected": -8.517967224121094,
      "step": 1120
    },
    {
      "epoch": 2.36,
      "grad_norm": 12.447087206832133,
      "learning_rate": 6.544091316508646e-08,
      "logits/chosen": -1.1922690868377686,
      "logits/rejected": -0.7016154527664185,
      "logps/chosen": -337.29254150390625,
      "logps/rejected": -419.9600524902344,
      "loss": 0.0309,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -2.260216474533081,
      "rewards/margins": 6.044236660003662,
      "rewards/rejected": -8.30445384979248,
      "step": 1130
    },
    {
      "epoch": 2.38,
      "grad_norm": 6.670071848884272,
      "learning_rate": 6.138919252022435e-08,
      "logits/chosen": -1.2661590576171875,
      "logits/rejected": -0.726656973361969,
      "logps/chosen": -337.79095458984375,
      "logps/rejected": -444.1019592285156,
      "loss": 0.0321,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.347249746322632,
      "rewards/margins": 5.921750068664551,
      "rewards/rejected": -8.268999099731445,
      "step": 1140
    },
    {
      "epoch": 2.41,
      "grad_norm": 20.202187916611614,
      "learning_rate": 5.7449329459262895e-08,
      "logits/chosen": -1.3327900171279907,
      "logits/rejected": -0.7205911874771118,
      "logps/chosen": -327.96160888671875,
      "logps/rejected": -431.84246826171875,
      "loss": 0.0315,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -2.5763912200927734,
      "rewards/margins": 6.172744274139404,
      "rewards/rejected": -8.749135971069336,
      "step": 1150
    },
    {
      "epoch": 2.43,
      "grad_norm": 12.705067042675866,
      "learning_rate": 5.362366055860934e-08,
      "logits/chosen": -1.3245675563812256,
      "logits/rejected": -0.8294090032577515,
      "logps/chosen": -339.1673278808594,
      "logps/rejected": -427.9764709472656,
      "loss": 0.0264,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.199721336364746,
      "rewards/margins": 5.974953651428223,
      "rewards/rejected": -8.174674034118652,
      "step": 1160
    },
    {
      "epoch": 2.45,
      "grad_norm": 10.500886681852085,
      "learning_rate": 4.991445467064689e-08,
      "logits/chosen": -1.205335259437561,
      "logits/rejected": -0.8114490509033203,
      "logps/chosen": -309.63421630859375,
      "logps/rejected": -438.3241271972656,
      "loss": 0.0317,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -2.618736982345581,
      "rewards/margins": 6.0979084968566895,
      "rewards/rejected": -8.716645240783691,
      "step": 1170
    },
    {
      "epoch": 2.47,
      "grad_norm": 12.519852374223428,
      "learning_rate": 4.6323911578168146e-08,
      "logits/chosen": -1.1557085514068604,
      "logits/rejected": -0.6752947568893433,
      "logps/chosen": -306.9227294921875,
      "logps/rejected": -395.4352722167969,
      "loss": 0.0323,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -2.859976053237915,
      "rewards/margins": 5.740398406982422,
      "rewards/rejected": -8.600374221801758,
      "step": 1180
    },
    {
      "epoch": 2.49,
      "grad_norm": 14.10405859024082,
      "learning_rate": 4.285416068977166e-08,
      "logits/chosen": -1.2870628833770752,
      "logits/rejected": -0.8281751871109009,
      "logps/chosen": -295.5076904296875,
      "logps/rejected": -401.1368713378906,
      "loss": 0.0387,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -2.580979585647583,
      "rewards/margins": 6.097764015197754,
      "rewards/rejected": -8.678743362426758,
      "step": 1190
    },
    {
      "epoch": 2.51,
      "grad_norm": 10.124969755651362,
      "learning_rate": 3.9507259776993954e-08,
      "logits/chosen": -1.2816931009292603,
      "logits/rejected": -0.8490222096443176,
      "logps/chosen": -333.55950927734375,
      "logps/rejected": -454.38433837890625,
      "loss": 0.0317,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -2.4796204566955566,
      "rewards/margins": 6.314810752868652,
      "rewards/rejected": -8.794431686401367,
      "step": 1200
    },
    {
      "epoch": 2.51,
      "eval_logits/chosen": -1.2927337884902954,
      "eval_logits/rejected": -0.9226770401000977,
      "eval_logps/chosen": -341.82611083984375,
      "eval_logps/rejected": -401.94482421875,
      "eval_loss": 0.6710954308509827,
      "eval_rewards/accuracies": 0.77734375,
      "eval_rewards/chosen": -3.9616329669952393,
      "eval_rewards/margins": 3.002251148223877,
      "eval_rewards/rejected": -6.963884353637695,
      "eval_runtime": 96.6177,
      "eval_samples_per_second": 20.7,
      "eval_steps_per_second": 0.331,
      "step": 1200
    },
    {
      "epoch": 2.53,
      "grad_norm": 9.343788604433174,
      "learning_rate": 3.6285193753926995e-08,
      "logits/chosen": -1.2478582859039307,
      "logits/rejected": -0.6749597787857056,
      "logps/chosen": -335.22381591796875,
      "logps/rejected": -448.06427001953125,
      "loss": 0.0372,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -3.044740915298462,
      "rewards/margins": 5.88831090927124,
      "rewards/rejected": -8.933051109313965,
      "step": 1210
    },
    {
      "epoch": 2.55,
      "grad_norm": 15.370939596872683,
      "learning_rate": 3.3189873500044376e-08,
      "logits/chosen": -1.2345765829086304,
      "logits/rejected": -0.7880679368972778,
      "logps/chosen": -316.099853515625,
      "logps/rejected": -424.1470642089844,
      "loss": 0.0304,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -2.545562267303467,
      "rewards/margins": 6.050265312194824,
      "rewards/rejected": -8.595827102661133,
      "step": 1220
    },
    {
      "epoch": 2.57,
      "grad_norm": 9.256523996293426,
      "learning_rate": 3.022313472693447e-08,
      "logits/chosen": -1.2926331758499146,
      "logits/rejected": -0.7623311877250671,
      "logps/chosen": -323.86981201171875,
      "logps/rejected": -432.7613220214844,
      "loss": 0.0326,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -3.008089065551758,
      "rewards/margins": 5.656081676483154,
      "rewards/rejected": -8.66417121887207,
      "step": 1230
    },
    {
      "epoch": 2.59,
      "grad_norm": 13.108228798737402,
      "learning_rate": 2.738673688961296e-08,
      "logits/chosen": -1.4012349843978882,
      "logits/rejected": -0.9218941926956177,
      "logps/chosen": -351.8048400878906,
      "logps/rejected": -434.9781799316406,
      "loss": 0.0339,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -2.5394856929779053,
      "rewards/margins": 5.643149375915527,
      "rewards/rejected": -8.182635307312012,
      "step": 1240
    },
    {
      "epoch": 2.62,
      "grad_norm": 7.525159999057864,
      "learning_rate": 2.4682362143059797e-08,
      "logits/chosen": -1.2464749813079834,
      "logits/rejected": -0.6382617354393005,
      "logps/chosen": -326.1867370605469,
      "logps/rejected": -427.4964294433594,
      "loss": 0.0245,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -2.6778645515441895,
      "rewards/margins": 6.1839213371276855,
      "rewards/rejected": -8.861784934997559,
      "step": 1250
    },
    {
      "epoch": 2.64,
      "grad_norm": 20.76488422355617,
      "learning_rate": 2.2111614344599684e-08,
      "logits/chosen": -1.1929394006729126,
      "logits/rejected": -0.6367761492729187,
      "logps/chosen": -346.1959533691406,
      "logps/rejected": -437.5431213378906,
      "loss": 0.0344,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -2.5477092266082764,
      "rewards/margins": 5.813665390014648,
      "rewards/rejected": -8.361373901367188,
      "step": 1260
    },
    {
      "epoch": 2.66,
      "grad_norm": 8.669045699213816,
      "learning_rate": 1.9676018102718213e-08,
      "logits/chosen": -1.247870922088623,
      "logits/rejected": -0.6409175395965576,
      "logps/chosen": -365.83392333984375,
      "logps/rejected": -399.61962890625,
      "loss": 0.0338,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -2.6575913429260254,
      "rewards/margins": 5.685526371002197,
      "rewards/rejected": -8.343117713928223,
      "step": 1270
    },
    {
      "epoch": 2.68,
      "grad_norm": 22.62219866138702,
      "learning_rate": 1.7377017872876987e-08,
      "logits/chosen": -1.208968162536621,
      "logits/rejected": -0.6456762552261353,
      "logps/chosen": -317.1292419433594,
      "logps/rejected": -430.59613037109375,
      "loss": 0.0336,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.8459768295288086,
      "rewards/margins": 5.9941020011901855,
      "rewards/rejected": -8.840079307556152,
      "step": 1280
    },
    {
      "epoch": 2.7,
      "grad_norm": 20.33027332534465,
      "learning_rate": 1.521597710086439e-08,
      "logits/chosen": -1.1466628313064575,
      "logits/rejected": -0.7244657278060913,
      "logps/chosen": -356.29986572265625,
      "logps/rejected": -458.36279296875,
      "loss": 0.0348,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -2.7150323390960693,
      "rewards/margins": 6.071440696716309,
      "rewards/rejected": -8.786473274230957,
      "step": 1290
    },
    {
      "epoch": 2.72,
      "grad_norm": 11.138423191199344,
      "learning_rate": 1.3194177414189905e-08,
      "logits/chosen": -1.0380289554595947,
      "logits/rejected": -0.5761706233024597,
      "logps/chosen": -299.1542663574219,
      "logps/rejected": -413.3262634277344,
      "loss": 0.0264,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -2.816028118133545,
      "rewards/margins": 5.865799903869629,
      "rewards/rejected": -8.681828498840332,
      "step": 1300
    },
    {
      "epoch": 2.72,
      "eval_logits/chosen": -1.2189825773239136,
      "eval_logits/rejected": -0.8369507789611816,
      "eval_logps/chosen": -347.2215576171875,
      "eval_logps/rejected": -407.835205078125,
      "eval_loss": 0.6777693033218384,
      "eval_rewards/accuracies": 0.77734375,
      "eval_rewards/chosen": -4.231404781341553,
      "eval_rewards/margins": 3.0269954204559326,
      "eval_rewards/rejected": -7.258399486541748,
      "eval_runtime": 96.5907,
      "eval_samples_per_second": 20.706,
      "eval_steps_per_second": 0.331,
      "step": 1300
    },
    {
      "epoch": 2.74,
      "grad_norm": 18.646420478229594,
      "learning_rate": 1.1312817862001945e-08,
      "logits/chosen": -1.174865961074829,
      "logits/rejected": -0.5988924503326416,
      "logps/chosen": -335.7987060546875,
      "logps/rejected": -438.8302307128906,
      "loss": 0.0323,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.0069799423217773,
      "rewards/margins": 6.027297496795654,
      "rewards/rejected": -9.034276962280273,
      "step": 1310
    },
    {
      "epoch": 2.76,
      "grad_norm": 9.37653587990746,
      "learning_rate": 9.57301420397924e-09,
      "logits/chosen": -1.2077335119247437,
      "logits/rejected": -0.6788758039474487,
      "logps/chosen": -344.1173400878906,
      "logps/rejected": -449.73822021484375,
      "loss": 0.0296,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -2.7568278312683105,
      "rewards/margins": 6.043575763702393,
      "rewards/rejected": -8.800403594970703,
      "step": 1320
    },
    {
      "epoch": 2.78,
      "grad_norm": 19.319012047791922,
      "learning_rate": 7.975798248618076e-09,
      "logits/chosen": -1.2139251232147217,
      "logits/rejected": -0.5430259108543396,
      "logps/chosen": -304.11981201171875,
      "logps/rejected": -399.5338134765625,
      "loss": 0.0403,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -2.8824665546417236,
      "rewards/margins": 5.712244510650635,
      "rewards/rejected": -8.594710350036621,
      "step": 1330
    },
    {
      "epoch": 2.8,
      "grad_norm": 15.988430571781114,
      "learning_rate": 6.522117241307606e-09,
      "logits/chosen": -1.257893681526184,
      "logits/rejected": -0.7202562093734741,
      "logps/chosen": -358.0582275390625,
      "logps/rejected": -453.0985412597656,
      "loss": 0.0249,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -2.752913475036621,
      "rewards/margins": 5.904249668121338,
      "rewards/rejected": -8.6571626663208,
      "step": 1340
    },
    {
      "epoch": 2.82,
      "grad_norm": 18.522259659368,
      "learning_rate": 5.212833302556258e-09,
      "logits/chosen": -1.0940848588943481,
      "logits/rejected": -0.642852783203125,
      "logps/chosen": -330.330810546875,
      "logps/rejected": -439.7862243652344,
      "loss": 0.0316,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -3.221640110015869,
      "rewards/margins": 5.774725914001465,
      "rewards/rejected": -8.996365547180176,
      "step": 1350
    },
    {
      "epoch": 2.85,
      "grad_norm": 14.304064916507468,
      "learning_rate": 4.048722916702302e-09,
      "logits/chosen": -1.2064802646636963,
      "logits/rejected": -0.7113040685653687,
      "logps/chosen": -350.12359619140625,
      "logps/rejected": -433.10784912109375,
      "loss": 0.0292,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -3.004544734954834,
      "rewards/margins": 5.731839179992676,
      "rewards/rejected": -8.736384391784668,
      "step": 1360
    },
    {
      "epoch": 2.87,
      "grad_norm": 9.460405793660744,
      "learning_rate": 3.030476471411664e-09,
      "logits/chosen": -1.2229253053665161,
      "logits/rejected": -0.5610889196395874,
      "logps/chosen": -358.481689453125,
      "logps/rejected": -432.617919921875,
      "loss": 0.0255,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -3.1794846057891846,
      "rewards/margins": 6.04774284362793,
      "rewards/rejected": -9.227226257324219,
      "step": 1370
    },
    {
      "epoch": 2.89,
      "grad_norm": 12.222248359736108,
      "learning_rate": 2.158697848236607e-09,
      "logits/chosen": -1.3239306211471558,
      "logits/rejected": -0.696345865726471,
      "logps/chosen": -345.43890380859375,
      "logps/rejected": -420.57415771484375,
      "loss": 0.0324,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -2.8451218605041504,
      "rewards/margins": 5.87528657913208,
      "rewards/rejected": -8.72040843963623,
      "step": 1380
    },
    {
      "epoch": 2.91,
      "grad_norm": 12.712161033692837,
      "learning_rate": 1.4339040644774092e-09,
      "logits/chosen": -1.1367696523666382,
      "logits/rejected": -0.6753785014152527,
      "logps/chosen": -300.333984375,
      "logps/rejected": -410.33966064453125,
      "loss": 0.0346,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -2.8657116889953613,
      "rewards/margins": 5.644627571105957,
      "rewards/rejected": -8.510337829589844,
      "step": 1390
    },
    {
      "epoch": 2.93,
      "grad_norm": 7.479511958985555,
      "learning_rate": 8.56524966559885e-10,
      "logits/chosen": -1.297228455543518,
      "logits/rejected": -0.9393932223320007,
      "logps/chosen": -337.4761962890625,
      "logps/rejected": -469.80499267578125,
      "loss": 0.0343,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -2.661848306655884,
      "rewards/margins": 5.969611644744873,
      "rewards/rejected": -8.631460189819336,
      "step": 1400
    },
    {
      "epoch": 2.93,
      "eval_logits/chosen": -1.2134709358215332,
      "eval_logits/rejected": -0.8310315608978271,
      "eval_logps/chosen": -347.1476135253906,
      "eval_logps/rejected": -408.3960876464844,
      "eval_loss": 0.6824045181274414,
      "eval_rewards/accuracies": 0.77734375,
      "eval_rewards/chosen": -4.227706432342529,
      "eval_rewards/margins": 3.058739423751831,
      "eval_rewards/rejected": -7.286445617675781,
      "eval_runtime": 96.8438,
      "eval_samples_per_second": 20.652,
      "eval_steps_per_second": 0.33,
      "step": 1400
    },
    {
      "epoch": 2.95,
      "grad_norm": 25.645964088265643,
      "learning_rate": 4.269029751107489e-10,
      "logits/chosen": -1.169325590133667,
      "logits/rejected": -0.646837055683136,
      "logps/chosen": -327.53558349609375,
      "logps/rejected": -438.29022216796875,
      "loss": 0.0315,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -2.6820549964904785,
      "rewards/margins": 6.348910331726074,
      "rewards/rejected": -9.030964851379395,
      "step": 1410
    },
    {
      "epoch": 2.97,
      "grad_norm": 10.010864148743238,
      "learning_rate": 1.4529288188125377e-10,
      "logits/chosen": -1.1855485439300537,
      "logits/rejected": -0.6920149326324463,
      "logps/chosen": -337.4898376464844,
      "logps/rejected": -440.05987548828125,
      "loss": 0.0257,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -2.923854351043701,
      "rewards/margins": 6.196907997131348,
      "rewards/rejected": -9.120763778686523,
      "step": 1420
    },
    {
      "epoch": 2.99,
      "grad_norm": 37.578079095213596,
      "learning_rate": 1.1861698640563966e-11,
      "logits/chosen": -1.1965336799621582,
      "logits/rejected": -0.6326996684074402,
      "logps/chosen": -348.0246276855469,
      "logps/rejected": -445.17431640625,
      "loss": 0.0318,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -2.8414111137390137,
      "rewards/margins": 6.086348533630371,
      "rewards/rejected": -8.927759170532227,
      "step": 1430
    },
    {
      "epoch": 3.0,
      "step": 1434,
      "total_flos": 0.0,
      "train_loss": 0.007373018379906397,
      "train_runtime": 5373.2533,
      "train_samples_per_second": 34.132,
      "train_steps_per_second": 0.267
    }
  ],
  "logging_steps": 10,
  "max_steps": 1434,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 3,
  "save_steps": 100,
  "total_flos": 0.0,
  "train_batch_size": 8,
  "trial_name": null,
  "trial_params": null
}