{
  "best_metric": 0.9069767594337463,
  "best_model_checkpoint": "./llama3/27-06-24-Weni-ZeroShot-Agents-Llama3-4.0.37-DPO_Experiment with DPO and Llama3 8B, zeroshot 4.0.37-2_max_steps-570_batch_16_2024-06-27_ppid_9/checkpoint-80",
  "epoch": 3.34640522875817,
  "eval_steps": 20,
  "global_step": 320,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.10457516339869281,
      "grad_norm": 3.655749559402466,
      "learning_rate": 3.3333333333333337e-06,
      "logits/chosen": -0.22948360443115234,
      "logits/rejected": -0.22978875041007996,
      "logps/chosen": -39.710899353027344,
      "logps/rejected": -39.52346420288086,
      "loss": 0.662,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.11001075804233551,
      "rewards/margins": 0.06790003925561905,
      "rewards/rejected": 0.04211071878671646,
      "step": 10
    },
    {
      "epoch": 0.20915032679738563,
      "grad_norm": 2.4934024810791016,
      "learning_rate": 5.978260869565218e-06,
      "logits/chosen": -0.22906163334846497,
      "logits/rejected": -0.22876068949699402,
      "logps/chosen": -30.569751739501953,
      "logps/rejected": -33.86491012573242,
      "loss": 0.5549,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": 0.9981435537338257,
      "rewards/margins": 0.38557058572769165,
      "rewards/rejected": 0.6125729084014893,
      "step": 20
    },
    {
      "epoch": 0.20915032679738563,
      "eval_logits/chosen": -0.19793638586997986,
      "eval_logits/rejected": -0.197507843375206,
      "eval_logps/chosen": -26.848573684692383,
      "eval_logps/rejected": -35.13609313964844,
      "eval_loss": 0.43225446343421936,
      "eval_rewards/accuracies": 0.8081395626068115,
      "eval_rewards/chosen": 1.355695366859436,
      "eval_rewards/margins": 0.861485481262207,
      "eval_rewards/rejected": 0.4942099153995514,
      "eval_runtime": 76.6712,
      "eval_samples_per_second": 2.23,
      "eval_steps_per_second": 1.122,
      "step": 20
    },
    {
      "epoch": 0.3137254901960784,
      "grad_norm": 1.9067373275756836,
      "learning_rate": 5.869565217391305e-06,
      "logits/chosen": -0.23883526027202606,
      "logits/rejected": -0.2374078780412674,
      "logps/chosen": -30.109268188476562,
      "logps/rejected": -43.55142593383789,
      "loss": 0.3732,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 1.0676757097244263,
      "rewards/margins": 1.4297826290130615,
      "rewards/rejected": -0.3621070086956024,
      "step": 30
    },
    {
      "epoch": 0.41830065359477125,
      "grad_norm": 3.629997730255127,
      "learning_rate": 5.760869565217392e-06,
      "logits/chosen": -0.2035980522632599,
      "logits/rejected": -0.20044592022895813,
      "logps/chosen": -28.863727569580078,
      "logps/rejected": -50.26195526123047,
      "loss": 0.3276,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 1.200588345527649,
      "rewards/margins": 2.2083239555358887,
      "rewards/rejected": -1.0077354907989502,
      "step": 40
    },
    {
      "epoch": 0.41830065359477125,
      "eval_logits/chosen": -0.16349415481090546,
      "eval_logits/rejected": -0.15954892337322235,
      "eval_logps/chosen": -24.61951446533203,
      "eval_logps/rejected": -48.91069412231445,
      "eval_loss": 0.34378084540367126,
      "eval_rewards/accuracies": 0.8895348906517029,
      "eval_rewards/chosen": 1.5786010026931763,
      "eval_rewards/margins": 2.4618515968322754,
      "eval_rewards/rejected": -0.8832504749298096,
      "eval_runtime": 76.7395,
      "eval_samples_per_second": 2.228,
      "eval_steps_per_second": 1.121,
      "step": 40
    },
    {
      "epoch": 0.5228758169934641,
      "grad_norm": 2.5461719036102295,
      "learning_rate": 5.652173913043479e-06,
      "logits/chosen": -0.16493651270866394,
      "logits/rejected": -0.16144290566444397,
      "logps/chosen": -24.76712989807129,
      "logps/rejected": -46.993003845214844,
      "loss": 0.3648,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 1.5913469791412354,
      "rewards/margins": 2.25089955329895,
      "rewards/rejected": -0.6595526933670044,
      "step": 50
    },
    {
      "epoch": 0.6274509803921569,
      "grad_norm": 2.0358729362487793,
      "learning_rate": 5.543478260869566e-06,
      "logits/chosen": -0.2202371060848236,
      "logits/rejected": -0.21734721958637238,
      "logps/chosen": -26.78680419921875,
      "logps/rejected": -47.03938674926758,
      "loss": 0.3448,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": 1.3999578952789307,
      "rewards/margins": 2.0680160522460938,
      "rewards/rejected": -0.6680583357810974,
      "step": 60
    },
    {
      "epoch": 0.6274509803921569,
      "eval_logits/chosen": -0.17944073677062988,
      "eval_logits/rejected": -0.1763658970594406,
      "eval_logps/chosen": -27.3469295501709,
      "eval_logps/rejected": -48.937984466552734,
      "eval_loss": 0.2700035870075226,
      "eval_rewards/accuracies": 0.9069767594337463,
      "eval_rewards/chosen": 1.3058594465255737,
      "eval_rewards/margins": 2.191838026046753,
      "eval_rewards/rejected": -0.8859787583351135,
      "eval_runtime": 76.7029,
      "eval_samples_per_second": 2.229,
      "eval_steps_per_second": 1.121,
      "step": 60
    },
    {
      "epoch": 0.7320261437908496,
      "grad_norm": 1.7749762535095215,
      "learning_rate": 5.4347826086956525e-06,
      "logits/chosen": -0.1932402402162552,
      "logits/rejected": -0.19045117497444153,
      "logps/chosen": -26.827350616455078,
      "logps/rejected": -46.88459014892578,
      "loss": 0.2909,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": 1.3978151082992554,
      "rewards/margins": 2.0628795623779297,
      "rewards/rejected": -0.6650643348693848,
      "step": 70
    },
    {
      "epoch": 0.8366013071895425,
      "grad_norm": 2.9730594158172607,
      "learning_rate": 5.326086956521739e-06,
      "logits/chosen": -0.15139932930469513,
      "logits/rejected": -0.14830470085144043,
      "logps/chosen": -26.229084014892578,
      "logps/rejected": -47.40288162231445,
      "loss": 0.2844,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 1.493841290473938,
      "rewards/margins": 2.2263193130493164,
      "rewards/rejected": -0.7324780225753784,
      "step": 80
    },
    {
      "epoch": 0.8366013071895425,
      "eval_logits/chosen": -0.15539461374282837,
      "eval_logits/rejected": -0.15173271298408508,
      "eval_logps/chosen": -30.73025894165039,
      "eval_logps/rejected": -54.970027923583984,
      "eval_loss": 0.262494832277298,
      "eval_rewards/accuracies": 0.9069767594337463,
      "eval_rewards/chosen": 0.967526912689209,
      "eval_rewards/margins": 2.4567105770111084,
      "eval_rewards/rejected": -1.4891836643218994,
      "eval_runtime": 76.69,
      "eval_samples_per_second": 2.23,
      "eval_steps_per_second": 1.121,
      "step": 80
    },
    {
      "epoch": 0.9411764705882353,
      "grad_norm": 2.7646617889404297,
      "learning_rate": 5.2173913043478265e-06,
      "logits/chosen": -0.17744764685630798,
      "logits/rejected": -0.1731792390346527,
      "logps/chosen": -25.179697036743164,
      "logps/rejected": -51.22761154174805,
      "loss": 0.2203,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 1.5833141803741455,
      "rewards/margins": 2.6879732608795166,
      "rewards/rejected": -1.104659080505371,
      "step": 90
    },
    {
      "epoch": 1.0457516339869282,
      "grad_norm": 1.8633959293365479,
      "learning_rate": 5.1086956521739134e-06,
      "logits/chosen": -0.1642988920211792,
      "logits/rejected": -0.1597273051738739,
      "logps/chosen": -23.413448333740234,
      "logps/rejected": -53.029579162597656,
      "loss": 0.3134,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 1.7295477390289307,
      "rewards/margins": 3.0015645027160645,
      "rewards/rejected": -1.2720168828964233,
      "step": 100
    },
    {
      "epoch": 1.0457516339869282,
      "eval_logits/chosen": -0.14374618232250214,
      "eval_logits/rejected": -0.13886626064777374,
      "eval_logps/chosen": -25.8454532623291,
      "eval_logps/rejected": -56.691768646240234,
      "eval_loss": 0.23542174696922302,
      "eval_rewards/accuracies": 0.9244186282157898,
      "eval_rewards/chosen": 1.456007480621338,
      "eval_rewards/margins": 3.1173653602600098,
      "eval_rewards/rejected": -1.6613577604293823,
      "eval_runtime": 76.7594,
      "eval_samples_per_second": 2.228,
      "eval_steps_per_second": 1.12,
      "step": 100
    },
    {
      "epoch": 1.1503267973856208,
      "grad_norm": 3.7231855392456055,
      "learning_rate": 5e-06,
      "logits/chosen": -0.17482638359069824,
      "logits/rejected": -0.1706036627292633,
      "logps/chosen": -27.381885528564453,
      "logps/rejected": -53.334877014160156,
      "loss": 0.3177,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": 1.3857864141464233,
      "rewards/margins": 2.708588123321533,
      "rewards/rejected": -1.3228017091751099,
      "step": 110
    },
    {
      "epoch": 1.2549019607843137,
      "grad_norm": 5.588420391082764,
      "learning_rate": 4.8913043478260865e-06,
      "logits/chosen": -0.19949769973754883,
      "logits/rejected": -0.19602252542972565,
      "logps/chosen": -21.503002166748047,
      "logps/rejected": -45.17253494262695,
      "loss": 0.2497,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": 1.9555631875991821,
      "rewards/margins": 2.432082414627075,
      "rewards/rejected": -0.47651925683021545,
      "step": 120
    },
    {
      "epoch": 1.2549019607843137,
      "eval_logits/chosen": -0.14820654690265656,
      "eval_logits/rejected": -0.14320875704288483,
      "eval_logps/chosen": -24.672155380249023,
      "eval_logps/rejected": -54.095088958740234,
      "eval_loss": 0.21676376461982727,
      "eval_rewards/accuracies": 0.9186046719551086,
      "eval_rewards/chosen": 1.5733370780944824,
      "eval_rewards/margins": 2.9750266075134277,
      "eval_rewards/rejected": -1.4016892910003662,
      "eval_runtime": 76.7524,
      "eval_samples_per_second": 2.228,
      "eval_steps_per_second": 1.12,
      "step": 120
    },
    {
      "epoch": 1.3594771241830066,
      "grad_norm": 4.653593063354492,
      "learning_rate": 4.782608695652174e-06,
      "logits/chosen": -0.16941645741462708,
      "logits/rejected": -0.16412410140037537,
      "logps/chosen": -28.397680282592773,
      "logps/rejected": -59.14925003051758,
      "loss": 0.2098,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 1.2265132665634155,
      "rewards/margins": 3.106191873550415,
      "rewards/rejected": -1.879678726196289,
      "step": 130
    },
    {
      "epoch": 1.4640522875816995,
      "grad_norm": 4.618969440460205,
      "learning_rate": 4.673913043478261e-06,
      "logits/chosen": -0.1314123570919037,
      "logits/rejected": -0.12528486549854279,
      "logps/chosen": -18.617746353149414,
      "logps/rejected": -51.451507568359375,
      "loss": 0.2442,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 2.209608554840088,
      "rewards/margins": 3.3631699085235596,
      "rewards/rejected": -1.1535612344741821,
      "step": 140
    },
    {
      "epoch": 1.4640522875816995,
      "eval_logits/chosen": -0.13051746785640717,
      "eval_logits/rejected": -0.1261546015739441,
      "eval_logps/chosen": -14.140735626220703,
      "eval_logps/rejected": -38.07098388671875,
      "eval_loss": 0.2653515338897705,
      "eval_rewards/accuracies": 0.9069767594337463,
      "eval_rewards/chosen": 2.626479148864746,
      "eval_rewards/margins": 2.425758123397827,
      "eval_rewards/rejected": 0.20072098076343536,
      "eval_runtime": 76.7224,
      "eval_samples_per_second": 2.229,
      "eval_steps_per_second": 1.121,
      "step": 140
    },
    {
      "epoch": 1.5686274509803921,
      "grad_norm": 2.419158697128296,
      "learning_rate": 4.565217391304348e-06,
      "logits/chosen": -0.14353547990322113,
      "logits/rejected": -0.13822032511234283,
      "logps/chosen": -17.764816284179688,
      "logps/rejected": -48.77958679199219,
      "loss": 0.1411,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": 2.28568172454834,
      "rewards/margins": 3.1483988761901855,
      "rewards/rejected": -0.8627172708511353,
      "step": 150
    },
    {
      "epoch": 1.673202614379085,
      "grad_norm": 2.6137094497680664,
      "learning_rate": 4.456521739130434e-06,
      "logits/chosen": -0.0922718346118927,
      "logits/rejected": -0.08614876121282578,
      "logps/chosen": -30.0429630279541,
      "logps/rejected": -62.348907470703125,
      "loss": 0.2677,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 1.0795209407806396,
      "rewards/margins": 3.2884597778320312,
      "rewards/rejected": -2.2089390754699707,
      "step": 160
    },
    {
      "epoch": 1.673202614379085,
      "eval_logits/chosen": -0.10249081254005432,
      "eval_logits/rejected": -0.09574974328279495,
      "eval_logps/chosen": -22.81549835205078,
      "eval_logps/rejected": -60.084381103515625,
      "eval_loss": 0.19992607831954956,
      "eval_rewards/accuracies": 0.9244186282157898,
      "eval_rewards/chosen": 1.7590028047561646,
      "eval_rewards/margins": 3.75962233543396,
      "eval_rewards/rejected": -2.000619411468506,
      "eval_runtime": 76.7585,
      "eval_samples_per_second": 2.228,
      "eval_steps_per_second": 1.12,
      "step": 160
    },
    {
      "epoch": 1.7777777777777777,
      "grad_norm": 3.627882480621338,
      "learning_rate": 4.347826086956522e-06,
      "logits/chosen": -0.1151178628206253,
      "logits/rejected": -0.10893462598323822,
      "logps/chosen": -21.336753845214844,
      "logps/rejected": -56.53479766845703,
      "loss": 0.1838,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 1.9686607122421265,
      "rewards/margins": 3.594453811645508,
      "rewards/rejected": -1.6257928609848022,
      "step": 170
    },
    {
      "epoch": 1.8823529411764706,
      "grad_norm": 2.876845121383667,
      "learning_rate": 4.239130434782609e-06,
      "logits/chosen": -0.08697254955768585,
      "logits/rejected": -0.08123140037059784,
      "logps/chosen": -20.807594299316406,
      "logps/rejected": -49.710174560546875,
      "loss": 0.2598,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 1.9989045858383179,
      "rewards/margins": 2.9567558765411377,
      "rewards/rejected": -0.957851231098175,
      "step": 180
    },
    {
      "epoch": 1.8823529411764706,
      "eval_logits/chosen": -0.11206170916557312,
      "eval_logits/rejected": -0.1060233786702156,
      "eval_logps/chosen": -18.79454231262207,
      "eval_logps/rejected": -51.605831146240234,
      "eval_loss": 0.20344915986061096,
      "eval_rewards/accuracies": 0.930232584476471,
      "eval_rewards/chosen": 2.1610984802246094,
      "eval_rewards/margins": 3.3138630390167236,
      "eval_rewards/rejected": -1.152764081954956,
      "eval_runtime": 76.6954,
      "eval_samples_per_second": 2.23,
      "eval_steps_per_second": 1.121,
      "step": 180
    },
    {
      "epoch": 1.9869281045751634,
      "grad_norm": 4.388978004455566,
      "learning_rate": 4.130434782608695e-06,
      "logits/chosen": -0.12727566063404083,
      "logits/rejected": -0.12193255126476288,
      "logps/chosen": -20.929378509521484,
      "logps/rejected": -50.021419525146484,
      "loss": 0.3231,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 1.967181921005249,
      "rewards/margins": 2.968064785003662,
      "rewards/rejected": -1.000882863998413,
      "step": 190
    },
    {
      "epoch": 2.0915032679738563,
      "grad_norm": 3.108682870864868,
      "learning_rate": 4.021739130434782e-06,
      "logits/chosen": -0.15400271117687225,
      "logits/rejected": -0.14889715611934662,
      "logps/chosen": -20.020977020263672,
      "logps/rejected": -49.0496940612793,
      "loss": 0.165,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": 2.0844578742980957,
      "rewards/margins": 2.970615863800049,
      "rewards/rejected": -0.8861583471298218,
      "step": 200
    },
    {
      "epoch": 2.0915032679738563,
      "eval_logits/chosen": -0.10228858143091202,
      "eval_logits/rejected": -0.09614047408103943,
      "eval_logps/chosen": -25.265357971191406,
      "eval_logps/rejected": -58.75029373168945,
      "eval_loss": 0.17103791236877441,
      "eval_rewards/accuracies": 0.9593023061752319,
      "eval_rewards/chosen": 1.5140167474746704,
      "eval_rewards/margins": 3.3812272548675537,
      "eval_rewards/rejected": -1.8672102689743042,
      "eval_runtime": 76.7398,
      "eval_samples_per_second": 2.228,
      "eval_steps_per_second": 1.121,
      "step": 200
    },
    {
      "epoch": 2.196078431372549,
      "grad_norm": 1.1581649780273438,
      "learning_rate": 3.91304347826087e-06,
      "logits/chosen": -0.10600709915161133,
      "logits/rejected": -0.09994350373744965,
      "logps/chosen": -28.068603515625,
      "logps/rejected": -62.65039825439453,
      "loss": 0.1286,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 1.2790197134017944,
      "rewards/margins": 3.5068142414093018,
      "rewards/rejected": -2.2277944087982178,
      "step": 210
    },
    {
      "epoch": 2.3006535947712417,
      "grad_norm": 7.228103160858154,
      "learning_rate": 3.804347826086957e-06,
      "logits/chosen": -0.1612926423549652,
      "logits/rejected": -0.1542719006538391,
      "logps/chosen": -17.81163215637207,
      "logps/rejected": -55.7554817199707,
      "loss": 0.266,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 2.290220260620117,
      "rewards/margins": 3.835573673248291,
      "rewards/rejected": -1.5453532934188843,
      "step": 220
    },
    {
      "epoch": 2.3006535947712417,
      "eval_logits/chosen": -0.08572749048471451,
      "eval_logits/rejected": -0.07834314554929733,
      "eval_logps/chosen": -16.123777389526367,
      "eval_logps/rejected": -52.568172454833984,
      "eval_loss": 0.2138950228691101,
      "eval_rewards/accuracies": 0.930232584476471,
      "eval_rewards/chosen": 2.4281749725341797,
      "eval_rewards/margins": 3.677172899246216,
      "eval_rewards/rejected": -1.2489980459213257,
      "eval_runtime": 76.6691,
      "eval_samples_per_second": 2.23,
      "eval_steps_per_second": 1.122,
      "step": 220
    },
    {
      "epoch": 2.4052287581699345,
      "grad_norm": 2.1655898094177246,
      "learning_rate": 3.695652173913043e-06,
      "logits/chosen": -0.1236579641699791,
      "logits/rejected": -0.11640377342700958,
      "logps/chosen": -18.28099822998047,
      "logps/rejected": -55.26251983642578,
      "loss": 0.1805,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": 2.2774600982666016,
      "rewards/margins": 3.7921690940856934,
      "rewards/rejected": -1.5147093534469604,
      "step": 230
    },
    {
      "epoch": 2.5098039215686274,
      "grad_norm": 3.522686243057251,
      "learning_rate": 3.5869565217391305e-06,
      "logits/chosen": -0.12028801441192627,
      "logits/rejected": -0.11475691944360733,
      "logps/chosen": -20.220806121826172,
      "logps/rejected": -50.0861701965332,
      "loss": 0.2234,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": 2.0429701805114746,
      "rewards/margins": 3.0199804306030273,
      "rewards/rejected": -0.9770105481147766,
      "step": 240
    },
    {
      "epoch": 2.5098039215686274,
      "eval_logits/chosen": -0.08792821317911148,
      "eval_logits/rejected": -0.08158135414123535,
      "eval_logps/chosen": -18.402149200439453,
      "eval_logps/rejected": -51.57028579711914,
      "eval_loss": 0.18542896211147308,
      "eval_rewards/accuracies": 0.9418604373931885,
      "eval_rewards/chosen": 2.2003378868103027,
      "eval_rewards/margins": 3.3495473861694336,
      "eval_rewards/rejected": -1.1492092609405518,
      "eval_runtime": 76.7548,
      "eval_samples_per_second": 2.228,
      "eval_steps_per_second": 1.12,
      "step": 240
    },
    {
      "epoch": 2.6143790849673203,
      "grad_norm": 2.4707725048065186,
      "learning_rate": 3.4782608695652175e-06,
      "logits/chosen": -0.09261623024940491,
      "logits/rejected": -0.08608702570199966,
      "logps/chosen": -20.839956283569336,
      "logps/rejected": -55.0688362121582,
      "loss": 0.152,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": 1.9972784519195557,
      "rewards/margins": 3.4800357818603516,
      "rewards/rejected": -1.482757568359375,
      "step": 250
    },
    {
      "epoch": 2.718954248366013,
      "grad_norm": 3.7463388442993164,
      "learning_rate": 3.369565217391305e-06,
      "logits/chosen": -0.10273708403110504,
      "logits/rejected": -0.09585189074277878,
      "logps/chosen": -27.3509521484375,
      "logps/rejected": -63.99445724487305,
      "loss": 0.1878,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": 1.3538360595703125,
      "rewards/margins": 3.7591099739074707,
      "rewards/rejected": -2.405273914337158,
      "step": 260
    },
    {
      "epoch": 2.718954248366013,
      "eval_logits/chosen": -0.06562437862157822,
      "eval_logits/rejected": -0.058443356305360794,
      "eval_logps/chosen": -29.113916397094727,
      "eval_logps/rejected": -68.89017486572266,
      "eval_loss": 0.15422259271144867,
      "eval_rewards/accuracies": 0.9534883499145508,
      "eval_rewards/chosen": 1.12916100025177,
      "eval_rewards/margins": 4.010359287261963,
      "eval_rewards/rejected": -2.881197929382324,
      "eval_runtime": 76.7159,
      "eval_samples_per_second": 2.229,
      "eval_steps_per_second": 1.121,
      "step": 260
    },
    {
      "epoch": 2.8235294117647056,
      "grad_norm": 2.4779317378997803,
      "learning_rate": 3.260869565217391e-06,
      "logits/chosen": -0.08126804977655411,
      "logits/rejected": -0.07377848774194717,
      "logps/chosen": -29.53684425354004,
      "logps/rejected": -69.46771240234375,
      "loss": 0.1214,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 1.1481727361679077,
      "rewards/margins": 4.097565650939941,
      "rewards/rejected": -2.9493932723999023,
      "step": 270
    },
    {
      "epoch": 2.928104575163399,
      "grad_norm": 5.711748123168945,
      "learning_rate": 3.1521739130434784e-06,
      "logits/chosen": -0.055128227919340134,
      "logits/rejected": -0.047185707837343216,
      "logps/chosen": -16.9158935546875,
      "logps/rejected": -55.01350784301758,
      "loss": 0.1515,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": 2.360804796218872,
      "rewards/margins": 3.8613457679748535,
      "rewards/rejected": -1.5005409717559814,
      "step": 280
    },
    {
      "epoch": 2.928104575163399,
      "eval_logits/chosen": -0.06331682205200195,
      "eval_logits/rejected": -0.05519242212176323,
      "eval_logps/chosen": -14.443292617797852,
      "eval_logps/rejected": -52.58645248413086,
      "eval_loss": 0.20675985515117645,
      "eval_rewards/accuracies": 0.9418604373931885,
      "eval_rewards/chosen": 2.5962235927581787,
      "eval_rewards/margins": 3.8470497131347656,
      "eval_rewards/rejected": -1.2508265972137451,
      "eval_runtime": 76.6773,
      "eval_samples_per_second": 2.23,
      "eval_steps_per_second": 1.122,
      "step": 280
    },
    {
      "epoch": 3.0326797385620914,
      "grad_norm": NaN,
      "learning_rate": 3.054347826086957e-06,
      "logits/chosen": -0.07114674150943756,
      "logits/rejected": -0.0620611310005188,
      "logps/chosen": -16.190364837646484,
      "logps/rejected": -58.654823303222656,
      "loss": 0.1621,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 2.476926565170288,
      "rewards/margins": 4.345733165740967,
      "rewards/rejected": -1.86880624294281,
      "step": 290
    },
    {
      "epoch": 3.1372549019607843,
      "grad_norm": 3.2887370586395264,
      "learning_rate": 2.9456521739130436e-06,
      "logits/chosen": -0.06162800267338753,
      "logits/rejected": -0.05298132449388504,
      "logps/chosen": -17.659170150756836,
      "logps/rejected": -59.8099250793457,
      "loss": 0.1259,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": 2.3060736656188965,
      "rewards/margins": 4.271978855133057,
      "rewards/rejected": -1.965904951095581,
      "step": 300
    },
    {
      "epoch": 3.1372549019607843,
      "eval_logits/chosen": -0.05268235132098198,
      "eval_logits/rejected": -0.04376488924026489,
      "eval_logps/chosen": -14.083008766174316,
      "eval_logps/rejected": -54.79096984863281,
      "eval_loss": 0.15972751379013062,
      "eval_rewards/accuracies": 0.9476743936538696,
      "eval_rewards/chosen": 2.632251739501953,
      "eval_rewards/margins": 4.10352897644043,
      "eval_rewards/rejected": -1.4712772369384766,
      "eval_runtime": 76.6816,
      "eval_samples_per_second": 2.23,
      "eval_steps_per_second": 1.122,
      "step": 300
    },
    {
      "epoch": 3.241830065359477,
      "grad_norm": 1.3481501340866089,
      "learning_rate": 2.8369565217391305e-06,
      "logits/chosen": -0.09273257106542587,
      "logits/rejected": -0.08304957300424576,
      "logps/chosen": -14.50407886505127,
      "logps/rejected": -60.208778381347656,
      "loss": 0.0975,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": 2.651323080062866,
      "rewards/margins": 4.6572089195251465,
      "rewards/rejected": -2.005885601043701,
      "step": 310
    },
    {
      "epoch": 3.34640522875817,
      "grad_norm": 1.853318691253662,
      "learning_rate": 2.7282608695652175e-06,
      "logits/chosen": -0.10112150758504868,
      "logits/rejected": -0.09216316789388657,
      "logps/chosen": -17.982646942138672,
      "logps/rejected": -64.50444793701172,
      "loss": 0.1342,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": 2.268524408340454,
      "rewards/margins": 4.688179969787598,
      "rewards/rejected": -2.4196553230285645,
      "step": 320
    },
    {
      "epoch": 3.34640522875817,
      "eval_logits/chosen": -0.04029928892850876,
      "eval_logits/rejected": -0.030180998146533966,
      "eval_logps/chosen": -16.03810691833496,
      "eval_logps/rejected": -63.949459075927734,
      "eval_loss": 0.15685829520225525,
      "eval_rewards/accuracies": 0.9476743936538696,
      "eval_rewards/chosen": 2.436742067337036,
      "eval_rewards/margins": 4.8238677978515625,
      "eval_rewards/rejected": -2.3871262073516846,
      "eval_runtime": 76.6972,
      "eval_samples_per_second": 2.23,
      "eval_steps_per_second": 1.121,
      "step": 320
    }
  ],
  "logging_steps": 10,
  "max_steps": 570,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 6,
  "save_steps": 80,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": false
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 2,
  "trial_name": null,
  "trial_params": null
}