trainer_state.json

{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 1.0,
  "eval_steps": 100,
  "global_step": 478,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.0,
      "grad_norm": 8.988967747415137,
      "learning_rate": 1.0416666666666666e-08,
      "logits/chosen": -2.7205495834350586,
      "logits/rejected": -2.556058883666992,
      "logps/chosen": -245.96978759765625,
      "logps/rejected": -198.16636657714844,
      "loss": 0.6931,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 1
    },
    {
      "epoch": 0.02,
      "grad_norm": 8.972775503730468,
      "learning_rate": 1.0416666666666667e-07,
      "logits/chosen": -2.706871271133423,
      "logits/rejected": -2.728226900100708,
      "logps/chosen": -299.9327087402344,
      "logps/rejected": -282.5617370605469,
      "loss": 0.6931,
      "rewards/accuracies": 0.4444444477558136,
      "rewards/chosen": -0.0002042697014985606,
      "rewards/margins": -0.00041398685425519943,
      "rewards/rejected": 0.0002097171382047236,
      "step": 10
    },
    {
      "epoch": 0.04,
      "grad_norm": 8.863587069504122,
      "learning_rate": 2.0833333333333333e-07,
      "logits/chosen": -2.7274839878082275,
      "logits/rejected": -2.6729283332824707,
      "logps/chosen": -302.313232421875,
      "logps/rejected": -296.7969665527344,
      "loss": 0.6924,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": 0.0005617194110527635,
      "rewards/margins": 0.001121552544645965,
      "rewards/rejected": -0.0005598333082161844,
      "step": 20
    },
    {
      "epoch": 0.06,
      "grad_norm": 8.692790204491699,
      "learning_rate": 3.1249999999999997e-07,
      "logits/chosen": -2.668670415878296,
      "logits/rejected": -2.6144020557403564,
      "logps/chosen": -250.7909393310547,
      "logps/rejected": -229.73324584960938,
      "loss": 0.6879,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": 0.00824995618313551,
      "rewards/margins": 0.010436030104756355,
      "rewards/rejected": -0.0021860746201127768,
      "step": 30
    },
    {
      "epoch": 0.08,
      "grad_norm": 8.064484646395186,
      "learning_rate": 4.1666666666666667e-07,
      "logits/chosen": -2.6780455112457275,
      "logits/rejected": -2.65267276763916,
      "logps/chosen": -300.8070983886719,
      "logps/rejected": -262.91119384765625,
      "loss": 0.6788,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.040893860161304474,
      "rewards/margins": 0.03940137103199959,
      "rewards/rejected": 0.0014924891293048859,
      "step": 40
    },
    {
      "epoch": 0.1,
      "grad_norm": 10.103335337067259,
      "learning_rate": 4.999733114418725e-07,
      "logits/chosen": -2.5915541648864746,
      "logits/rejected": -2.560572624206543,
      "logps/chosen": -279.72900390625,
      "logps/rejected": -273.87506103515625,
      "loss": 0.6638,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": 0.039791837334632874,
      "rewards/margins": 0.09251143783330917,
      "rewards/rejected": -0.052719611674547195,
      "step": 50
    },
    {
      "epoch": 0.13,
      "grad_norm": 11.50466311253595,
      "learning_rate": 4.990398100856366e-07,
      "logits/chosen": -2.5843663215637207,
      "logits/rejected": -2.552401304244995,
      "logps/chosen": -267.9306335449219,
      "logps/rejected": -254.7063751220703,
      "loss": 0.6306,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.029593652114272118,
      "rewards/margins": 0.16877658665180206,
      "rewards/rejected": -0.19837023317813873,
      "step": 60
    },
    {
      "epoch": 0.15,
      "grad_norm": 11.669269626157586,
      "learning_rate": 4.967775735898179e-07,
      "logits/chosen": -2.5480875968933105,
      "logits/rejected": -2.4989640712738037,
      "logps/chosen": -279.6777038574219,
      "logps/rejected": -264.2624206542969,
      "loss": 0.6215,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.05579886958003044,
      "rewards/margins": 0.20252302289009094,
      "rewards/rejected": -0.2583218514919281,
      "step": 70
    },
    {
      "epoch": 0.17,
      "grad_norm": 14.944916591668084,
      "learning_rate": 4.931986719649298e-07,
      "logits/chosen": -2.443418025970459,
      "logits/rejected": -2.3847784996032715,
      "logps/chosen": -280.1128845214844,
      "logps/rejected": -303.17889404296875,
      "loss": 0.5961,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.059926122426986694,
      "rewards/margins": 0.2583328187465668,
      "rewards/rejected": -0.31825894117355347,
      "step": 80
    },
    {
      "epoch": 0.19,
      "grad_norm": 14.497805589416837,
      "learning_rate": 4.883222001996351e-07,
      "logits/chosen": -2.4620890617370605,
      "logits/rejected": -2.39471173286438,
      "logps/chosen": -307.2780456542969,
      "logps/rejected": -313.0209045410156,
      "loss": 0.5964,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.45970097184181213,
      "rewards/margins": 0.2758905589580536,
      "rewards/rejected": -0.7355915307998657,
      "step": 90
    },
    {
      "epoch": 0.21,
      "grad_norm": 24.280462666043302,
      "learning_rate": 4.821741763807186e-07,
      "logits/chosen": -2.2708218097686768,
      "logits/rejected": -2.1873369216918945,
      "logps/chosen": -369.2454833984375,
      "logps/rejected": -385.3036804199219,
      "loss": 0.5767,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.7413482666015625,
      "rewards/margins": 0.38414236903190613,
      "rewards/rejected": -1.125490665435791,
      "step": 100
    },
    {
      "epoch": 0.21,
      "eval_logits/chosen": -1.7623814344406128,
      "eval_logits/rejected": -1.7635838985443115,
      "eval_logps/chosen": -316.7738952636719,
      "eval_logps/rejected": -371.9847412109375,
      "eval_loss": 0.5761595964431763,
      "eval_rewards/accuracies": 0.71484375,
      "eval_rewards/chosen": -0.5417935848236084,
      "eval_rewards/margins": 0.5511539578437805,
      "eval_rewards/rejected": -1.0929476022720337,
      "eval_runtime": 91.9636,
      "eval_samples_per_second": 21.748,
      "eval_steps_per_second": 0.348,
      "step": 100
    },
    {
      "epoch": 0.23,
      "grad_norm": 21.276395436310406,
      "learning_rate": 4.747874028753375e-07,
      "logits/chosen": -1.6442521810531616,
      "logits/rejected": -1.3894662857055664,
      "logps/chosen": -327.5511169433594,
      "logps/rejected": -338.51129150390625,
      "loss": 0.5675,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.5326968431472778,
      "rewards/margins": 0.6152414083480835,
      "rewards/rejected": -1.1479383707046509,
      "step": 110
    },
    {
      "epoch": 0.25,
      "grad_norm": 31.303120852952276,
      "learning_rate": 4.662012913161997e-07,
      "logits/chosen": -1.3023754358291626,
      "logits/rejected": -1.3187789916992188,
      "logps/chosen": -339.4796142578125,
      "logps/rejected": -399.40771484375,
      "loss": 0.5346,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.6499887704849243,
      "rewards/margins": 0.667434811592102,
      "rewards/rejected": -1.3174235820770264,
      "step": 120
    },
    {
      "epoch": 0.27,
      "grad_norm": 27.028821109643975,
      "learning_rate": 4.5646165232345103e-07,
      "logits/chosen": -0.633685290813446,
      "logits/rejected": -0.5499342679977417,
      "logps/chosen": -389.3062438964844,
      "logps/rejected": -421.11199951171875,
      "loss": 0.5396,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -1.011283278465271,
      "rewards/margins": 0.5636219382286072,
      "rewards/rejected": -1.5749051570892334,
      "step": 130
    },
    {
      "epoch": 0.29,
      "grad_norm": 18.250619925903354,
      "learning_rate": 4.456204510851956e-07,
      "logits/chosen": -1.371741771697998,
      "logits/rejected": -0.8369172215461731,
      "logps/chosen": -359.0638122558594,
      "logps/rejected": -422.20184326171875,
      "loss": 0.5313,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.7278968095779419,
      "rewards/margins": 0.7612239718437195,
      "rewards/rejected": -1.4891207218170166,
      "step": 140
    },
    {
      "epoch": 0.31,
      "grad_norm": 28.832530099775454,
      "learning_rate": 4.337355301007335e-07,
      "logits/chosen": -1.2246290445327759,
      "logits/rejected": -0.8883364796638489,
      "logps/chosen": -313.1046447753906,
      "logps/rejected": -389.2489013671875,
      "loss": 0.525,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.7995457053184509,
      "rewards/margins": 0.6876853704452515,
      "rewards/rejected": -1.4872313737869263,
      "step": 150
    },
    {
      "epoch": 0.33,
      "grad_norm": 26.24455420439153,
      "learning_rate": 4.2087030056579986e-07,
      "logits/chosen": -0.6042385101318359,
      "logits/rejected": -0.29473963379859924,
      "logps/chosen": -385.8511047363281,
      "logps/rejected": -433.1654357910156,
      "loss": 0.5306,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -1.2439239025115967,
      "rewards/margins": 0.5850938558578491,
      "rewards/rejected": -1.8290178775787354,
      "step": 160
    },
    {
      "epoch": 0.36,
      "grad_norm": 21.466673908999535,
      "learning_rate": 4.070934040463998e-07,
      "logits/chosen": -1.0178248882293701,
      "logits/rejected": -0.41918307542800903,
      "logps/chosen": -386.2198181152344,
      "logps/rejected": -461.2767028808594,
      "loss": 0.5222,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.9928082227706909,
      "rewards/margins": 0.9411848783493042,
      "rewards/rejected": -1.9339931011199951,
      "step": 170
    },
    {
      "epoch": 0.38,
      "grad_norm": 24.51504275177038,
      "learning_rate": 3.9247834624635404e-07,
      "logits/chosen": -1.2531163692474365,
      "logits/rejected": -0.820971667766571,
      "logps/chosen": -364.41583251953125,
      "logps/rejected": -421.32159423828125,
      "loss": 0.5303,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8646942973136902,
      "rewards/margins": 0.7480314373970032,
      "rewards/rejected": -1.6127256155014038,
      "step": 180
    },
    {
      "epoch": 0.4,
      "grad_norm": 28.270571178832476,
      "learning_rate": 3.7710310482256523e-07,
      "logits/chosen": -0.3600873351097107,
      "logits/rejected": 0.8002864122390747,
      "logps/chosen": -387.2593994140625,
      "logps/rejected": -469.677978515625,
      "loss": 0.5083,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.9201375842094421,
      "rewards/margins": 1.1224908828735352,
      "rewards/rejected": -2.042628526687622,
      "step": 190
    },
    {
      "epoch": 0.42,
      "grad_norm": 19.44371035878493,
      "learning_rate": 3.610497133404795e-07,
      "logits/chosen": -0.532671332359314,
      "logits/rejected": -0.07579895108938217,
      "logps/chosen": -405.0849304199219,
      "logps/rejected": -445.6201171875,
      "loss": 0.5185,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -1.0358976125717163,
      "rewards/margins": 0.725575864315033,
      "rewards/rejected": -1.7614734172821045,
      "step": 200
    },
    {
      "epoch": 0.42,
      "eval_logits/chosen": -1.0192277431488037,
      "eval_logits/rejected": -0.6928036212921143,
      "eval_logps/chosen": -368.00103759765625,
      "eval_logps/rejected": -442.16375732421875,
      "eval_loss": 0.5312252044677734,
      "eval_rewards/accuracies": 0.765625,
      "eval_rewards/chosen": -1.0540651082992554,
      "eval_rewards/margins": 0.7406728863716125,
      "eval_rewards/rejected": -1.7947381734848022,
      "eval_runtime": 92.0234,
      "eval_samples_per_second": 21.734,
      "eval_steps_per_second": 0.348,
      "step": 200
    },
    {
      "epoch": 0.44,
      "grad_norm": 24.268514536481177,
      "learning_rate": 3.4440382358952115e-07,
      "logits/chosen": -0.8844076991081238,
      "logits/rejected": -0.5308712720870972,
      "logps/chosen": -373.499755859375,
      "logps/rejected": -436.2884216308594,
      "loss": 0.5033,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.951114296913147,
      "rewards/margins": 0.8899469375610352,
      "rewards/rejected": -1.8410612344741821,
      "step": 210
    },
    {
      "epoch": 0.46,
      "grad_norm": 22.946027324747533,
      "learning_rate": 3.272542485937368e-07,
      "logits/chosen": -0.5064371824264526,
      "logits/rejected": 0.23283937573432922,
      "logps/chosen": -383.8258972167969,
      "logps/rejected": -451.389892578125,
      "loss": 0.5141,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -1.0778086185455322,
      "rewards/margins": 0.8345571756362915,
      "rewards/rejected": -1.9123656749725342,
      "step": 220
    },
    {
      "epoch": 0.48,
      "grad_norm": 26.66216276685979,
      "learning_rate": 3.096924887558854e-07,
      "logits/chosen": -0.37716662883758545,
      "logits/rejected": 0.36690014600753784,
      "logps/chosen": -421.6444396972656,
      "logps/rejected": -493.63543701171875,
      "loss": 0.5256,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -1.408128261566162,
      "rewards/margins": 0.7821764945983887,
      "rewards/rejected": -2.19030499458313,
      "step": 230
    },
    {
      "epoch": 0.5,
      "grad_norm": 20.689211336064318,
      "learning_rate": 2.9181224366319943e-07,
      "logits/chosen": -0.7800679206848145,
      "logits/rejected": 0.013299018144607544,
      "logps/chosen": -401.562744140625,
      "logps/rejected": -449.37384033203125,
      "loss": 0.4774,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -1.1366697549819946,
      "rewards/margins": 0.8018972277641296,
      "rewards/rejected": -1.93856680393219,
      "step": 240
    },
    {
      "epoch": 0.52,
      "grad_norm": 30.629232248630764,
      "learning_rate": 2.7370891215954565e-07,
      "logits/chosen": -0.8394690752029419,
      "logits/rejected": -0.07095638662576675,
      "logps/chosen": -385.07147216796875,
      "logps/rejected": -440.8536682128906,
      "loss": 0.5339,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -1.0770814418792725,
      "rewards/margins": 0.7553308606147766,
      "rewards/rejected": -1.8324124813079834,
      "step": 250
    },
    {
      "epoch": 0.54,
      "grad_norm": 22.359924477710283,
      "learning_rate": 2.55479083351317e-07,
      "logits/chosen": -0.6873140931129456,
      "logits/rejected": -0.23746924102306366,
      "logps/chosen": -373.2782287597656,
      "logps/rejected": -458.29498291015625,
      "loss": 0.4991,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -1.0945470333099365,
      "rewards/margins": 0.8836139440536499,
      "rewards/rejected": -1.9781608581542969,
      "step": 260
    },
    {
      "epoch": 0.56,
      "grad_norm": 24.352377084503114,
      "learning_rate": 2.3722002126275822e-07,
      "logits/chosen": -1.4399245977401733,
      "logits/rejected": -0.610245943069458,
      "logps/chosen": -363.4355773925781,
      "logps/rejected": -440.0086364746094,
      "loss": 0.5162,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.7005236148834229,
      "rewards/margins": 1.0190067291259766,
      "rewards/rejected": -1.719530701637268,
      "step": 270
    },
    {
      "epoch": 0.59,
      "grad_norm": 27.016744918953567,
      "learning_rate": 2.19029145890313e-07,
      "logits/chosen": -1.5887335538864136,
      "logits/rejected": -0.9962981343269348,
      "logps/chosen": -386.28643798828125,
      "logps/rejected": -435.70574951171875,
      "loss": 0.5132,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.7019662857055664,
      "rewards/margins": 0.9548131227493286,
      "rewards/rejected": -1.6567795276641846,
      "step": 280
    },
    {
      "epoch": 0.61,
      "grad_norm": 26.13224420416667,
      "learning_rate": 2.0100351342479216e-07,
      "logits/chosen": -1.5274226665496826,
      "logits/rejected": -0.6684475541114807,
      "logps/chosen": -367.9737243652344,
      "logps/rejected": -433.135986328125,
      "loss": 0.518,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.8575841188430786,
      "rewards/margins": 0.9145325422286987,
      "rewards/rejected": -1.7721166610717773,
      "step": 290
    },
    {
      "epoch": 0.63,
      "grad_norm": 25.118365240619237,
      "learning_rate": 1.8323929841460178e-07,
      "logits/chosen": -1.3744531869888306,
      "logits/rejected": -0.6543569564819336,
      "logps/chosen": -374.47381591796875,
      "logps/rejected": -437.312255859375,
      "loss": 0.5166,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -1.1720250844955444,
      "rewards/margins": 0.730822741985321,
      "rewards/rejected": -1.9028480052947998,
      "step": 300
    },
    {
      "epoch": 0.63,
      "eval_logits/chosen": -1.2976927757263184,
      "eval_logits/rejected": -0.6567528247833252,
      "eval_logps/chosen": -377.8804016113281,
      "eval_logps/rejected": -469.2291564941406,
      "eval_loss": 0.5114231705665588,
      "eval_rewards/accuracies": 0.75390625,
      "eval_rewards/chosen": -1.1528586149215698,
      "eval_rewards/margins": 0.9125331044197083,
      "eval_rewards/rejected": -2.065391778945923,
      "eval_runtime": 91.9206,
      "eval_samples_per_second": 21.758,
      "eval_steps_per_second": 0.348,
      "step": 300
    },
    {
      "epoch": 0.65,
      "grad_norm": 28.40886458453979,
      "learning_rate": 1.6583128063291573e-07,
      "logits/chosen": -1.1402945518493652,
      "logits/rejected": -0.41577059030532837,
      "logps/chosen": -433.451416015625,
      "logps/rejected": -484.820068359375,
      "loss": 0.5033,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -1.3210188150405884,
      "rewards/margins": 0.8473700284957886,
      "rewards/rejected": -2.168388843536377,
      "step": 310
    },
    {
      "epoch": 0.67,
      "grad_norm": 28.30583390412626,
      "learning_rate": 1.488723393865766e-07,
      "logits/chosen": -0.8680181503295898,
      "logits/rejected": -0.25706297159194946,
      "logps/chosen": -432.978759765625,
      "logps/rejected": -507.5543518066406,
      "loss": 0.4662,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -1.384337067604065,
      "rewards/margins": 0.9784086346626282,
      "rewards/rejected": -2.362745761871338,
      "step": 320
    },
    {
      "epoch": 0.69,
      "grad_norm": 23.94955155668742,
      "learning_rate": 1.3245295796480788e-07,
      "logits/chosen": -1.0475196838378906,
      "logits/rejected": -0.02384273335337639,
      "logps/chosen": -420.4109802246094,
      "logps/rejected": -500.77447509765625,
      "loss": 0.5191,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -1.4072248935699463,
      "rewards/margins": 0.9297706484794617,
      "rewards/rejected": -2.3369956016540527,
      "step": 330
    },
    {
      "epoch": 0.71,
      "grad_norm": 26.88145170346802,
      "learning_rate": 1.1666074087171627e-07,
      "logits/chosen": -1.1260825395584106,
      "logits/rejected": -0.4045921862125397,
      "logps/chosen": -386.1277160644531,
      "logps/rejected": -480.0000915527344,
      "loss": 0.4851,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -1.2617071866989136,
      "rewards/margins": 0.9609416723251343,
      "rewards/rejected": -2.222649097442627,
      "step": 340
    },
    {
      "epoch": 0.73,
      "grad_norm": 32.127198200608866,
      "learning_rate": 1.0157994641835734e-07,
      "logits/chosen": -1.157327651977539,
      "logits/rejected": -0.39939287304878235,
      "logps/chosen": -410.75958251953125,
      "logps/rejected": -485.366455078125,
      "loss": 0.5184,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -1.3121212720870972,
      "rewards/margins": 0.8859411478042603,
      "rewards/rejected": -2.1980621814727783,
      "step": 350
    },
    {
      "epoch": 0.75,
      "grad_norm": 23.47201224640711,
      "learning_rate": 8.729103716819111e-08,
      "logits/chosen": -1.2730977535247803,
      "logits/rejected": -0.21225647628307343,
      "logps/chosen": -370.95758056640625,
      "logps/rejected": -467.11773681640625,
      "loss": 0.4815,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -1.087627649307251,
      "rewards/margins": 1.1211528778076172,
      "rewards/rejected": -2.2087807655334473,
      "step": 360
    },
    {
      "epoch": 0.77,
      "grad_norm": 25.572187464491854,
      "learning_rate": 7.387025063449081e-08,
      "logits/chosen": -1.204538345336914,
      "logits/rejected": -0.6417055726051331,
      "logps/chosen": -380.7297668457031,
      "logps/rejected": -450.6810607910156,
      "loss": 0.5012,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.9736254811286926,
      "rewards/margins": 0.8355528712272644,
      "rewards/rejected": -1.8091785907745361,
      "step": 370
    },
    {
      "epoch": 0.79,
      "grad_norm": 24.442713207132563,
      "learning_rate": 6.138919252022435e-08,
      "logits/chosen": -0.7307707667350769,
      "logits/rejected": 0.02534063719213009,
      "logps/chosen": -357.3538818359375,
      "logps/rejected": -461.0918884277344,
      "loss": 0.4757,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -1.3051798343658447,
      "rewards/margins": 0.8298481702804565,
      "rewards/rejected": -2.135028123855591,
      "step": 380
    },
    {
      "epoch": 0.82,
      "grad_norm": 31.55979798759149,
      "learning_rate": 4.991445467064689e-08,
      "logits/chosen": -0.8228236436843872,
      "logits/rejected": 0.12945809960365295,
      "logps/chosen": -374.20416259765625,
      "logps/rejected": -449.0730895996094,
      "loss": 0.5154,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -1.171090841293335,
      "rewards/margins": 0.9261363744735718,
      "rewards/rejected": -2.0972273349761963,
      "step": 390
    },
    {
      "epoch": 0.84,
      "grad_norm": 25.25735911338149,
      "learning_rate": 3.9507259776993954e-08,
      "logits/chosen": -0.7915847301483154,
      "logits/rejected": 0.3253408670425415,
      "logps/chosen": -416.76593017578125,
      "logps/rejected": -465.4586486816406,
      "loss": 0.5003,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.2516233921051025,
      "rewards/margins": 0.8954402208328247,
      "rewards/rejected": -2.147063732147217,
      "step": 400
    },
    {
      "epoch": 0.84,
      "eval_logits/chosen": -1.104366660118103,
      "eval_logits/rejected": -0.3666176497936249,
      "eval_logps/chosen": -371.3431701660156,
      "eval_logps/rejected": -469.5586242675781,
      "eval_loss": 0.5023010969161987,
      "eval_rewards/accuracies": 0.76953125,
      "eval_rewards/chosen": -1.0874860286712646,
      "eval_rewards/margins": 0.9812002778053284,
      "eval_rewards/rejected": -2.0686862468719482,
      "eval_runtime": 91.8817,
      "eval_samples_per_second": 21.767,
      "eval_steps_per_second": 0.348,
      "step": 400
    },
    {
      "epoch": 0.86,
      "grad_norm": 32.88722785510891,
      "learning_rate": 3.022313472693447e-08,
      "logits/chosen": -1.1924324035644531,
      "logits/rejected": -0.2936061918735504,
      "logps/chosen": -390.41717529296875,
      "logps/rejected": -483.9921875,
      "loss": 0.505,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -1.2092020511627197,
      "rewards/margins": 0.7791404128074646,
      "rewards/rejected": -1.98834228515625,
      "step": 410
    },
    {
      "epoch": 0.88,
      "grad_norm": 25.99174775937039,
      "learning_rate": 2.2111614344599684e-08,
      "logits/chosen": -1.1985347270965576,
      "logits/rejected": -0.4221798777580261,
      "logps/chosen": -412.852294921875,
      "logps/rejected": -503.3661193847656,
      "loss": 0.4894,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -1.1841309070587158,
      "rewards/margins": 0.8794403076171875,
      "rewards/rejected": -2.0635714530944824,
      "step": 420
    },
    {
      "epoch": 0.9,
      "grad_norm": 27.355228620514268,
      "learning_rate": 1.521597710086439e-08,
      "logits/chosen": -0.8364534378051758,
      "logits/rejected": -0.08403267711400986,
      "logps/chosen": -412.7071228027344,
      "logps/rejected": -494.9200744628906,
      "loss": 0.5068,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -1.187975525856018,
      "rewards/margins": 1.0082122087478638,
      "rewards/rejected": -2.1961874961853027,
      "step": 430
    },
    {
      "epoch": 0.92,
      "grad_norm": 27.4807676908952,
      "learning_rate": 9.57301420397924e-09,
      "logits/chosen": -1.0082981586456299,
      "logits/rejected": -0.2474936544895172,
      "logps/chosen": -454.13555908203125,
      "logps/rejected": -466.0726623535156,
      "loss": 0.4931,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -1.326205849647522,
      "rewards/margins": 0.6482640504837036,
      "rewards/rejected": -1.9744701385498047,
      "step": 440
    },
    {
      "epoch": 0.94,
      "grad_norm": 28.832598416935205,
      "learning_rate": 5.212833302556258e-09,
      "logits/chosen": -1.285772681236267,
      "logits/rejected": -0.31841930747032166,
      "logps/chosen": -393.72039794921875,
      "logps/rejected": -450.94482421875,
      "loss": 0.4967,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.1269813776016235,
      "rewards/margins": 0.8901702761650085,
      "rewards/rejected": -2.0171515941619873,
      "step": 450
    },
    {
      "epoch": 0.96,
      "grad_norm": 24.265990822993565,
      "learning_rate": 2.158697848236607e-09,
      "logits/chosen": -1.114770770072937,
      "logits/rejected": -0.18640299141407013,
      "logps/chosen": -425.88958740234375,
      "logps/rejected": -475.578125,
      "loss": 0.5203,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.2242448329925537,
      "rewards/margins": 0.7737809419631958,
      "rewards/rejected": -1.99802565574646,
      "step": 460
    },
    {
      "epoch": 0.98,
      "grad_norm": 24.93617729504318,
      "learning_rate": 4.269029751107489e-10,
      "logits/chosen": -1.35390305519104,
      "logits/rejected": -0.3422853946685791,
      "logps/chosen": -402.8608703613281,
      "logps/rejected": -424.72259521484375,
      "loss": 0.4941,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -1.0088200569152832,
      "rewards/margins": 0.8776105046272278,
      "rewards/rejected": -1.8864305019378662,
      "step": 470
    },
    {
      "epoch": 1.0,
      "step": 478,
      "total_flos": 0.0,
      "train_loss": 0.5374924991918907,
      "train_runtime": 7413.7576,
      "train_samples_per_second": 8.246,
      "train_steps_per_second": 0.064
    }
  ],
  "logging_steps": 10,
  "max_steps": 478,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 100,
  "total_flos": 0.0,
  "train_batch_size": 8,
  "trial_name": null,
  "trial_params": null
}