{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 1.0,
  "eval_steps": 100,
  "global_step": 478,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "debug/losses": 0.36003828048706055,
      "debug/policy_weights": 0.5194254517555237,
      "debug/raw_losses": 0.6931471824645996,
      "epoch": 0.0,
      "grad_norm": 5.3582401111914875,
      "learning_rate": 1.0416666666666666e-08,
      "logits/chosen": -2.888590097427368,
      "logits/rejected": -2.871957540512085,
      "logps/chosen": -261.88616943359375,
      "logps/rejected": -355.31793212890625,
      "loss": 0.3648,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 1
    },
    {
      "debug/losses": 0.3548344671726227,
      "debug/policy_weights": 0.5118452906608582,
      "debug/raw_losses": 0.6932389736175537,
      "epoch": 0.02,
      "grad_norm": 5.014402110225361,
      "learning_rate": 1.0416666666666667e-07,
      "logits/chosen": -2.7435038089752197,
      "logits/rejected": -2.7258925437927246,
      "logps/chosen": -256.5353088378906,
      "logps/rejected": -244.49546813964844,
      "loss": 0.3619,
      "rewards/accuracies": 0.4236111044883728,
      "rewards/chosen": -8.211319800466299e-05,
      "rewards/margins": -0.00017834355821833014,
      "rewards/rejected": 9.623030200600624e-05,
      "step": 10
    },
    {
      "debug/losses": 0.3572663962841034,
      "debug/policy_weights": 0.5156484842300415,
      "debug/raw_losses": 0.6928108930587769,
      "epoch": 0.04,
      "grad_norm": 4.715098712970233,
      "learning_rate": 2.0833333333333333e-07,
      "logits/chosen": -2.8007731437683105,
      "logits/rejected": -2.7729618549346924,
      "logps/chosen": -271.37860107421875,
      "logps/rejected": -288.34271240234375,
      "loss": 0.3584,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": 0.0006374852382577956,
      "rewards/margins": 0.0006833201623521745,
      "rewards/rejected": -4.583501868182793e-05,
      "step": 20
    },
    {
      "debug/losses": 0.35951218008995056,
      "debug/policy_weights": 0.521370530128479,
      "debug/raw_losses": 0.6895479559898376,
      "epoch": 0.06,
      "grad_norm": 4.7982052271779825,
      "learning_rate": 3.1249999999999997e-07,
      "logits/chosen": -2.810387134552002,
      "logits/rejected": -2.773030996322632,
      "logps/chosen": -291.4071350097656,
      "logps/rejected": -258.0267639160156,
      "loss": 0.3579,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": 0.004485988523811102,
      "rewards/margins": 0.00730940094217658,
      "rewards/rejected": -0.002823411952704191,
      "step": 30
    },
    {
      "debug/losses": 0.35061347484588623,
      "debug/policy_weights": 0.5189496278762817,
      "debug/raw_losses": 0.6759483218193054,
      "epoch": 0.08,
      "grad_norm": 4.841334528975048,
      "learning_rate": 4.1666666666666667e-07,
      "logits/chosen": -2.7438368797302246,
      "logits/rejected": -2.7376155853271484,
      "logps/chosen": -252.50436401367188,
      "logps/rejected": -240.8105010986328,
      "loss": 0.3573,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": 0.015405787155032158,
      "rewards/margins": 0.03552238270640373,
      "rewards/rejected": -0.020116599276661873,
      "step": 40
    },
    {
      "debug/losses": 0.34173911809921265,
      "debug/policy_weights": 0.5125550627708435,
      "debug/raw_losses": 0.666517436504364,
      "epoch": 0.1,
      "grad_norm": 5.135483442753442,
      "learning_rate": 4.999733114418725e-07,
      "logits/chosen": -2.6830456256866455,
      "logits/rejected": -2.6514101028442383,
      "logps/chosen": -269.3827819824219,
      "logps/rejected": -287.2772216796875,
      "loss": 0.3437,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": 0.0019381232559680939,
      "rewards/margins": 0.05897800251841545,
      "rewards/rejected": -0.05703987926244736,
      "step": 50
    },
    {
      "debug/losses": 0.3377828598022461,
      "debug/policy_weights": 0.5224286317825317,
      "debug/raw_losses": 0.6448591947555542,
      "epoch": 0.13,
      "grad_norm": 5.5606740216520905,
      "learning_rate": 4.990398100856366e-07,
      "logits/chosen": -2.745028018951416,
      "logits/rejected": -2.713804244995117,
      "logps/chosen": -266.05279541015625,
      "logps/rejected": -305.28271484375,
      "loss": 0.3305,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.02193317376077175,
      "rewards/margins": 0.1177050918340683,
      "rewards/rejected": -0.1396382749080658,
      "step": 60
    },
    {
      "debug/losses": 0.28029459714889526,
      "debug/policy_weights": 0.47101029753685,
      "debug/raw_losses": 0.6030661463737488,
      "epoch": 0.15,
      "grad_norm": 6.284287160526739,
      "learning_rate": 4.967775735898179e-07,
      "logits/chosen": -2.7072081565856934,
      "logits/rejected": -2.7113521099090576,
      "logps/chosen": -272.530029296875,
      "logps/rejected": -289.87713623046875,
      "loss": 0.2826,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.1557193100452423,
      "rewards/margins": 0.24685576558113098,
      "rewards/rejected": -0.4025750756263733,
      "step": 70
    },
    {
      "debug/losses": 0.26233458518981934,
      "debug/policy_weights": 0.42553257942199707,
      "debug/raw_losses": 0.6218484044075012,
      "epoch": 0.17,
      "grad_norm": 6.8893773770621305,
      "learning_rate": 4.931986719649298e-07,
      "logits/chosen": -2.800093412399292,
      "logits/rejected": -2.7768216133117676,
      "logps/chosen": -360.3621520996094,
      "logps/rejected": -329.7616882324219,
      "loss": 0.2516,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.45188745856285095,
      "rewards/margins": 0.26955321431159973,
      "rewards/rejected": -0.7214406728744507,
      "step": 80
    },
    {
      "debug/losses": 0.18998686969280243,
      "debug/policy_weights": 0.34327536821365356,
      "debug/raw_losses": 0.5527450442314148,
      "epoch": 0.19,
      "grad_norm": 6.5870040213670435,
      "learning_rate": 4.883222001996351e-07,
      "logits/chosen": -2.724050998687744,
      "logits/rejected": -2.690390110015869,
      "logps/chosen": -303.0259704589844,
      "logps/rejected": -344.49505615234375,
      "loss": 0.2198,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.5087189078330994,
      "rewards/margins": 0.4743797779083252,
      "rewards/rejected": -0.9830986857414246,
      "step": 90
    },
    {
      "debug/losses": 0.16577747464179993,
      "debug/policy_weights": 0.29914408922195435,
      "debug/raw_losses": 0.5552432537078857,
      "epoch": 0.21,
      "grad_norm": 7.249323730073561,
      "learning_rate": 4.821741763807186e-07,
      "logits/chosen": -2.6653432846069336,
      "logits/rejected": -2.658141613006592,
      "logps/chosen": -325.9325866699219,
      "logps/rejected": -375.12127685546875,
      "loss": 0.1781,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.7297804951667786,
      "rewards/margins": 0.511761486530304,
      "rewards/rejected": -1.2415419816970825,
      "step": 100
    },
    {
      "epoch": 0.21,
      "eval_debug/losses": 0.19525206089019775,
      "eval_debug/policy_weights": 0.3511899709701538,
      "eval_debug/raw_losses": 0.5590351819992065,
      "eval_logits/chosen": -2.70804500579834,
      "eval_logits/rejected": -2.6910133361816406,
      "eval_logps/chosen": -319.9805603027344,
      "eval_logps/rejected": -373.18670654296875,
      "eval_loss": 0.20066915452480316,
      "eval_rewards/accuracies": 0.734375,
      "eval_rewards/chosen": -0.6478448510169983,
      "eval_rewards/margins": 0.5214487910270691,
      "eval_rewards/rejected": -1.1692936420440674,
      "eval_runtime": 43.2006,
      "eval_samples_per_second": 46.296,
      "eval_steps_per_second": 0.741,
      "step": 100
    },
    {
      "debug/losses": 0.1987285315990448,
      "debug/policy_weights": 0.34678488969802856,
      "debug/raw_losses": 0.5721240043640137,
      "epoch": 0.23,
      "grad_norm": 6.9123559213202554,
      "learning_rate": 4.747874028753375e-07,
      "logits/chosen": -2.7045469284057617,
      "logits/rejected": -2.672999858856201,
      "logps/chosen": -369.9187927246094,
      "logps/rejected": -373.6562194824219,
      "loss": 0.1986,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.7313551902770996,
      "rewards/margins": 0.47859472036361694,
      "rewards/rejected": -1.2099498510360718,
      "step": 110
    },
    {
      "debug/losses": 0.18334314227104187,
      "debug/policy_weights": 0.3257436156272888,
      "debug/raw_losses": 0.5537685751914978,
      "epoch": 0.25,
      "grad_norm": 7.316158832496888,
      "learning_rate": 4.662012913161997e-07,
      "logits/chosen": -2.6026859283447266,
      "logits/rejected": -2.600971221923828,
      "logps/chosen": -346.50555419921875,
      "logps/rejected": -379.53887939453125,
      "loss": 0.1846,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.7285534739494324,
      "rewards/margins": 0.5237767100334167,
      "rewards/rejected": -1.2523301839828491,
      "step": 120
    },
    {
      "debug/losses": 0.17840833961963654,
      "debug/policy_weights": 0.3271704614162445,
      "debug/raw_losses": 0.5529276132583618,
      "epoch": 0.27,
      "grad_norm": 6.11630399118371,
      "learning_rate": 4.5646165232345103e-07,
      "logits/chosen": -2.5919995307922363,
      "logits/rejected": -2.598027467727661,
      "logps/chosen": -346.44610595703125,
      "logps/rejected": -391.03778076171875,
      "loss": 0.1758,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.7799400687217712,
      "rewards/margins": 0.5067718029022217,
      "rewards/rejected": -1.2867119312286377,
      "step": 130
    },
    {
      "debug/losses": 0.18315494060516357,
      "debug/policy_weights": 0.3452090620994568,
      "debug/raw_losses": 0.56131511926651,
      "epoch": 0.29,
      "grad_norm": 6.845717287191781,
      "learning_rate": 4.456204510851956e-07,
      "logits/chosen": -2.572934627532959,
      "logits/rejected": -2.5713508129119873,
      "logps/chosen": -390.03253173828125,
      "logps/rejected": -440.16461181640625,
      "loss": 0.1668,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.8578819036483765,
      "rewards/margins": 0.5912094712257385,
      "rewards/rejected": -1.4490914344787598,
      "step": 140
    },
    {
      "debug/losses": 0.19362398982048035,
      "debug/policy_weights": 0.33687275648117065,
      "debug/raw_losses": 0.559166669845581,
      "epoch": 0.31,
      "grad_norm": 7.029648911332738,
      "learning_rate": 4.337355301007335e-07,
      "logits/chosen": -2.512932300567627,
      "logits/rejected": -2.4797143936157227,
      "logps/chosen": -357.82464599609375,
      "logps/rejected": -402.2618713378906,
      "loss": 0.1771,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.8432974815368652,
      "rewards/margins": 0.48099932074546814,
      "rewards/rejected": -1.3242968320846558,
      "step": 150
    },
    {
      "debug/losses": 0.14652669429779053,
      "debug/policy_weights": 0.28093475103378296,
      "debug/raw_losses": 0.5318498611450195,
      "epoch": 0.33,
      "grad_norm": 6.358610241024355,
      "learning_rate": 4.2087030056579986e-07,
      "logits/chosen": -2.464831590652466,
      "logits/rejected": -2.4293084144592285,
      "logps/chosen": -370.14971923828125,
      "logps/rejected": -408.4079284667969,
      "loss": 0.1583,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -1.0606319904327393,
      "rewards/margins": 0.596233606338501,
      "rewards/rejected": -1.6568657159805298,
      "step": 160
    },
    {
      "debug/losses": 0.1372167319059372,
      "debug/policy_weights": 0.2504947781562805,
      "debug/raw_losses": 0.6000548005104065,
      "epoch": 0.36,
      "grad_norm": 7.75801911972225,
      "learning_rate": 4.070934040463998e-07,
      "logits/chosen": -2.361786365509033,
      "logits/rejected": -2.332592010498047,
      "logps/chosen": -389.86767578125,
      "logps/rejected": -423.97698974609375,
      "loss": 0.1417,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.4737131595611572,
      "rewards/margins": 0.511743426322937,
      "rewards/rejected": -1.9854564666748047,
      "step": 170
    },
    {
      "debug/losses": 0.11748126894235611,
      "debug/policy_weights": 0.2296670377254486,
      "debug/raw_losses": 0.5410301089286804,
      "epoch": 0.38,
      "grad_norm": 6.142786182484198,
      "learning_rate": 3.9247834624635404e-07,
      "logits/chosen": -2.3254575729370117,
      "logits/rejected": -2.3115153312683105,
      "logps/chosen": -391.5261535644531,
      "logps/rejected": -429.21875,
      "loss": 0.1327,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -1.446401834487915,
      "rewards/margins": 0.6310542821884155,
      "rewards/rejected": -2.07745623588562,
      "step": 180
    },
    {
      "debug/losses": 0.16298121213912964,
      "debug/policy_weights": 0.2831325829029083,
      "debug/raw_losses": 0.5709558725357056,
      "epoch": 0.4,
      "grad_norm": 6.941055961208667,
      "learning_rate": 3.7710310482256523e-07,
      "logits/chosen": -2.378763198852539,
      "logits/rejected": -2.3435521125793457,
      "logps/chosen": -381.296630859375,
      "logps/rejected": -435.13494873046875,
      "loss": 0.1447,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.178696870803833,
      "rewards/margins": 0.6175613403320312,
      "rewards/rejected": -1.7962583303451538,
      "step": 190
    },
    {
      "debug/losses": 0.1531984657049179,
      "debug/policy_weights": 0.27346447110176086,
      "debug/raw_losses": 0.5578541159629822,
      "epoch": 0.42,
      "grad_norm": 9.54639374222291,
      "learning_rate": 3.610497133404795e-07,
      "logits/chosen": -2.3584351539611816,
      "logits/rejected": -2.362753391265869,
      "logps/chosen": -359.6307067871094,
      "logps/rejected": -422.2249450683594,
      "loss": 0.1616,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -1.0809690952301025,
      "rewards/margins": 0.6879221796989441,
      "rewards/rejected": -1.7688913345336914,
      "step": 200
    },
    {
      "epoch": 0.42,
      "eval_debug/losses": 0.16714033484458923,
      "eval_debug/policy_weights": 0.3173619508743286,
      "eval_debug/raw_losses": 0.5079437494277954,
      "eval_logits/chosen": -2.44987154006958,
      "eval_logits/rejected": -2.4276504516601562,
      "eval_logps/chosen": -343.49139404296875,
      "eval_logps/rejected": -416.28436279296875,
      "eval_loss": 0.16694626212120056,
      "eval_rewards/accuracies": 0.7109375,
      "eval_rewards/chosen": -0.8829529881477356,
      "eval_rewards/margins": 0.7173169851303101,
      "eval_rewards/rejected": -1.6002700328826904,
      "eval_runtime": 43.2126,
      "eval_samples_per_second": 46.283,
      "eval_steps_per_second": 0.741,
      "step": 200
    },
    {
      "debug/losses": 0.14734311401844025,
      "debug/policy_weights": 0.2870820164680481,
      "debug/raw_losses": 0.5041711330413818,
      "epoch": 0.44,
      "grad_norm": 7.104709810051969,
      "learning_rate": 3.4440382358952115e-07,
      "logits/chosen": -2.3574612140655518,
      "logits/rejected": -2.326927900314331,
      "logps/chosen": -381.2717590332031,
      "logps/rejected": -413.890869140625,
      "loss": 0.1636,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.9982665777206421,
      "rewards/margins": 0.6929588317871094,
      "rewards/rejected": -1.6912254095077515,
      "step": 210
    },
    {
      "debug/losses": 0.13881739974021912,
      "debug/policy_weights": 0.27268368005752563,
      "debug/raw_losses": 0.5210752487182617,
      "epoch": 0.46,
      "grad_norm": 8.340698654196919,
      "learning_rate": 3.272542485937368e-07,
      "logits/chosen": -2.3073763847351074,
      "logits/rejected": -2.288245677947998,
      "logps/chosen": -370.5925598144531,
      "logps/rejected": -423.53021240234375,
      "loss": 0.1549,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.1713535785675049,
      "rewards/margins": 0.7361623644828796,
      "rewards/rejected": -1.9075158834457397,
      "step": 220
    },
    {
      "debug/losses": 0.15816743671894073,
      "debug/policy_weights": 0.2902621626853943,
      "debug/raw_losses": 0.509421706199646,
      "epoch": 0.48,
      "grad_norm": 7.2612463171873225,
      "learning_rate": 3.096924887558854e-07,
      "logits/chosen": -2.339087724685669,
      "logits/rejected": -2.31463360786438,
      "logps/chosen": -342.2999572753906,
      "logps/rejected": -425.86181640625,
      "loss": 0.169,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -1.0157617330551147,
      "rewards/margins": 0.8210276365280151,
      "rewards/rejected": -1.8367893695831299,
      "step": 230
    },
    {
      "debug/losses": 0.15576457977294922,
      "debug/policy_weights": 0.29614073038101196,
      "debug/raw_losses": 0.4828997552394867,
      "epoch": 0.5,
      "grad_norm": 7.345094511114747,
      "learning_rate": 2.9181224366319943e-07,
      "logits/chosen": -2.3611202239990234,
      "logits/rejected": -2.317119836807251,
      "logps/chosen": -368.0779113769531,
      "logps/rejected": -434.31549072265625,
      "loss": 0.1514,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -1.1054710149765015,
      "rewards/margins": 0.8346185684204102,
      "rewards/rejected": -1.9400895833969116,
      "step": 240
    },
    {
      "debug/losses": 0.12131881713867188,
      "debug/policy_weights": 0.2461313009262085,
      "debug/raw_losses": 0.46923789381980896,
      "epoch": 0.52,
      "grad_norm": 6.902697741869489,
      "learning_rate": 2.7370891215954565e-07,
      "logits/chosen": -2.2654645442962646,
      "logits/rejected": -2.2106964588165283,
      "logps/chosen": -434.3672790527344,
      "logps/rejected": -491.00750732421875,
      "loss": 0.1285,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -1.4400012493133545,
      "rewards/margins": 0.9503369331359863,
      "rewards/rejected": -2.390338182449341,
      "step": 250
    },
    {
      "debug/losses": 0.1332876831293106,
      "debug/policy_weights": 0.27705293893814087,
      "debug/raw_losses": 0.5086624622344971,
      "epoch": 0.54,
      "grad_norm": 7.179715362550834,
      "learning_rate": 2.55479083351317e-07,
      "logits/chosen": -2.316641330718994,
      "logits/rejected": -2.3014323711395264,
      "logps/chosen": -410.114990234375,
      "logps/rejected": -454.8689880371094,
      "loss": 0.1406,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -1.2414360046386719,
      "rewards/margins": 0.8403455018997192,
      "rewards/rejected": -2.0817813873291016,
      "step": 260
    },
    {
      "debug/losses": 0.168529212474823,
      "debug/policy_weights": 0.2988266348838806,
      "debug/raw_losses": 0.5682334899902344,
      "epoch": 0.56,
      "grad_norm": 7.216808591858915,
      "learning_rate": 2.3722002126275822e-07,
      "logits/chosen": -2.3377718925476074,
      "logits/rejected": -2.3152408599853516,
      "logps/chosen": -379.34661865234375,
      "logps/rejected": -430.7207946777344,
      "loss": 0.1519,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.0926826000213623,
      "rewards/margins": 0.6951464414596558,
      "rewards/rejected": -1.7878293991088867,
      "step": 270
    },
    {
      "debug/losses": 0.15156733989715576,
      "debug/policy_weights": 0.2692407965660095,
      "debug/raw_losses": 0.5413981080055237,
      "epoch": 0.59,
      "grad_norm": 9.464551455783996,
      "learning_rate": 2.19029145890313e-07,
      "logits/chosen": -2.295574426651001,
      "logits/rejected": -2.2587995529174805,
      "logps/chosen": -383.4193420410156,
      "logps/rejected": -444.91680908203125,
      "loss": 0.1498,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -1.302856683731079,
      "rewards/margins": 0.7735105156898499,
      "rewards/rejected": -2.076367139816284,
      "step": 280
    },
    {
      "debug/losses": 0.14380133152008057,
      "debug/policy_weights": 0.26454827189445496,
      "debug/raw_losses": 0.5405210256576538,
      "epoch": 0.61,
      "grad_norm": 7.542791313020671,
      "learning_rate": 2.0100351342479216e-07,
      "logits/chosen": -2.3062846660614014,
      "logits/rejected": -2.2651028633117676,
      "logps/chosen": -386.890625,
      "logps/rejected": -450.9478454589844,
      "loss": 0.1352,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -1.4489481449127197,
      "rewards/margins": 0.7288294434547424,
      "rewards/rejected": -2.1777775287628174,
      "step": 290
    },
    {
      "debug/losses": 0.14622345566749573,
      "debug/policy_weights": 0.2636463940143585,
      "debug/raw_losses": 0.536266028881073,
      "epoch": 0.63,
      "grad_norm": 7.687631612230176,
      "learning_rate": 1.8323929841460178e-07,
      "logits/chosen": -2.2653095722198486,
      "logits/rejected": -2.2190821170806885,
      "logps/chosen": -444.52960205078125,
      "logps/rejected": -489.06817626953125,
      "loss": 0.1343,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.5472103357315063,
      "rewards/margins": 0.8187928199768066,
      "rewards/rejected": -2.3660032749176025,
      "step": 300
    },
    {
      "epoch": 0.63,
      "eval_debug/losses": 0.13651318848133087,
      "eval_debug/policy_weights": 0.26656585931777954,
      "eval_debug/raw_losses": 0.49531808495521545,
      "eval_logits/chosen": -2.2618136405944824,
      "eval_logits/rejected": -2.228300094604492,
      "eval_logps/chosen": -405.40423583984375,
      "eval_logps/rejected": -493.411376953125,
      "eval_loss": 0.13676287233829498,
      "eval_rewards/accuracies": 0.7578125,
      "eval_rewards/chosen": -1.5020815134048462,
      "eval_rewards/margins": 0.8694581985473633,
      "eval_rewards/rejected": -2.371539831161499,
      "eval_runtime": 43.5081,
      "eval_samples_per_second": 45.968,
      "eval_steps_per_second": 0.735,
      "step": 300
    },
    {
      "debug/losses": 0.1321336030960083,
      "debug/policy_weights": 0.2647426724433899,
      "debug/raw_losses": 0.4956802725791931,
      "epoch": 0.65,
      "grad_norm": 7.197171827503445,
      "learning_rate": 1.6583128063291573e-07,
      "logits/chosen": -2.186096429824829,
      "logits/rejected": -2.185868501663208,
      "logps/chosen": -443.67803955078125,
      "logps/rejected": -486.25091552734375,
      "loss": 0.1268,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -1.5781536102294922,
      "rewards/margins": 0.7728461027145386,
      "rewards/rejected": -2.3510000705718994,
      "step": 310
    },
    {
      "debug/losses": 0.13546636700630188,
      "debug/policy_weights": 0.2598869204521179,
      "debug/raw_losses": 0.5187257528305054,
      "epoch": 0.67,
      "grad_norm": 7.989288502873238,
      "learning_rate": 1.488723393865766e-07,
      "logits/chosen": -2.2410237789154053,
      "logits/rejected": -2.2062346935272217,
      "logps/chosen": -433.8523864746094,
      "logps/rejected": -467.3929138183594,
      "loss": 0.1308,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -1.493381142616272,
      "rewards/margins": 0.7878578901290894,
      "rewards/rejected": -2.2812390327453613,
      "step": 320
    },
    {
      "debug/losses": 0.14906349778175354,
      "debug/policy_weights": 0.2727554440498352,
      "debug/raw_losses": 0.5552006959915161,
      "epoch": 0.69,
      "grad_norm": 6.170487059058624,
      "learning_rate": 1.3245295796480788e-07,
      "logits/chosen": -2.236531972885132,
      "logits/rejected": -2.1864519119262695,
      "logps/chosen": -390.4073791503906,
      "logps/rejected": -459.62347412109375,
      "loss": 0.1411,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -1.373223066329956,
      "rewards/margins": 0.6836794018745422,
      "rewards/rejected": -2.0569024085998535,
      "step": 330
    },
    {
      "debug/losses": 0.1494503617286682,
      "debug/policy_weights": 0.28068724274635315,
      "debug/raw_losses": 0.4996885359287262,
      "epoch": 0.71,
      "grad_norm": 9.416700000227674,
      "learning_rate": 1.1666074087171627e-07,
      "logits/chosen": -2.2197887897491455,
      "logits/rejected": -2.1604628562927246,
      "logps/chosen": -418.9659118652344,
      "logps/rejected": -511.0225524902344,
      "loss": 0.1446,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -1.3679134845733643,
      "rewards/margins": 1.0408259630203247,
      "rewards/rejected": -2.4087395668029785,
      "step": 340
    },
    {
      "debug/losses": 0.12328938394784927,
      "debug/policy_weights": 0.24812527000904083,
      "debug/raw_losses": 0.48738375306129456,
      "epoch": 0.73,
      "grad_norm": 8.40666805347326,
      "learning_rate": 1.0157994641835734e-07,
      "logits/chosen": -2.172837018966675,
      "logits/rejected": -2.1325600147247314,
      "logps/chosen": -398.5860900878906,
      "logps/rejected": -471.38397216796875,
      "loss": 0.1286,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -1.4896047115325928,
      "rewards/margins": 0.9131780862808228,
      "rewards/rejected": -2.402782917022705,
      "step": 350
    },
    {
      "debug/losses": 0.1280422806739807,
      "debug/policy_weights": 0.2567124366760254,
      "debug/raw_losses": 0.5079829692840576,
      "epoch": 0.75,
      "grad_norm": 7.5652554505895955,
      "learning_rate": 8.729103716819111e-08,
      "logits/chosen": -2.230677366256714,
      "logits/rejected": -2.138909101486206,
      "logps/chosen": -448.53179931640625,
      "logps/rejected": -494.7422790527344,
      "loss": 0.1317,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -1.5306211709976196,
      "rewards/margins": 0.8633092641830444,
      "rewards/rejected": -2.393930435180664,
      "step": 360
    },
    {
      "debug/losses": 0.13079939782619476,
      "debug/policy_weights": 0.23427358269691467,
      "debug/raw_losses": 0.5890456438064575,
      "epoch": 0.77,
      "grad_norm": 9.87793971826204,
      "learning_rate": 7.387025063449081e-08,
      "logits/chosen": -2.167722225189209,
      "logits/rejected": -2.1336138248443604,
      "logps/chosen": -432.41314697265625,
      "logps/rejected": -470.364501953125,
      "loss": 0.1313,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -1.6386140584945679,
      "rewards/margins": 0.7472943067550659,
      "rewards/rejected": -2.385908365249634,
      "step": 370
    },
    {
      "debug/losses": 0.1057448610663414,
      "debug/policy_weights": 0.2194373905658722,
      "debug/raw_losses": 0.5027541518211365,
      "epoch": 0.79,
      "grad_norm": 7.002597773553437,
      "learning_rate": 6.138919252022435e-08,
      "logits/chosen": -2.093229055404663,
      "logits/rejected": -2.0845484733581543,
      "logps/chosen": -409.2686462402344,
      "logps/rejected": -525.614013671875,
      "loss": 0.1212,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.7782268524169922,
      "rewards/margins": 0.96544349193573,
      "rewards/rejected": -2.743670701980591,
      "step": 380
    },
    {
      "debug/losses": 0.14649717509746552,
      "debug/policy_weights": 0.27705007791519165,
      "debug/raw_losses": 0.5246635675430298,
      "epoch": 0.82,
      "grad_norm": 7.870379900484857,
      "learning_rate": 4.991445467064689e-08,
      "logits/chosen": -2.1749279499053955,
      "logits/rejected": -2.138232946395874,
      "logps/chosen": -439.6005859375,
      "logps/rejected": -510.23260498046875,
      "loss": 0.1384,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -1.4645540714263916,
      "rewards/margins": 0.8761545419692993,
      "rewards/rejected": -2.3407082557678223,
      "step": 390
    },
    {
      "debug/losses": 0.1481957882642746,
      "debug/policy_weights": 0.25738272070884705,
      "debug/raw_losses": 0.5208276510238647,
      "epoch": 0.84,
      "grad_norm": 7.317947546801369,
      "learning_rate": 3.9507259776993954e-08,
      "logits/chosen": -2.1648905277252197,
      "logits/rejected": -2.1050162315368652,
      "logps/chosen": -416.4757385253906,
      "logps/rejected": -496.00927734375,
      "loss": 0.1398,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.536012887954712,
      "rewards/margins": 0.8311750292778015,
      "rewards/rejected": -2.367187976837158,
      "step": 400
    },
    {
      "epoch": 0.84,
      "eval_debug/losses": 0.13974608480930328,
      "eval_debug/policy_weights": 0.28152135014533997,
      "eval_debug/raw_losses": 0.47273018956184387,
      "eval_logits/chosen": -2.2036194801330566,
      "eval_logits/rejected": -2.1694607734680176,
      "eval_logps/chosen": -386.73663330078125,
      "eval_logps/rejected": -483.9327392578125,
      "eval_loss": 0.14222779870033264,
      "eval_rewards/accuracies": 0.76171875,
      "eval_rewards/chosen": -1.3154058456420898,
      "eval_rewards/margins": 0.9613481163978577,
      "eval_rewards/rejected": -2.2767536640167236,
      "eval_runtime": 98.9464,
      "eval_samples_per_second": 20.213,
      "eval_steps_per_second": 0.323,
      "step": 400
    },
    {
      "debug/losses": 0.13129359483718872,
      "debug/policy_weights": 0.25963759422302246,
      "debug/raw_losses": 0.5042656660079956,
      "epoch": 0.86,
      "grad_norm": 7.093812921964164,
      "learning_rate": 3.022313472693447e-08,
      "logits/chosen": -2.2121942043304443,
      "logits/rejected": -2.1647019386291504,
      "logps/chosen": -432.4383850097656,
      "logps/rejected": -489.4990234375,
      "loss": 0.1404,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.4152858257293701,
      "rewards/margins": 0.8453054428100586,
      "rewards/rejected": -2.260591506958008,
      "step": 410
    },
    {
      "debug/losses": 0.15901803970336914,
      "debug/policy_weights": 0.2908507287502289,
      "debug/raw_losses": 0.5441660284996033,
      "epoch": 0.88,
      "grad_norm": 7.194889894590537,
      "learning_rate": 2.2111614344599684e-08,
      "logits/chosen": -2.1668639183044434,
      "logits/rejected": -2.1467199325561523,
      "logps/chosen": -427.835693359375,
      "logps/rejected": -492.3506774902344,
      "loss": 0.1412,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -1.4115890264511108,
      "rewards/margins": 0.840238094329834,
      "rewards/rejected": -2.2518272399902344,
      "step": 420
    },
    {
      "debug/losses": 0.1401083916425705,
      "debug/policy_weights": 0.2716709077358246,
      "debug/raw_losses": 0.49931907653808594,
      "epoch": 0.9,
      "grad_norm": 6.971935536096578,
      "learning_rate": 1.521597710086439e-08,
      "logits/chosen": -2.106959342956543,
      "logits/rejected": -2.051405668258667,
      "logps/chosen": -412.853759765625,
      "logps/rejected": -481.4043884277344,
      "loss": 0.1291,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -1.4160857200622559,
      "rewards/margins": 0.8900654911994934,
      "rewards/rejected": -2.3061509132385254,
      "step": 430
    },
    {
      "debug/losses": 0.14032089710235596,
      "debug/policy_weights": 0.28002962470054626,
      "debug/raw_losses": 0.4943667948246002,
      "epoch": 0.92,
      "grad_norm": 7.598363791526828,
      "learning_rate": 9.57301420397924e-09,
      "logits/chosen": -2.1854939460754395,
      "logits/rejected": -2.1297545433044434,
      "logps/chosen": -409.72625732421875,
      "logps/rejected": -484.60546875,
      "loss": 0.1401,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -1.338160753250122,
      "rewards/margins": 0.8834611773490906,
      "rewards/rejected": -2.2216219902038574,
      "step": 440
    },
    {
      "debug/losses": 0.1532571017742157,
      "debug/policy_weights": 0.28182727098464966,
      "debug/raw_losses": 0.5403295159339905,
      "epoch": 0.94,
      "grad_norm": 7.420969377583743,
      "learning_rate": 5.212833302556258e-09,
      "logits/chosen": -2.108179807662964,
      "logits/rejected": -2.0896925926208496,
      "logps/chosen": -421.1512756347656,
      "logps/rejected": -522.5540771484375,
      "loss": 0.1398,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.3998409509658813,
      "rewards/margins": 0.8198366165161133,
      "rewards/rejected": -2.219677448272705,
      "step": 450
    },
    {
      "debug/losses": 0.14014151692390442,
      "debug/policy_weights": 0.27090734243392944,
      "debug/raw_losses": 0.5257419347763062,
      "epoch": 0.96,
      "grad_norm": 6.950959627699968,
      "learning_rate": 2.158697848236607e-09,
      "logits/chosen": -2.1658027172088623,
      "logits/rejected": -2.1314563751220703,
      "logps/chosen": -402.38995361328125,
      "logps/rejected": -448.8475646972656,
      "loss": 0.1362,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -1.3720080852508545,
      "rewards/margins": 0.7829836010932922,
      "rewards/rejected": -2.154991626739502,
      "step": 460
    },
    {
      "debug/losses": 0.14437520503997803,
      "debug/policy_weights": 0.27037832140922546,
      "debug/raw_losses": 0.5343233942985535,
      "epoch": 0.98,
      "grad_norm": 8.44671954335882,
      "learning_rate": 4.269029751107489e-10,
      "logits/chosen": -2.151285171508789,
      "logits/rejected": -2.119642972946167,
      "logps/chosen": -403.640625,
      "logps/rejected": -483.1455078125,
      "loss": 0.1371,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -1.3858269453048706,
      "rewards/margins": 0.757634699344635,
      "rewards/rejected": -2.1434617042541504,
      "step": 470
    },
    {
      "epoch": 1.0,
      "step": 478,
      "total_flos": 0.0,
      "train_loss": 0.17903520272865456,
      "train_runtime": 4631.245,
      "train_samples_per_second": 13.2,
      "train_steps_per_second": 0.103
    }
  ],
  "logging_steps": 10,
  "max_steps": 478,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 100,
  "total_flos": 0.0,
  "train_batch_size": 8,
  "trial_name": null,
  "trial_params": null
}