{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 0.9998706171561651,
  "eval_steps": 200,
  "global_step": 966,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.010350627506792599,
      "grad_norm": 36.896687952176364,
      "kl": 0.006904316134750843,
      "learning_rate": 2.126812117966759e-07,
      "logps/chosen": -417.2011954066265,
      "logps/rejected": -375.1744622564935,
      "loss": 0.4997,
      "rewards/chosen": -0.003417713455407016,
      "rewards/margins": 0.002690252778750596,
      "rewards/rejected": -0.006107966234157612,
      "step": 10
    },
    {
      "epoch": 0.020701255013585197,
      "grad_norm": 36.344980205107255,
      "kl": 0.004612588789314032,
      "learning_rate": 3.096603651432316e-07,
      "logps/chosen": -277.7969021267361,
      "logps/rejected": -407.0654296875,
      "loss": 0.4892,
      "rewards/chosen": -0.04762052165137397,
      "rewards/margins": 0.07216862355819856,
      "rewards/rejected": -0.11978914520957253,
      "step": 20
    },
    {
      "epoch": 0.0310518825203778,
      "grad_norm": 36.71259965236908,
      "kl": 0.0,
      "learning_rate": 3.602235071779947e-07,
      "logps/chosen": -355.01975574712645,
      "logps/rejected": -431.1890785530822,
      "loss": 0.4788,
      "rewards/chosen": -0.2371558485359981,
      "rewards/margins": 0.25850485494016306,
      "rewards/rejected": -0.49566070347616115,
      "step": 30
    },
    {
      "epoch": 0.041402510027170394,
      "grad_norm": 38.99807359143627,
      "kl": 0.016344498842954636,
      "learning_rate": 3.9466076978545386e-07,
      "logps/chosen": -347.33896998355266,
      "logps/rejected": -404.498046875,
      "loss": 0.4398,
      "rewards/chosen": -0.5145087995027241,
      "rewards/margins": 0.5702773885320601,
      "rewards/rejected": -1.0847861880347842,
      "step": 40
    },
    {
      "epoch": 0.051753137533962996,
      "grad_norm": 34.4366373643818,
      "kl": 0.0,
      "learning_rate": 4.208077428062608e-07,
      "logps/chosen": -401.31200610632186,
      "logps/rejected": -408.78579837328766,
      "loss": 0.4906,
      "rewards/chosen": -1.0610687431247754,
      "rewards/margins": 0.6205490982038848,
      "rewards/rejected": -1.6816178413286602,
      "step": 50
    },
    {
      "epoch": 0.0621037650407556,
      "grad_norm": 40.6191164803455,
      "kl": 0.0,
      "learning_rate": 4.4189144263242994e-07,
      "logps/chosen": -292.2215844131098,
      "logps/rejected": -431.56860977564105,
      "loss": 0.4649,
      "rewards/chosen": -0.6987755007860137,
      "rewards/margins": 0.5288097293321754,
      "rewards/rejected": -1.227585230118189,
      "step": 60
    },
    {
      "epoch": 0.0724543925475482,
      "grad_norm": 30.15586760876392,
      "kl": 0.0,
      "learning_rate": 4.5955828020052655e-07,
      "logps/chosen": -355.0156035370879,
      "logps/rejected": -401.8425045289855,
      "loss": 0.4658,
      "rewards/chosen": -0.5742165701729911,
      "rewards/margins": 0.8813798008004576,
      "rewards/rejected": -1.4555963709734487,
      "step": 70
    },
    {
      "epoch": 0.08280502005434079,
      "grad_norm": 28.330817825249255,
      "kl": 0.0,
      "learning_rate": 4.7476282570257156e-07,
      "logps/chosen": -394.8970209478022,
      "logps/rejected": -387.24026268115944,
      "loss": 0.4731,
      "rewards/chosen": -0.7301217383080787,
      "rewards/margins": 0.8021065933268852,
      "rewards/rejected": -1.5322283316349639,
      "step": 80
    },
    {
      "epoch": 0.0931556475611334,
      "grad_norm": 30.67181137678842,
      "kl": 0.0,
      "learning_rate": 4.881082258136016e-07,
      "logps/chosen": -294.80318509615387,
      "logps/rejected": -385.4679163490854,
      "loss": 0.4613,
      "rewards/chosen": -0.6603363232734876,
      "rewards/margins": 0.5270779856001309,
      "rewards/rejected": -1.1874143088736184,
      "step": 90
    },
    {
      "epoch": 0.10350627506792599,
      "grad_norm": 31.541642718713373,
      "kl": 0.0,
      "learning_rate": 5e-07,
      "logps/chosen": -356.5553466796875,
      "logps/rejected": -427.236279296875,
      "loss": 0.4466,
      "rewards/chosen": -0.5922697067260743,
      "rewards/margins": 0.8280625343322754,
      "rewards/rejected": -1.4203322410583497,
      "step": 100
    },
    {
      "epoch": 0.11385690257471859,
      "grad_norm": 28.188607938438196,
      "kl": 0.0,
      "learning_rate": 5e-07,
      "logps/chosen": -349.71470997431504,
      "logps/rejected": -431.29777298850576,
      "loss": 0.445,
      "rewards/chosen": -0.7172038457165025,
      "rewards/margins": 0.5990428885518614,
      "rewards/rejected": -1.316246734268364,
      "step": 110
    },
    {
      "epoch": 0.1242075300815112,
      "grad_norm": 28.777259577988843,
      "kl": 0.0,
      "learning_rate": 5e-07,
      "logps/chosen": -317.6811767578125,
      "logps/rejected": -400.8,
      "loss": 0.4577,
      "rewards/chosen": -0.748396921157837,
      "rewards/margins": 0.6573972225189209,
      "rewards/rejected": -1.4057941436767578,
      "step": 120
    },
    {
      "epoch": 0.13455815758830378,
      "grad_norm": 27.073111094683828,
      "kl": 0.0,
      "learning_rate": 5e-07,
      "logps/chosen": -334.3945529513889,
      "logps/rejected": -424.9839564732143,
      "loss": 0.4654,
      "rewards/chosen": -0.797715589735243,
      "rewards/margins": 0.9449826437329489,
      "rewards/rejected": -1.742698233468192,
      "step": 130
    },
    {
      "epoch": 0.1449087850950964,
      "grad_norm": 29.19719970356803,
      "kl": 0.0,
      "learning_rate": 5e-07,
      "logps/chosen": -368.06354166666665,
      "logps/rejected": -338.69952566964287,
      "loss": 0.487,
      "rewards/chosen": -0.6288536071777344,
      "rewards/margins": 0.4991338457380021,
      "rewards/rejected": -1.1279874529157365,
      "step": 140
    },
    {
      "epoch": 0.155259412601889,
      "grad_norm": 31.37561442050933,
      "kl": 0.0,
      "learning_rate": 5e-07,
      "logps/chosen": -308.6735341061828,
      "logps/rejected": -390.50953241604475,
      "loss": 0.4775,
      "rewards/chosen": -0.5661141180223034,
      "rewards/margins": 0.7382167362512507,
      "rewards/rejected": -1.304330854273554,
      "step": 150
    },
    {
      "epoch": 0.16561004010868158,
      "grad_norm": 36.002916810630985,
      "kl": 0.07424011081457138,
      "learning_rate": 5e-07,
      "logps/chosen": -395.24665850903614,
      "logps/rejected": -402.71707589285717,
      "loss": 0.4599,
      "rewards/chosen": -0.5169859277196678,
      "rewards/margins": 0.6465992892484227,
      "rewards/rejected": -1.1635852169680905,
      "step": 160
    },
    {
      "epoch": 0.1759606676154742,
      "grad_norm": 27.151915007789793,
      "kl": 0.0,
      "learning_rate": 5e-07,
      "logps/chosen": -380.0856370192308,
      "logps/rejected": -437.3331269054878,
      "loss": 0.434,
      "rewards/chosen": -0.5822516710330279,
      "rewards/margins": 0.9586351846739081,
      "rewards/rejected": -1.540886855706936,
      "step": 170
    },
    {
      "epoch": 0.1863112951222668,
      "grad_norm": 22.70085892654007,
      "kl": 0.0,
      "learning_rate": 5e-07,
      "logps/chosen": -352.72755281690144,
      "logps/rejected": -438.0743504213483,
      "loss": 0.4081,
      "rewards/chosen": -0.6838695364938655,
      "rewards/margins": 0.9942577951862609,
      "rewards/rejected": -1.6781273316801264,
      "step": 180
    },
    {
      "epoch": 0.19666192262905938,
      "grad_norm": 27.14667046044915,
      "kl": 0.0,
      "learning_rate": 5e-07,
      "logps/chosen": -320.2009880514706,
      "logps/rejected": -407.538046875,
      "loss": 0.449,
      "rewards/chosen": -1.0842503267176011,
      "rewards/margins": 1.4791047758214615,
      "rewards/rejected": -2.5633551025390626,
      "step": 190
    },
    {
      "epoch": 0.20701255013585199,
      "grad_norm": 27.694967881656005,
      "kl": 0.005686330609023571,
      "learning_rate": 5e-07,
      "logps/chosen": -337.79836856617646,
      "logps/rejected": -427.5978645833333,
      "loss": 0.4522,
      "rewards/chosen": -0.9811132094439339,
      "rewards/margins": 1.1809233302696076,
      "rewards/rejected": -2.1620365397135415,
      "step": 200
    },
    {
      "epoch": 0.20701255013585199,
      "eval_kl": 0.0010393437696620822,
      "eval_logps/chosen": -345.2487181263858,
      "eval_logps/rejected": -393.2139168432203,
      "eval_loss": 0.44461360573768616,
      "eval_rewards/chosen": -1.1207509516612388,
      "eval_rewards/margins": 0.9445489068199584,
      "eval_rewards/rejected": -2.0652998584811972,
      "eval_runtime": 261.4133,
      "eval_samples_per_second": 7.062,
      "eval_steps_per_second": 3.531,
      "step": 200
    },
    {
      "epoch": 0.2173631776426446,
      "grad_norm": 30.464898770807604,
      "kl": 0.0,
      "learning_rate": 5e-07,
      "logps/chosen": -365.0289713541667,
      "logps/rejected": -427.5989879261364,
      "loss": 0.4016,
      "rewards/chosen": -0.8537895944383409,
      "rewards/margins": 1.3811903818689213,
      "rewards/rejected": -2.234979976307262,
      "step": 210
    },
    {
      "epoch": 0.22771380514943718,
      "grad_norm": 23.41466055625897,
      "kl": 0.0,
      "learning_rate": 5e-07,
      "logps/chosen": -345.475933908046,
      "logps/rejected": -393.412189640411,
      "loss": 0.4443,
      "rewards/chosen": -0.9509018645889458,
      "rewards/margins": 1.2836219608605406,
      "rewards/rejected": -2.2345238254494864,
      "step": 220
    },
    {
      "epoch": 0.23806443265622979,
      "grad_norm": 26.98695760593119,
      "kl": 0.0,
      "learning_rate": 5e-07,
      "logps/chosen": -323.52855282738096,
      "logps/rejected": -349.73843544407896,
      "loss": 0.4764,
      "rewards/chosen": -0.9678686232793898,
      "rewards/margins": 0.599584660733254,
      "rewards/rejected": -1.5674532840126438,
      "step": 230
    },
    {
      "epoch": 0.2484150601630224,
      "grad_norm": 26.51688363505412,
      "kl": 0.0,
      "learning_rate": 5e-07,
      "logps/chosen": -339.72511772260276,
      "logps/rejected": -392.6373024425287,
      "loss": 0.4254,
      "rewards/chosen": -0.7914297548058915,
      "rewards/margins": 0.7201490352668456,
      "rewards/rejected": -1.511578790072737,
      "step": 240
    },
    {
      "epoch": 0.258765687669815,
      "grad_norm": 38.3631109147077,
      "kl": 0.0,
      "learning_rate": 5e-07,
      "logps/chosen": -362.05623478084414,
      "logps/rejected": -361.7480233433735,
      "loss": 0.4123,
      "rewards/chosen": -0.5259268129026735,
      "rewards/margins": 1.3802443193803648,
      "rewards/rejected": -1.9061711322830384,
      "step": 250
    },
    {
      "epoch": 0.26911631517660756,
      "grad_norm": 28.783330131851603,
      "kl": 0.0,
      "learning_rate": 5e-07,
      "logps/chosen": -318.40223672945206,
      "logps/rejected": -472.43588362068965,
      "loss": 0.4135,
      "rewards/chosen": -0.8065869579576466,
      "rewards/margins": 1.0660637146918686,
      "rewards/rejected": -1.8726506726495151,
      "step": 260
    },
    {
      "epoch": 0.27946694268340017,
      "grad_norm": 25.596794311830312,
      "kl": 0.0,
      "learning_rate": 5e-07,
      "logps/chosen": -367.7001139322917,
      "logps/rejected": -436.5064808238636,
      "loss": 0.4093,
      "rewards/chosen": -0.8103501001993815,
      "rewards/margins": 1.0771059267448657,
      "rewards/rejected": -1.887456026944247,
      "step": 270
    },
    {
      "epoch": 0.2898175701901928,
      "grad_norm": 31.49395928287787,
      "kl": 0.0206025131046772,
      "learning_rate": 5e-07,
      "logps/chosen": -456.68581081081084,
      "logps/rejected": -399.91547056686045,
      "loss": 0.4345,
      "rewards/chosen": -1.0405741511164486,
      "rewards/margins": 0.9193381711718871,
      "rewards/rejected": -1.9599123222883357,
      "step": 280
    },
    {
      "epoch": 0.3001681976969854,
      "grad_norm": 27.507156588516853,
      "kl": 0.0,
      "learning_rate": 5e-07,
      "logps/chosen": -354.3474633487654,
      "logps/rejected": -430.79647943037975,
      "loss": 0.4246,
      "rewards/chosen": -0.8167637954523534,
      "rewards/margins": 1.6583199540531843,
      "rewards/rejected": -2.4750837495055378,
      "step": 290
    },
    {
      "epoch": 0.310518825203778,
      "grad_norm": 27.243120877089865,
      "kl": 0.0,
      "learning_rate": 5e-07,
      "logps/chosen": -369.0854611280488,
      "logps/rejected": -411.7598407451923,
      "loss": 0.4515,
      "rewards/chosen": -0.9885020372344226,
      "rewards/margins": 1.07946980364849,
      "rewards/rejected": -2.0679718408829126,
      "step": 300
    },
    {
      "epoch": 0.3208694527105706,
      "grad_norm": 26.41738454716243,
      "kl": 0.0,
      "learning_rate": 5e-07,
      "logps/chosen": -328.5754642210145,
      "logps/rejected": -473.32679429945057,
      "loss": 0.4,
      "rewards/chosen": -0.8000211853911912,
      "rewards/margins": 1.4340147676694202,
      "rewards/rejected": -2.2340359530606113,
      "step": 310
    },
    {
      "epoch": 0.33122008021736316,
      "grad_norm": 27.90947815196134,
      "kl": 0.0,
      "learning_rate": 5e-07,
      "logps/chosen": -379.9532833614865,
      "logps/rejected": -414.0909792877907,
      "loss": 0.4206,
      "rewards/chosen": -0.707832078675966,
      "rewards/margins": 1.2218697929741975,
      "rewards/rejected": -1.9297018716501635,
      "step": 320
    },
    {
      "epoch": 0.34157070772415576,
      "grad_norm": 34.323093394556274,
      "kl": 0.0,
      "learning_rate": 5e-07,
      "logps/chosen": -302.920654296875,
      "logps/rejected": -365.2463107638889,
      "loss": 0.4599,
      "rewards/chosen": -0.8200391422618519,
      "rewards/margins": 1.4710271334407303,
      "rewards/rejected": -2.2910662757025824,
      "step": 330
    },
    {
      "epoch": 0.3519213352309484,
      "grad_norm": 35.00194682599148,
      "kl": 0.020750045776367188,
      "learning_rate": 5e-07,
      "logps/chosen": -370.7525414156627,
      "logps/rejected": -407.6445819805195,
      "loss": 0.4401,
      "rewards/chosen": -0.4423764699912933,
      "rewards/margins": 0.795768243571511,
      "rewards/rejected": -1.2381447135628043,
      "step": 340
    },
    {
      "epoch": 0.362271962737741,
      "grad_norm": 28.74449923281838,
      "kl": 0.0,
      "learning_rate": 5e-07,
      "logps/chosen": -369.2329220655488,
      "logps/rejected": -412.28390424679486,
      "loss": 0.443,
      "rewards/chosen": -0.5598751161156631,
      "rewards/margins": 0.7612275152820732,
      "rewards/rejected": -1.3211026313977363,
      "step": 350
    },
    {
      "epoch": 0.3726225902445336,
      "grad_norm": 33.80893504974849,
      "kl": 0.0,
      "learning_rate": 5e-07,
      "logps/chosen": -333.06757269965277,
      "logps/rejected": -476.0582386363636,
      "loss": 0.3942,
      "rewards/chosen": -0.8373040093315972,
      "rewards/margins": 1.5102612081200184,
      "rewards/rejected": -2.3475652174516157,
      "step": 360
    },
    {
      "epoch": 0.3829732177513262,
      "grad_norm": 23.093501234844034,
      "kl": 0.0,
      "learning_rate": 5e-07,
      "logps/chosen": -344.985234375,
      "logps/rejected": -478.50422794117645,
      "loss": 0.3956,
      "rewards/chosen": -1.264248046875,
      "rewards/margins": 1.5773571059283087,
      "rewards/rejected": -2.8416051528033086,
      "step": 370
    },
    {
      "epoch": 0.39332384525811875,
      "grad_norm": 23.6165146626171,
      "kl": 0.0,
      "learning_rate": 5e-07,
      "logps/chosen": -331.19694890202703,
      "logps/rejected": -458.86123728197674,
      "loss": 0.4153,
      "rewards/chosen": -1.1533899049501162,
      "rewards/margins": 1.4770645798563884,
      "rewards/rejected": -2.6304544848065046,
      "step": 380
    },
    {
      "epoch": 0.40367447276491136,
      "grad_norm": 24.677426766885045,
      "kl": 0.045375823974609375,
      "learning_rate": 5e-07,
      "logps/chosen": -335.65542204483694,
      "logps/rejected": -439.2108800551471,
      "loss": 0.4565,
      "rewards/chosen": -0.7980768784232761,
      "rewards/margins": 1.7186397981765629,
      "rewards/rejected": -2.516716676599839,
      "step": 390
    },
    {
      "epoch": 0.41402510027170397,
      "grad_norm": 32.96461257238746,
      "kl": 0.0,
      "learning_rate": 5e-07,
      "logps/chosen": -313.4358512581169,
      "logps/rejected": -439.3407379518072,
      "loss": 0.4056,
      "rewards/chosen": -0.6841482187246347,
      "rewards/margins": 1.2257567208579143,
      "rewards/rejected": -1.909904939582549,
      "step": 400
    },
    {
      "epoch": 0.41402510027170397,
      "eval_kl": 0.02226920612156391,
      "eval_logps/chosen": -340.0967987804878,
      "eval_logps/rejected": -387.47169623940675,
      "eval_loss": 0.44011881947517395,
      "eval_rewards/chosen": -0.6055575284090909,
      "eval_rewards/margins": 0.885514011000999,
      "eval_rewards/rejected": -1.49107153941009,
      "eval_runtime": 260.8826,
      "eval_samples_per_second": 7.076,
      "eval_steps_per_second": 3.538,
      "step": 400
    },
    {
      "epoch": 0.4243757277784966,
      "grad_norm": 26.751958968613145,
      "kl": 0.0,
      "learning_rate": 5e-07,
      "logps/chosen": -372.82060185185185,
      "logps/rejected": -399.93740110759495,
      "loss": 0.4265,
      "rewards/chosen": -0.5553302058467159,
      "rewards/margins": 1.1236735458839013,
      "rewards/rejected": -1.6790037517306171,
      "step": 410
    },
    {
      "epoch": 0.4347263552852892,
      "grad_norm": 32.246235152731096,
      "kl": 0.0,
      "learning_rate": 5e-07,
      "logps/chosen": -348.120418595679,
      "logps/rejected": -402.0041287579114,
      "loss": 0.4239,
      "rewards/chosen": -0.7425044495382427,
      "rewards/margins": 1.1616015940238618,
      "rewards/rejected": -1.9041060435621044,
      "step": 420
    },
    {
      "epoch": 0.44507698279208174,
      "grad_norm": 27.212254824473547,
      "kl": 0.04713239520788193,
      "learning_rate": 5e-07,
      "logps/chosen": -330.6474880642361,
      "logps/rejected": -389.7398792613636,
      "loss": 0.3978,
      "rewards/chosen": -0.9934198591444228,
      "rewards/margins": 1.0819970525876441,
      "rewards/rejected": -2.075416911732067,
      "step": 430
    },
    {
      "epoch": 0.45542761029887435,
      "grad_norm": 24.894169784907362,
      "kl": 0.0,
      "learning_rate": 5e-07,
      "logps/chosen": -376.64564344618054,
      "logps/rejected": -515.4582297585227,
      "loss": 0.3809,
      "rewards/chosen": -0.8414801491631402,
      "rewards/margins": 1.9854850094727796,
      "rewards/rejected": -2.82696515863592,
      "step": 440
    },
    {
      "epoch": 0.46577823780566696,
      "grad_norm": 33.54283688924568,
      "kl": 0.0,
      "learning_rate": 5e-07,
      "logps/chosen": -373.34893120659723,
      "logps/rejected": -440.4869495738636,
      "loss": 0.4141,
      "rewards/chosen": -1.090722295973036,
      "rewards/margins": 1.4239928987291124,
      "rewards/rejected": -2.5147151947021484,
      "step": 450
    },
    {
      "epoch": 0.47612886531245957,
      "grad_norm": 25.873975620632326,
      "kl": 0.0,
      "learning_rate": 5e-07,
      "logps/chosen": -297.38337725903614,
      "logps/rejected": -368.12974330357144,
      "loss": 0.4304,
      "rewards/chosen": -0.642763620399567,
      "rewards/margins": 1.4404461233478962,
      "rewards/rejected": -2.0832097437474633,
      "step": 460
    },
    {
      "epoch": 0.4864794928192522,
      "grad_norm": 31.22528359201901,
      "kl": 0.0,
      "learning_rate": 5e-07,
      "logps/chosen": -364.6297576121795,
      "logps/rejected": -414.5650247713415,
      "loss": 0.397,
      "rewards/chosen": -0.4758866138947316,
      "rewards/margins": 1.4139596296147006,
      "rewards/rejected": -1.889846243509432,
      "step": 470
    },
    {
      "epoch": 0.4968301203260448,
      "grad_norm": 26.938362242757048,
      "kl": 0.0,
      "learning_rate": 5e-07,
      "logps/chosen": -398.0412109375,
      "logps/rejected": -487.195654296875,
      "loss": 0.4055,
      "rewards/chosen": -0.5108624458312988,
      "rewards/margins": 1.3080674171447755,
      "rewards/rejected": -1.8189298629760742,
      "step": 480
    },
    {
      "epoch": 0.5071807478328374,
      "grad_norm": 29.10971517563742,
      "kl": 0.0,
      "learning_rate": 5e-07,
      "logps/chosen": -390.874140625,
      "logps/rejected": -379.295703125,
      "loss": 0.4363,
      "rewards/chosen": -0.9983409627278645,
      "rewards/margins": 0.9311472754384957,
      "rewards/rejected": -1.9294882381663603,
      "step": 490
    },
    {
      "epoch": 0.51753137533963,
      "grad_norm": 27.404424128068055,
      "kl": 0.0,
      "learning_rate": 5e-07,
      "logps/chosen": -330.4240828804348,
      "logps/rejected": -394.64285714285717,
      "loss": 0.3714,
      "rewards/chosen": -0.724442468173262,
      "rewards/margins": 1.7530021910493512,
      "rewards/rejected": -2.477444659222613,
      "step": 500
    },
    {
      "epoch": 0.5278820028464226,
      "grad_norm": 30.205347992720988,
      "kl": 0.010777664370834827,
      "learning_rate": 5e-07,
      "logps/chosen": -371.4545238597973,
      "logps/rejected": -425.5056776889535,
      "loss": 0.4051,
      "rewards/chosen": -0.7893987088590055,
      "rewards/margins": 1.3341055749573099,
      "rewards/rejected": -2.1235042838163154,
      "step": 510
    },
    {
      "epoch": 0.5382326303532151,
      "grad_norm": 27.47044972378467,
      "kl": 0.0,
      "learning_rate": 5e-07,
      "logps/chosen": -317.4935569324713,
      "logps/rejected": -394.0045751284247,
      "loss": 0.433,
      "rewards/chosen": -0.9540053619735542,
      "rewards/margins": 1.3108872161514458,
      "rewards/rejected": -2.264892578125,
      "step": 520
    },
    {
      "epoch": 0.5485832578600077,
      "grad_norm": 32.30343597091197,
      "kl": 0.06133537366986275,
      "learning_rate": 5e-07,
      "logps/chosen": -377.94091796875,
      "logps/rejected": -412.06171875,
      "loss": 0.415,
      "rewards/chosen": -0.42492337226867677,
      "rewards/margins": 1.4800034999847413,
      "rewards/rejected": -1.904926872253418,
      "step": 530
    },
    {
      "epoch": 0.5589338853668003,
      "grad_norm": 34.615081184959564,
      "kl": 0.0,
      "learning_rate": 5e-07,
      "logps/chosen": -326.7767721036585,
      "logps/rejected": -424.0320012019231,
      "loss": 0.4396,
      "rewards/chosen": -0.4712153178889577,
      "rewards/margins": 0.8699719880505454,
      "rewards/rejected": -1.3411873059395032,
      "step": 540
    },
    {
      "epoch": 0.5692845128735929,
      "grad_norm": 34.56564210195164,
      "kl": 0.06511452049016953,
      "learning_rate": 5e-07,
      "logps/chosen": -407.66327617694805,
      "logps/rejected": -371.1233998493976,
      "loss": 0.375,
      "rewards/chosen": -0.19861872784503096,
      "rewards/margins": 1.8991040725361816,
      "rewards/rejected": -2.0977228003812125,
      "step": 550
    },
    {
      "epoch": 0.5796351403803855,
      "grad_norm": 26.904457572554023,
      "kl": 0.14332695305347443,
      "learning_rate": 5e-07,
      "logps/chosen": -336.1937744140625,
      "logps/rejected": -404.095556640625,
      "loss": 0.4105,
      "rewards/chosen": -0.39203429222106934,
      "rewards/margins": 1.2247087955474854,
      "rewards/rejected": -1.6167430877685547,
      "step": 560
    },
    {
      "epoch": 0.5899857678871782,
      "grad_norm": 29.784323096457744,
      "kl": 0.005803870968520641,
      "learning_rate": 5e-07,
      "logps/chosen": -300.69694346005156,
      "logps/rejected": -407.36216517857144,
      "loss": 0.4499,
      "rewards/chosen": -0.4008376917888209,
      "rewards/margins": 1.1861952923846664,
      "rewards/rejected": -1.5870329841734871,
      "step": 570
    },
    {
      "epoch": 0.6003363953939708,
      "grad_norm": 24.46799204128634,
      "kl": 0.22002115845680237,
      "learning_rate": 5e-07,
      "logps/chosen": -347.5768229166667,
      "logps/rejected": -443.0283717105263,
      "loss": 0.4189,
      "rewards/chosen": -0.3804002716427758,
      "rewards/margins": 1.428496646403071,
      "rewards/rejected": -1.808896918045847,
      "step": 580
    },
    {
      "epoch": 0.6106870229007634,
      "grad_norm": 30.275312642751995,
      "kl": 0.13701924681663513,
      "learning_rate": 5e-07,
      "logps/chosen": -355.59707919034093,
      "logps/rejected": -430.7814670138889,
      "loss": 0.4381,
      "rewards/chosen": -0.5888070193204012,
      "rewards/margins": 1.315426489319464,
      "rewards/rejected": -1.9042335086398654,
      "step": 590
    },
    {
      "epoch": 0.621037650407556,
      "grad_norm": 32.60832471668693,
      "kl": 0.02227201499044895,
      "learning_rate": 5e-07,
      "logps/chosen": -350.7683919270833,
      "logps/rejected": -444.4885896381579,
      "loss": 0.4163,
      "rewards/chosen": -0.5356872195289248,
      "rewards/margins": 1.2915597977793605,
      "rewards/rejected": -1.8272470173082853,
      "step": 600
    },
    {
      "epoch": 0.621037650407556,
      "eval_kl": 0.009310548193752766,
      "eval_logps/chosen": -339.2911238913525,
      "eval_logps/rejected": -389.9666313559322,
      "eval_loss": 0.418056845664978,
      "eval_rewards/chosen": -0.5249900056623302,
      "eval_rewards/margins": 1.2155782523322407,
      "eval_rewards/rejected": -1.740568257994571,
      "eval_runtime": 261.1582,
      "eval_samples_per_second": 7.069,
      "eval_steps_per_second": 3.534,
      "step": 600
    },
    {
      "epoch": 0.6313882779143486,
      "grad_norm": 22.18017793377208,
      "kl": 0.039247892796993256,
      "learning_rate": 5e-07,
      "logps/chosen": -383.04136439732144,
      "logps/rejected": -429.54263466282896,
      "loss": 0.4132,
      "rewards/chosen": -0.4417642865862165,
      "rewards/margins": 1.5167117298097539,
      "rewards/rejected": -1.9584760163959705,
      "step": 610
    },
    {
      "epoch": 0.6417389054211412,
      "grad_norm": 29.924696721027633,
      "kl": 0.03644561767578125,
      "learning_rate": 5e-07,
      "logps/chosen": -382.4176720727848,
      "logps/rejected": -498.68258101851853,
      "loss": 0.3878,
      "rewards/chosen": -0.47994140431850774,
      "rewards/margins": 1.639255923095169,
      "rewards/rejected": -2.119197327413677,
      "step": 620
    },
    {
      "epoch": 0.6520895329279337,
      "grad_norm": 28.116353403382174,
      "kl": 0.0513916015625,
      "learning_rate": 5e-07,
      "logps/chosen": -311.66650390625,
      "logps/rejected": -419.412939453125,
      "loss": 0.3978,
      "rewards/chosen": -0.5402119159698486,
      "rewards/margins": 1.4218003749847412,
      "rewards/rejected": -1.9620122909545898,
      "step": 630
    },
    {
      "epoch": 0.6624401604347263,
      "grad_norm": 30.136959971403833,
      "kl": 0.0,
      "learning_rate": 5e-07,
      "logps/chosen": -367.5465806934931,
      "logps/rejected": -451.5183638649425,
      "loss": 0.4204,
      "rewards/chosen": -1.145416991351402,
      "rewards/margins": 0.9065032307885044,
      "rewards/rejected": -2.0519202221399064,
      "step": 640
    },
    {
      "epoch": 0.6727907879415189,
      "grad_norm": 25.55983506887128,
      "kl": 0.0,
      "learning_rate": 5e-07,
      "logps/chosen": -331.976943597561,
      "logps/rejected": -408.9071514423077,
      "loss": 0.4119,
      "rewards/chosen": -0.9971130185010957,
      "rewards/margins": 1.7702796535241447,
      "rewards/rejected": -2.7673926720252404,
      "step": 650
    },
    {
      "epoch": 0.6831414154483115,
      "grad_norm": 25.82328491415139,
      "kl": 0.008263682946562767,
      "learning_rate": 5e-07,
      "logps/chosen": -361.71830610795456,
      "logps/rejected": -486.77197265625,
      "loss": 0.4233,
      "rewards/chosen": -0.7347448522394354,
      "rewards/margins": 2.0849816678750392,
      "rewards/rejected": -2.8197265201144748,
      "step": 660
    },
    {
      "epoch": 0.6934920429551041,
      "grad_norm": 26.68136550686645,
      "kl": 0.09514617919921875,
      "learning_rate": 5e-07,
      "logps/chosen": -355.8723958333333,
      "logps/rejected": -486.3736672794118,
      "loss": 0.3773,
      "rewards/chosen": -0.6042455546061198,
      "rewards/margins": 2.0892714347091377,
      "rewards/rejected": -2.6935169893152575,
      "step": 670
    },
    {
      "epoch": 0.7038426704618967,
      "grad_norm": 21.168784955584055,
      "kl": 0.0,
      "learning_rate": 5e-07,
      "logps/chosen": -330.87958757267444,
      "logps/rejected": -369.6824588260135,
      "loss": 0.4207,
      "rewards/chosen": -0.6469083830367687,
      "rewards/margins": 1.6826272301521037,
      "rewards/rejected": -2.3295356131888725,
      "step": 680
    },
    {
      "epoch": 0.7141932979686894,
      "grad_norm": 31.900093457461022,
      "kl": 0.08187294006347656,
      "learning_rate": 5e-07,
      "logps/chosen": -417.3225528492647,
      "logps/rejected": -440.6748471467391,
      "loss": 0.4112,
      "rewards/chosen": -0.7309647728415096,
      "rewards/margins": 1.2705956041965338,
      "rewards/rejected": -2.0015603770380435,
      "step": 690
    },
    {
      "epoch": 0.724543925475482,
      "grad_norm": 27.570529431002825,
      "kl": 0.0,
      "learning_rate": 5e-07,
      "logps/chosen": -337.6109751506024,
      "logps/rejected": -445.5989752435065,
      "loss": 0.4132,
      "rewards/chosen": -0.41143114021025506,
      "rewards/margins": 1.2929300198698992,
      "rewards/rejected": -1.7043611600801543,
      "step": 700
    },
    {
      "epoch": 0.7348945529822746,
      "grad_norm": 25.994342612564424,
      "kl": 0.004410457797348499,
      "learning_rate": 5e-07,
      "logps/chosen": -273.53585737179486,
      "logps/rejected": -435.5107660060976,
      "loss": 0.392,
      "rewards/chosen": -0.5441466111403245,
      "rewards/margins": 1.5854568910867144,
      "rewards/rejected": -2.129603502227039,
      "step": 710
    },
    {
      "epoch": 0.7452451804890672,
      "grad_norm": 28.362840310964046,
      "kl": 0.0,
      "learning_rate": 5e-07,
      "logps/chosen": -332.836171875,
      "logps/rejected": -474.0301470588235,
      "loss": 0.3782,
      "rewards/chosen": -0.7581790669759114,
      "rewards/margins": 1.799709726969401,
      "rewards/rejected": -2.5578887939453123,
      "step": 720
    },
    {
      "epoch": 0.7555958079958598,
      "grad_norm": 21.561327202318306,
      "kl": 0.0,
      "learning_rate": 5e-07,
      "logps/chosen": -275.46072571536143,
      "logps/rejected": -412.22519277597405,
      "loss": 0.3875,
      "rewards/chosen": -0.5184578493417028,
      "rewards/margins": 1.9156346847111583,
      "rewards/rejected": -2.434092534052861,
      "step": 730
    },
    {
      "epoch": 0.7659464355026524,
      "grad_norm": 18.54675354753111,
      "kl": 0.0,
      "learning_rate": 5e-07,
      "logps/chosen": -368.63963607594934,
      "logps/rejected": -425.9934895833333,
      "loss": 0.4036,
      "rewards/chosen": -0.8795772504202927,
      "rewards/margins": 1.8751527858089703,
      "rewards/rejected": -2.754730036229263,
      "step": 740
    },
    {
      "epoch": 0.7762970630094449,
      "grad_norm": 27.548342849043514,
      "kl": 0.0,
      "learning_rate": 5e-07,
      "logps/chosen": -395.18419471153845,
      "logps/rejected": -404.6112804878049,
      "loss": 0.4149,
      "rewards/chosen": -0.8401767046023638,
      "rewards/margins": 1.619423790526733,
      "rewards/rejected": -2.459600495129097,
      "step": 750
    },
    {
      "epoch": 0.7866476905162375,
      "grad_norm": 26.78674064602181,
      "kl": 0.0,
      "learning_rate": 5e-07,
      "logps/chosen": -361.57657251602564,
      "logps/rejected": -418.945693597561,
      "loss": 0.4087,
      "rewards/chosen": -0.6369634775015024,
      "rewards/margins": 1.3416846384474304,
      "rewards/rejected": -1.9786481159489329,
      "step": 760
    },
    {
      "epoch": 0.7969983180230301,
      "grad_norm": 25.603994333749306,
      "kl": 0.02446603775024414,
      "learning_rate": 5e-07,
      "logps/chosen": -319.98974609375,
      "logps/rejected": -407.2808314732143,
      "loss": 0.39,
      "rewards/chosen": -0.4739310615941098,
      "rewards/margins": 1.6105410496991381,
      "rewards/rejected": -2.084472111293248,
      "step": 770
    },
    {
      "epoch": 0.8073489455298227,
      "grad_norm": 26.587028848139315,
      "kl": 0.04417114332318306,
      "learning_rate": 5e-07,
      "logps/chosen": -318.715,
      "logps/rejected": -429.6086856617647,
      "loss": 0.3786,
      "rewards/chosen": -0.553302001953125,
      "rewards/margins": 1.9189411836511947,
      "rewards/rejected": -2.4722431856043197,
      "step": 780
    },
    {
      "epoch": 0.8176995730366153,
      "grad_norm": 23.329347974769387,
      "kl": 0.10457019507884979,
      "learning_rate": 5e-07,
      "logps/chosen": -382.36054180194805,
      "logps/rejected": -415.20811370481925,
      "loss": 0.3949,
      "rewards/chosen": -0.9674345734831574,
      "rewards/margins": 2.05568157274237,
      "rewards/rejected": -3.023116146225527,
      "step": 790
    },
    {
      "epoch": 0.8280502005434079,
      "grad_norm": 25.709767109519216,
      "kl": 0.0,
      "learning_rate": 5e-07,
      "logps/chosen": -314.2330375339674,
      "logps/rejected": -512.2797564338235,
      "loss": 0.4158,
      "rewards/chosen": -0.7708018759022588,
      "rewards/margins": 2.3479348399754985,
      "rewards/rejected": -3.1187367158777572,
      "step": 800
    },
    {
      "epoch": 0.8280502005434079,
      "eval_kl": 0.006037264596670866,
      "eval_logps/chosen": -341.35116407982264,
      "eval_logps/rejected": -395.00337658898303,
      "eval_loss": 0.41274696588516235,
      "eval_rewards/chosen": -0.7309938418098669,
      "eval_rewards/margins": 1.5132525602193967,
      "eval_rewards/rejected": -2.2442464020292636,
      "eval_runtime": 260.9097,
      "eval_samples_per_second": 7.075,
      "eval_steps_per_second": 3.538,
      "step": 800
    },
    {
      "epoch": 0.8384008280502006,
      "grad_norm": 35.370868792942815,
      "kl": 0.038701437413692474,
      "learning_rate": 5e-07,
      "logps/chosen": -300.92038143382354,
      "logps/rejected": -437.9970833333333,
      "loss": 0.4117,
      "rewards/chosen": -0.6198445039636948,
      "rewards/margins": 1.6185169055415134,
      "rewards/rejected": -2.2383614095052082,
      "step": 810
    },
    {
      "epoch": 0.8487514555569932,
      "grad_norm": 27.200562796310017,
      "kl": 0.03499946743249893,
      "learning_rate": 5e-07,
      "logps/chosen": -420.0014134457237,
      "logps/rejected": -474.64820498511904,
      "loss": 0.3712,
      "rewards/chosen": -0.21315298582378187,
      "rewards/margins": 2.169883309749135,
      "rewards/rejected": -2.3830362955729165,
      "step": 820
    },
    {
      "epoch": 0.8591020830637858,
      "grad_norm": 22.838766986028332,
      "kl": 0.08836288750171661,
      "learning_rate": 5e-07,
      "logps/chosen": -410.17025862068965,
      "logps/rejected": -479.935466609589,
      "loss": 0.4239,
      "rewards/chosen": -0.5182619642937321,
      "rewards/margins": 1.6473036309411782,
      "rewards/rejected": -2.16556559523491,
      "step": 830
    },
    {
      "epoch": 0.8694527105705784,
      "grad_norm": 31.190543721407206,
      "kl": 0.0,
      "learning_rate": 5e-07,
      "logps/chosen": -392.1194540895062,
      "logps/rejected": -380.65261570411394,
      "loss": 0.4195,
      "rewards/chosen": -0.5796352904519917,
      "rewards/margins": 1.1813462089422924,
      "rewards/rejected": -1.760981499394284,
      "step": 840
    },
    {
      "epoch": 0.879803338077371,
      "grad_norm": 27.922649096371728,
      "kl": 0.05376587063074112,
      "learning_rate": 5e-07,
      "logps/chosen": -342.7175263554217,
      "logps/rejected": -405.21989143668833,
      "loss": 0.3893,
      "rewards/chosen": -0.3039788625326501,
      "rewards/margins": 1.6599957309890625,
      "rewards/rejected": -1.9639745935217126,
      "step": 850
    },
    {
      "epoch": 0.8901539655841635,
      "grad_norm": 27.062117676313864,
      "kl": 0.029529189690947533,
      "learning_rate": 5e-07,
      "logps/chosen": -339.9334415584416,
      "logps/rejected": -389.23075112951807,
      "loss": 0.4056,
      "rewards/chosen": -0.7721986646776076,
      "rewards/margins": 1.4165069634991618,
      "rewards/rejected": -2.1887056281767694,
      "step": 860
    },
    {
      "epoch": 0.9005045930909561,
      "grad_norm": 26.777659950643177,
      "kl": 0.027013396844267845,
      "learning_rate": 5e-07,
      "logps/chosen": -359.30659239969134,
      "logps/rejected": -432.05760482594934,
      "loss": 0.4124,
      "rewards/chosen": -1.0398042466905382,
      "rewards/margins": 1.4762831525628244,
      "rewards/rejected": -2.5160873992533626,
      "step": 870
    },
    {
      "epoch": 0.9108552205977487,
      "grad_norm": 26.582657305921924,
      "kl": 0.0,
      "learning_rate": 5e-07,
      "logps/chosen": -347.1331422483766,
      "logps/rejected": -441.9758565512048,
      "loss": 0.3819,
      "rewards/chosen": -0.478736332484654,
      "rewards/margins": 2.0877041119101123,
      "rewards/rejected": -2.5664404443947664,
      "step": 880
    },
    {
      "epoch": 0.9212058481045413,
      "grad_norm": 26.122755291889042,
      "kl": 0.0009471893426962197,
      "learning_rate": 5e-07,
      "logps/chosen": -334.71205003955697,
      "logps/rejected": -477.93663194444446,
      "loss": 0.3907,
      "rewards/chosen": -0.6639707058290892,
      "rewards/margins": 2.118597389813754,
      "rewards/rejected": -2.7825680956428434,
      "step": 890
    },
    {
      "epoch": 0.9315564756113339,
      "grad_norm": 27.631179779669328,
      "kl": 0.03726501390337944,
      "learning_rate": 5e-07,
      "logps/chosen": -354.04930971746575,
      "logps/rejected": -398.39897629310343,
      "loss": 0.3698,
      "rewards/chosen": -0.6091255292500535,
      "rewards/margins": 1.8384917494519042,
      "rewards/rejected": -2.4476172787019577,
      "step": 900
    },
    {
      "epoch": 0.9419071031181265,
      "grad_norm": 31.5668708111869,
      "kl": 0.0027565001510083675,
      "learning_rate": 5e-07,
      "logps/chosen": -339.08727254746833,
      "logps/rejected": -447.73466435185185,
      "loss": 0.3717,
      "rewards/chosen": -0.4057273864746094,
      "rewards/margins": 2.1597686108247736,
      "rewards/rejected": -2.565495997299383,
      "step": 910
    },
    {
      "epoch": 0.9522577306249191,
      "grad_norm": 23.52230221185674,
      "kl": 0.0,
      "learning_rate": 5e-07,
      "logps/chosen": -331.8610026041667,
      "logps/rejected": -437.75386186079544,
      "loss": 0.3801,
      "rewards/chosen": -0.9904574288262261,
      "rewards/margins": 1.9297606053978504,
      "rewards/rejected": -2.9202180342240767,
      "step": 920
    },
    {
      "epoch": 0.9626083581317117,
      "grad_norm": 21.084240235211357,
      "kl": 0.005317878909409046,
      "learning_rate": 5e-07,
      "logps/chosen": -443.4196810787671,
      "logps/rejected": -434.53286637931035,
      "loss": 0.371,
      "rewards/chosen": -0.6944470340258455,
      "rewards/margins": 2.5923810180125884,
      "rewards/rejected": -3.286828052038434,
      "step": 930
    },
    {
      "epoch": 0.9729589856385044,
      "grad_norm": 20.680591232042584,
      "kl": 0.0,
      "learning_rate": 5e-07,
      "logps/chosen": -323.59056991185895,
      "logps/rejected": -446.6455792682927,
      "loss": 0.3827,
      "rewards/chosen": -0.837760729667468,
      "rewards/margins": 2.667289185181046,
      "rewards/rejected": -3.505049914848514,
      "step": 940
    },
    {
      "epoch": 0.983309613145297,
      "grad_norm": 27.520713861963205,
      "kl": 0.0,
      "learning_rate": 5e-07,
      "logps/chosen": -321.9452868009868,
      "logps/rejected": -363.61830357142856,
      "loss": 0.3648,
      "rewards/chosen": -0.34084164468865646,
      "rewards/margins": 1.999740703362869,
      "rewards/rejected": -2.3405823480515253,
      "step": 950
    },
    {
      "epoch": 0.9936602406520896,
      "grad_norm": 27.18091953431505,
      "kl": 0.0,
      "learning_rate": 5e-07,
      "logps/chosen": -348.9962173655063,
      "logps/rejected": -325.42737268518516,
      "loss": 0.3878,
      "rewards/chosen": -0.4670451200461086,
      "rewards/margins": 1.8737544706415845,
      "rewards/rejected": -2.340799590687693,
      "step": 960
    },
    {
      "epoch": 0.9998706171561651,
      "step": 966,
      "total_flos": 0.0,
      "train_loss": 0.420091498218955,
      "train_runtime": 6442.4359,
      "train_samples_per_second": 2.399,
      "train_steps_per_second": 0.15
    }
  ],
  "logging_steps": 10,
  "max_steps": 966,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 500,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": false,
        "should_training_stop": false
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 1,
  "trial_name": null,
  "trial_params": null
}