aftonposten-6b-align-scan / trainer_state.json
hugodk-sch's picture
Model save
9d33454 verified
{
"best_metric": null,
"best_model_checkpoint": null,
"epoch": 4.0,
"eval_steps": 100,
"global_step": 1540,
"is_hyper_param_search": false,
"is_local_process_zero": true,
"is_world_process_zero": true,
"log_history": [
{
"epoch": 0.0,
"learning_rate": 1.282051282051282e-07,
"logits/chosen": -1.7278180122375488,
"logits/rejected": -1.7377450466156006,
"logps/chosen": -29.553977966308594,
"logps/rejected": -42.813133239746094,
"loss": 0.5102,
"rewards/accuracies": 0.0,
"rewards/chosen": 0.0,
"rewards/margins": 0.0,
"rewards/rejected": 0.0,
"step": 1
},
{
"epoch": 0.03,
"learning_rate": 1.282051282051282e-06,
"logits/chosen": -1.866772174835205,
"logits/rejected": -1.8710780143737793,
"logps/chosen": -36.99943161010742,
"logps/rejected": -33.657470703125,
"loss": 0.4858,
"rewards/accuracies": 0.5277777910232544,
"rewards/chosen": 0.005016062408685684,
"rewards/margins": 0.02419126406311989,
"rewards/rejected": -0.019175197929143906,
"step": 10
},
{
"epoch": 0.05,
"learning_rate": 2.564102564102564e-06,
"logits/chosen": -1.9975645542144775,
"logits/rejected": -2.0002217292785645,
"logps/chosen": -29.651592254638672,
"logps/rejected": -29.066898345947266,
"loss": 0.5542,
"rewards/accuracies": 0.5,
"rewards/chosen": -0.006579822860658169,
"rewards/margins": -0.004162783268839121,
"rewards/rejected": -0.0024170405231416225,
"step": 20
},
{
"epoch": 0.08,
"learning_rate": 3.846153846153847e-06,
"logits/chosen": -1.9212992191314697,
"logits/rejected": -1.918600082397461,
"logps/chosen": -31.42719078063965,
"logps/rejected": -33.23390579223633,
"loss": 0.5492,
"rewards/accuracies": 0.5249999761581421,
"rewards/chosen": -0.00776728754863143,
"rewards/margins": 0.0030429032631218433,
"rewards/rejected": -0.010810190811753273,
"step": 30
},
{
"epoch": 0.1,
"learning_rate": 4.999896948438434e-06,
"logits/chosen": -2.0178468227386475,
"logits/rejected": -2.009111166000366,
"logps/chosen": -32.579471588134766,
"logps/rejected": -32.50147247314453,
"loss": 0.5878,
"rewards/accuracies": 0.44999998807907104,
"rewards/chosen": -0.001865685684606433,
"rewards/margins": -0.009705344215035439,
"rewards/rejected": 0.007839656434953213,
"step": 40
},
{
"epoch": 0.13,
"learning_rate": 4.987541037542187e-06,
"logits/chosen": -1.8617979288101196,
"logits/rejected": -1.8510282039642334,
"logps/chosen": -33.562862396240234,
"logps/rejected": -35.4472770690918,
"loss": 0.5775,
"rewards/accuracies": 0.42500001192092896,
"rewards/chosen": -0.0037666684947907925,
"rewards/margins": -0.007314275950193405,
"rewards/rejected": 0.0035476074554026127,
"step": 50
},
{
"epoch": 0.16,
"learning_rate": 4.954691471941119e-06,
"logits/chosen": -1.9399009943008423,
"logits/rejected": -1.9418485164642334,
"logps/chosen": -32.5726203918457,
"logps/rejected": -33.18496322631836,
"loss": 0.5967,
"rewards/accuracies": 0.5249999761581421,
"rewards/chosen": 0.019294817000627518,
"rewards/margins": 0.028803948312997818,
"rewards/rejected": -0.009509134106338024,
"step": 60
},
{
"epoch": 0.18,
"learning_rate": 4.901618883413549e-06,
"logits/chosen": -2.0729470252990723,
"logits/rejected": -2.077918529510498,
"logps/chosen": -33.93268585205078,
"logps/rejected": -36.637229919433594,
"loss": 0.5297,
"rewards/accuracies": 0.5874999761581421,
"rewards/chosen": 0.039063483476638794,
"rewards/margins": 0.08691467344760895,
"rewards/rejected": -0.04785118252038956,
"step": 70
},
{
"epoch": 0.21,
"learning_rate": 4.828760511501322e-06,
"logits/chosen": -1.9366486072540283,
"logits/rejected": -1.939780592918396,
"logps/chosen": -34.30228805541992,
"logps/rejected": -34.652069091796875,
"loss": 0.4209,
"rewards/accuracies": 0.637499988079071,
"rewards/chosen": 0.08786438405513763,
"rewards/margins": 0.12962770462036133,
"rewards/rejected": -0.041763320565223694,
"step": 80
},
{
"epoch": 0.23,
"learning_rate": 4.7367166013034295e-06,
"logits/chosen": -1.946979284286499,
"logits/rejected": -1.951521635055542,
"logps/chosen": -32.4210205078125,
"logps/rejected": -32.337799072265625,
"loss": 0.6191,
"rewards/accuracies": 0.5375000238418579,
"rewards/chosen": 0.03248247504234314,
"rewards/margins": 0.017702888697385788,
"rewards/rejected": 0.014779585413634777,
"step": 90
},
{
"epoch": 0.26,
"learning_rate": 4.626245458345211e-06,
"logits/chosen": -2.045017719268799,
"logits/rejected": -2.043015241622925,
"logps/chosen": -32.16361618041992,
"logps/rejected": -31.273128509521484,
"loss": 0.4711,
"rewards/accuracies": 0.5874999761581421,
"rewards/chosen": 0.05405203625559807,
"rewards/margins": 0.05889582633972168,
"rewards/rejected": -0.004843792412430048,
"step": 100
},
{
"epoch": 0.26,
"eval_logits/chosen": -2.240097761154175,
"eval_logits/rejected": -2.2352383136749268,
"eval_logps/chosen": -34.01129150390625,
"eval_logps/rejected": -37.49793243408203,
"eval_loss": 0.5754798054695129,
"eval_rewards/accuracies": 0.5195183157920837,
"eval_rewards/chosen": 0.016279777511954308,
"eval_rewards/margins": 0.0031977586913853884,
"eval_rewards/rejected": 0.013082021847367287,
"eval_runtime": 145.9275,
"eval_samples_per_second": 2.35,
"eval_steps_per_second": 0.295,
"step": 100
},
{
"epoch": 0.29,
"learning_rate": 4.498257201263691e-06,
"logits/chosen": -2.0017762184143066,
"logits/rejected": -1.9993689060211182,
"logps/chosen": -33.13564682006836,
"logps/rejected": -33.997642517089844,
"loss": 0.6759,
"rewards/accuracies": 0.625,
"rewards/chosen": 0.07571890205144882,
"rewards/margins": 0.05350743606686592,
"rewards/rejected": 0.022211460396647453,
"step": 110
},
{
"epoch": 0.31,
"learning_rate": 4.353806263777678e-06,
"logits/chosen": -2.013158082962036,
"logits/rejected": -2.004814624786377,
"logps/chosen": -32.32666015625,
"logps/rejected": -32.12488555908203,
"loss": 0.5572,
"rewards/accuracies": 0.550000011920929,
"rewards/chosen": 0.08277235180139542,
"rewards/margins": 0.05714136362075806,
"rewards/rejected": 0.02563098631799221,
"step": 120
},
{
"epoch": 0.34,
"learning_rate": 4.1940827077152755e-06,
"logits/chosen": -2.039836883544922,
"logits/rejected": -2.031825542449951,
"logps/chosen": -30.349987030029297,
"logps/rejected": -32.029136657714844,
"loss": 0.5109,
"rewards/accuracies": 0.574999988079071,
"rewards/chosen": 0.0880090743303299,
"rewards/margins": 0.07130132615566254,
"rewards/rejected": 0.016707751899957657,
"step": 130
},
{
"epoch": 0.36,
"learning_rate": 4.0204024186666215e-06,
"logits/chosen": -1.9694139957427979,
"logits/rejected": -1.9796711206436157,
"logps/chosen": -31.24441909790039,
"logps/rejected": -32.56440353393555,
"loss": 0.5037,
"rewards/accuracies": 0.6499999761581421,
"rewards/chosen": 0.12266886234283447,
"rewards/margins": 0.14333835244178772,
"rewards/rejected": -0.02066950313746929,
"step": 140
},
{
"epoch": 0.39,
"learning_rate": 3.834196265035119e-06,
"logits/chosen": -1.8828538656234741,
"logits/rejected": -1.8840004205703735,
"logps/chosen": -34.006561279296875,
"logps/rejected": -34.81753921508789,
"loss": 0.4564,
"rewards/accuracies": 0.6625000238418579,
"rewards/chosen": 0.13028928637504578,
"rewards/margins": 0.17248663306236267,
"rewards/rejected": -0.04219735041260719,
"step": 150
},
{
"epoch": 0.42,
"learning_rate": 3.636998309800573e-06,
"logits/chosen": -1.9349613189697266,
"logits/rejected": -1.9315464496612549,
"logps/chosen": -36.059181213378906,
"logps/rejected": -32.72267150878906,
"loss": 0.487,
"rewards/accuracies": 0.6625000238418579,
"rewards/chosen": 0.07874591648578644,
"rewards/margins": 0.07869003713130951,
"rewards/rejected": 5.587190389633179e-05,
"step": 160
},
{
"epoch": 0.44,
"learning_rate": 3.4304331721118078e-06,
"logits/chosen": -2.036032199859619,
"logits/rejected": -2.028651714324951,
"logps/chosen": -33.573509216308594,
"logps/rejected": -31.36956214904785,
"loss": 0.4465,
"rewards/accuracies": 0.6875,
"rewards/chosen": 0.15704451501369476,
"rewards/margins": 0.16944527626037598,
"rewards/rejected": -0.012400749139487743,
"step": 170
},
{
"epoch": 0.47,
"learning_rate": 3.2162026428305436e-06,
"logits/chosen": -2.042617082595825,
"logits/rejected": -2.0478556156158447,
"logps/chosen": -32.33074188232422,
"logps/rejected": -32.472496032714844,
"loss": 0.3727,
"rewards/accuracies": 0.7250000238418579,
"rewards/chosen": 0.14983226358890533,
"rewards/margins": 0.14098060131072998,
"rewards/rejected": 0.008851657621562481,
"step": 180
},
{
"epoch": 0.49,
"learning_rate": 2.996071664294641e-06,
"logits/chosen": -2.0445618629455566,
"logits/rejected": -2.0417864322662354,
"logps/chosen": -31.317855834960938,
"logps/rejected": -31.348400115966797,
"loss": 0.4609,
"rewards/accuracies": 0.7250000238418579,
"rewards/chosen": 0.11976919323205948,
"rewards/margins": 0.14090058207511902,
"rewards/rejected": -0.021131375804543495,
"step": 190
},
{
"epoch": 0.52,
"learning_rate": 2.7718537898066833e-06,
"logits/chosen": -1.9139589071273804,
"logits/rejected": -1.9186115264892578,
"logps/chosen": -31.420101165771484,
"logps/rejected": -32.79216766357422,
"loss": 0.5061,
"rewards/accuracies": 0.675000011920929,
"rewards/chosen": 0.13032536208629608,
"rewards/margins": 0.13119173049926758,
"rewards/rejected": -0.0008663811022415757,
"step": 200
},
{
"epoch": 0.52,
"eval_logits/chosen": -2.2385354042053223,
"eval_logits/rejected": -2.2336885929107666,
"eval_logps/chosen": -34.05002975463867,
"eval_logps/rejected": -37.54545974731445,
"eval_loss": 0.5877167582511902,
"eval_rewards/accuracies": 0.4991694688796997,
"eval_rewards/chosen": -0.010832725092768669,
"eval_rewards/margins": 0.009353035129606724,
"eval_rewards/rejected": -0.020185761153697968,
"eval_runtime": 145.6958,
"eval_samples_per_second": 2.354,
"eval_steps_per_second": 0.295,
"step": 200
},
{
"epoch": 0.55,
"learning_rate": 2.5453962426402006e-06,
"logits/chosen": -2.026780366897583,
"logits/rejected": -2.0374534130096436,
"logps/chosen": -31.786846160888672,
"logps/rejected": -33.922447204589844,
"loss": 0.4225,
"rewards/accuracies": 0.675000011920929,
"rewards/chosen": 0.12911871075630188,
"rewards/margins": 0.17094141244888306,
"rewards/rejected": -0.041822709143161774,
"step": 210
},
{
"epoch": 0.57,
"learning_rate": 2.3185646976551794e-06,
"logits/chosen": -1.9182548522949219,
"logits/rejected": -1.933038353919983,
"logps/chosen": -29.936147689819336,
"logps/rejected": -31.559993743896484,
"loss": 0.4429,
"rewards/accuracies": 0.7124999761581421,
"rewards/chosen": 0.12277615070343018,
"rewards/margins": 0.12404797971248627,
"rewards/rejected": -0.0012718416983261704,
"step": 220
},
{
"epoch": 0.6,
"learning_rate": 2.0932279108998323e-06,
"logits/chosen": -1.9737575054168701,
"logits/rejected": -1.9777438640594482,
"logps/chosen": -33.20977783203125,
"logps/rejected": -31.5734920501709,
"loss": 0.3768,
"rewards/accuracies": 0.637499988079071,
"rewards/chosen": 0.14538228511810303,
"rewards/margins": 0.16231416165828705,
"rewards/rejected": -0.016931891441345215,
"step": 230
},
{
"epoch": 0.62,
"learning_rate": 1.8712423238279358e-06,
"logits/chosen": -1.9738047122955322,
"logits/rejected": -1.9518985748291016,
"logps/chosen": -33.94301223754883,
"logps/rejected": -35.023075103759766,
"loss": 0.4304,
"rewards/accuracies": 0.7250000238418579,
"rewards/chosen": 0.1475195735692978,
"rewards/margins": 0.19811663031578064,
"rewards/rejected": -0.05059707164764404,
"step": 240
},
{
"epoch": 0.65,
"learning_rate": 1.6544367689701824e-06,
"logits/chosen": -2.0155911445617676,
"logits/rejected": -2.0122878551483154,
"logps/chosen": -32.76276397705078,
"logps/rejected": -36.25225830078125,
"loss": 0.4155,
"rewards/accuracies": 0.637499988079071,
"rewards/chosen": 0.11361332982778549,
"rewards/margins": 0.1327473670244217,
"rewards/rejected": -0.019134048372507095,
"step": 250
},
{
"epoch": 0.68,
"learning_rate": 1.4445974030621963e-06,
"logits/chosen": -1.8830368518829346,
"logits/rejected": -1.880629539489746,
"logps/chosen": -34.01710891723633,
"logps/rejected": -35.53314971923828,
"loss": 0.4219,
"rewards/accuracies": 0.7250000238418579,
"rewards/chosen": 0.11990444362163544,
"rewards/margins": 0.1398761123418808,
"rewards/rejected": -0.019971664994955063,
"step": 260
},
{
"epoch": 0.7,
"learning_rate": 1.243452991757889e-06,
"logits/chosen": -1.8685131072998047,
"logits/rejected": -1.8660367727279663,
"logps/chosen": -34.2363395690918,
"logps/rejected": -31.768321990966797,
"loss": 0.4918,
"rewards/accuracies": 0.675000011920929,
"rewards/chosen": 0.11288031190633774,
"rewards/margins": 0.11077453196048737,
"rewards/rejected": 0.0021057710982859135,
"step": 270
},
{
"epoch": 0.73,
"learning_rate": 1.0526606671603523e-06,
"logits/chosen": -1.9722087383270264,
"logits/rejected": -1.961612343788147,
"logps/chosen": -35.04926300048828,
"logps/rejected": -31.826486587524414,
"loss": 0.3417,
"rewards/accuracies": 0.737500011920929,
"rewards/chosen": 0.21516695618629456,
"rewards/margins": 0.20501188933849335,
"rewards/rejected": 0.010155050083994865,
"step": 280
},
{
"epoch": 0.75,
"learning_rate": 8.737922755071455e-07,
"logits/chosen": -2.068380832672119,
"logits/rejected": -2.05340313911438,
"logps/chosen": -30.748706817626953,
"logps/rejected": -32.63469314575195,
"loss": 0.4741,
"rewards/accuracies": 0.625,
"rewards/chosen": 0.12279413640499115,
"rewards/margins": 0.11079368740320206,
"rewards/rejected": 0.012000440619885921,
"step": 290
},
{
"epoch": 0.78,
"learning_rate": 7.08321427484816e-07,
"logits/chosen": -1.9397351741790771,
"logits/rejected": -1.9371974468231201,
"logps/chosen": -32.603660583496094,
"logps/rejected": -30.863941192626953,
"loss": 0.3371,
"rewards/accuracies": 0.7749999761581421,
"rewards/chosen": 0.2264125794172287,
"rewards/margins": 0.25940603017807007,
"rewards/rejected": -0.03299345448613167,
"step": 300
},
{
"epoch": 0.78,
"eval_logits/chosen": -2.2370686531066895,
"eval_logits/rejected": -2.2322022914886475,
"eval_logps/chosen": -34.03439712524414,
"eval_logps/rejected": -37.53529357910156,
"eval_loss": 0.5842657089233398,
"eval_rewards/accuracies": 0.5278239250183105,
"eval_rewards/chosen": 0.00010742468293756247,
"eval_rewards/margins": 0.013178782537579536,
"eval_rewards/rejected": -0.013071359135210514,
"eval_runtime": 145.5131,
"eval_samples_per_second": 2.357,
"eval_steps_per_second": 0.296,
"step": 300
},
{
"epoch": 0.81,
"grad_norm": 101.5,
"learning_rate": 4.84533120650964e-06,
"logits/chosen": -2.0717549324035645,
"logits/rejected": -2.058924913406372,
"logps/chosen": -32.21288299560547,
"logps/rejected": -32.803016662597656,
"loss": 0.2644,
"rewards/accuracies": 0.887499988079071,
"rewards/chosen": 0.22085602581501007,
"rewards/margins": 0.3342258632183075,
"rewards/rejected": -0.11336983740329742,
"step": 310
},
{
"epoch": 0.83,
"grad_norm": 28.625,
"learning_rate": 4.825108134172131e-06,
"logits/chosen": -1.98604416847229,
"logits/rejected": -1.9773375988006592,
"logps/chosen": -32.002044677734375,
"logps/rejected": -30.32120132446289,
"loss": 0.2537,
"rewards/accuracies": 0.8500000238418579,
"rewards/chosen": 0.24706146121025085,
"rewards/margins": 0.3029271960258484,
"rewards/rejected": -0.055865705013275146,
"step": 320
},
{
"epoch": 0.86,
"grad_norm": 23.125,
"learning_rate": 4.80369052967602e-06,
"logits/chosen": -1.923841118812561,
"logits/rejected": -1.935959815979004,
"logps/chosen": -30.132516860961914,
"logps/rejected": -33.53127670288086,
"loss": 0.3016,
"rewards/accuracies": 0.824999988079071,
"rewards/chosen": 0.2427288293838501,
"rewards/margins": 0.3330441117286682,
"rewards/rejected": -0.0903153270483017,
"step": 330
},
{
"epoch": 0.88,
"grad_norm": 56.5,
"learning_rate": 4.781089396387968e-06,
"logits/chosen": -1.8918178081512451,
"logits/rejected": -1.8825798034667969,
"logps/chosen": -34.266075134277344,
"logps/rejected": -36.046722412109375,
"loss": 0.283,
"rewards/accuracies": 0.887499988079071,
"rewards/chosen": 0.3029957413673401,
"rewards/margins": 0.4359920620918274,
"rewards/rejected": -0.1329963356256485,
"step": 340
},
{
"epoch": 0.91,
"grad_norm": 44.0,
"learning_rate": 4.757316345716554e-06,
"logits/chosen": -1.950615644454956,
"logits/rejected": -1.951266884803772,
"logps/chosen": -34.006526947021484,
"logps/rejected": -33.97644805908203,
"loss": 0.2869,
"rewards/accuracies": 0.862500011920929,
"rewards/chosen": 0.2791282534599304,
"rewards/margins": 0.3683601915836334,
"rewards/rejected": -0.08923190832138062,
"step": 350
},
{
"epoch": 0.94,
"grad_norm": 54.25,
"learning_rate": 4.73238359114687e-06,
"logits/chosen": -2.07924222946167,
"logits/rejected": -2.0854721069335938,
"logps/chosen": -31.282617568969727,
"logps/rejected": -32.800941467285156,
"loss": 0.3061,
"rewards/accuracies": 0.800000011920929,
"rewards/chosen": 0.2034403532743454,
"rewards/margins": 0.296302855014801,
"rewards/rejected": -0.09286249428987503,
"step": 360
},
{
"epoch": 0.96,
"grad_norm": 172.0,
"learning_rate": 4.706303941965804e-06,
"logits/chosen": -2.011671781539917,
"logits/rejected": -2.0111806392669678,
"logps/chosen": -33.01143264770508,
"logps/rejected": -36.115821838378906,
"loss": 0.4197,
"rewards/accuracies": 0.8374999761581421,
"rewards/chosen": 0.3034539818763733,
"rewards/margins": 0.349683940410614,
"rewards/rejected": -0.04622991383075714,
"step": 370
},
{
"epoch": 0.99,
"grad_norm": 32.5,
"learning_rate": 4.679090796681225e-06,
"logits/chosen": -2.047156572341919,
"logits/rejected": -2.04251766204834,
"logps/chosen": -30.326885223388672,
"logps/rejected": -29.325164794921875,
"loss": 0.2942,
"rewards/accuracies": 0.887499988079071,
"rewards/chosen": 0.26859989762306213,
"rewards/margins": 0.3464903235435486,
"rewards/rejected": -0.07789044082164764,
"step": 380
},
{
"epoch": 1.01,
"grad_norm": 52.25,
"learning_rate": 4.650758136138454e-06,
"logits/chosen": -1.8196346759796143,
"logits/rejected": -1.8260679244995117,
"logps/chosen": -32.0097541809082,
"logps/rejected": -36.38646697998047,
"loss": 0.4763,
"rewards/accuracies": 0.887499988079071,
"rewards/chosen": 0.26317089796066284,
"rewards/margins": 0.450895220041275,
"rewards/rejected": -0.1877242624759674,
"step": 390
},
{
"epoch": 1.04,
"grad_norm": 55.75,
"learning_rate": 4.621320516337559e-06,
"logits/chosen": -1.9777911901474,
"logits/rejected": -1.9714463949203491,
"logps/chosen": -33.3974609375,
"logps/rejected": -32.44751739501953,
"loss": 0.4001,
"rewards/accuracies": 0.8125,
"rewards/chosen": 0.3046652674674988,
"rewards/margins": 0.40171337127685547,
"rewards/rejected": -0.09704810380935669,
"step": 400
},
{
"epoch": 1.04,
"eval_logits/chosen": -2.2401604652404785,
"eval_logits/rejected": -2.235311508178711,
"eval_logps/chosen": -34.04499435424805,
"eval_logps/rejected": -37.51197052001953,
"eval_loss": 0.6349518299102783,
"eval_rewards/accuracies": 0.48380398750305176,
"eval_rewards/chosen": -0.00731015345081687,
"eval_rewards/margins": -0.010565591044723988,
"eval_rewards/rejected": 0.003255437593907118,
"eval_runtime": 145.4219,
"eval_samples_per_second": 2.359,
"eval_steps_per_second": 0.296,
"step": 400
},
{
"epoch": 1.06,
"grad_norm": 32.75,
"learning_rate": 4.590793060955158e-06,
"logits/chosen": -1.9763590097427368,
"logits/rejected": -1.9835665225982666,
"logps/chosen": -28.756824493408203,
"logps/rejected": -29.339588165283203,
"loss": 0.3007,
"rewards/accuracies": 0.9375,
"rewards/chosen": 0.2544856667518616,
"rewards/margins": 0.42344123125076294,
"rewards/rejected": -0.16895556449890137,
"step": 410
},
{
"epoch": 1.09,
"grad_norm": 83.0,
"learning_rate": 4.559191453574582e-06,
"logits/chosen": -1.9966684579849243,
"logits/rejected": -1.9955562353134155,
"logps/chosen": -33.83961868286133,
"logps/rejected": -31.041629791259766,
"loss": 0.3581,
"rewards/accuracies": 0.824999988079071,
"rewards/chosen": 0.28776243329048157,
"rewards/margins": 0.3567716181278229,
"rewards/rejected": -0.06900922954082489,
"step": 420
},
{
"epoch": 1.12,
"grad_norm": 96.0,
"learning_rate": 4.52653192962838e-06,
"logits/chosen": -1.9937080144882202,
"logits/rejected": -1.9763784408569336,
"logps/chosen": -30.642507553100586,
"logps/rejected": -33.117218017578125,
"loss": 0.5267,
"rewards/accuracies": 0.8999999761581421,
"rewards/chosen": 0.2342870533466339,
"rewards/margins": 0.36793988943099976,
"rewards/rejected": -0.13365286588668823,
"step": 430
},
{
"epoch": 1.14,
"grad_norm": 43.75,
"learning_rate": 4.492831268057307e-06,
"logits/chosen": -2.0257625579833984,
"logits/rejected": -2.0276401042938232,
"logps/chosen": -36.22648239135742,
"logps/rejected": -35.008148193359375,
"loss": 0.361,
"rewards/accuracies": 0.875,
"rewards/chosen": 0.2701130509376526,
"rewards/margins": 0.37383943796157837,
"rewards/rejected": -0.10372640937566757,
"step": 440
},
{
"epoch": 1.17,
"grad_norm": 24.875,
"learning_rate": 4.458106782690094e-06,
"logits/chosen": -2.1039767265319824,
"logits/rejected": -2.103851079940796,
"logps/chosen": -32.201255798339844,
"logps/rejected": -33.596595764160156,
"loss": 0.3431,
"rewards/accuracies": 0.8500000238418579,
"rewards/chosen": 0.30246877670288086,
"rewards/margins": 0.3280053436756134,
"rewards/rejected": -0.025536546483635902,
"step": 450
},
{
"epoch": 1.19,
"grad_norm": 73.0,
"learning_rate": 4.422376313348405e-06,
"logits/chosen": -2.0463452339172363,
"logits/rejected": -2.0388851165771484,
"logps/chosen": -32.02504348754883,
"logps/rejected": -36.26923370361328,
"loss": 0.504,
"rewards/accuracies": 0.800000011920929,
"rewards/chosen": 0.2666401267051697,
"rewards/margins": 0.41727694869041443,
"rewards/rejected": -0.15063682198524475,
"step": 460
},
{
"epoch": 1.22,
"grad_norm": 74.0,
"learning_rate": 4.3856582166815696e-06,
"logits/chosen": -1.9466873407363892,
"logits/rejected": -1.9430792331695557,
"logps/chosen": -33.49795913696289,
"logps/rejected": -33.04600143432617,
"loss": 0.2362,
"rewards/accuracies": 0.887499988079071,
"rewards/chosen": 0.3040338456630707,
"rewards/margins": 0.38951820135116577,
"rewards/rejected": -0.08548439294099808,
"step": 470
},
{
"epoch": 1.25,
"grad_norm": 74.0,
"learning_rate": 4.347971356735789e-06,
"logits/chosen": -2.0715551376342773,
"logits/rejected": -2.0645508766174316,
"logps/chosen": -30.481578826904297,
"logps/rejected": -32.14142990112305,
"loss": 0.4149,
"rewards/accuracies": 0.800000011920929,
"rewards/chosen": 0.242868572473526,
"rewards/margins": 0.3464292287826538,
"rewards/rejected": -0.10356064140796661,
"step": 480
},
{
"epoch": 1.27,
"grad_norm": 97.5,
"learning_rate": 4.309335095262675e-06,
"logits/chosen": -2.018632411956787,
"logits/rejected": -2.020221710205078,
"logps/chosen": -34.90156936645508,
"logps/rejected": -33.80845260620117,
"loss": 0.3635,
"rewards/accuracies": 0.824999988079071,
"rewards/chosen": 0.3161340653896332,
"rewards/margins": 0.4329928457736969,
"rewards/rejected": -0.11685874313116074,
"step": 490
},
{
"epoch": 1.3,
"grad_norm": 64.0,
"learning_rate": 4.269769281772082e-06,
"logits/chosen": -1.9072506427764893,
"logits/rejected": -1.9048874378204346,
"logps/chosen": -32.74415588378906,
"logps/rejected": -36.70649337768555,
"loss": 0.3401,
"rewards/accuracies": 0.9125000238418579,
"rewards/chosen": 0.2996928095817566,
"rewards/margins": 0.469214528799057,
"rewards/rejected": -0.16952167451381683,
"step": 500
},
{
"epoch": 1.3,
"eval_logits/chosen": -2.2401528358459473,
"eval_logits/rejected": -2.235276937484741,
"eval_logps/chosen": -34.05385971069336,
"eval_logps/rejected": -37.54425811767578,
"eval_loss": 0.623750627040863,
"eval_rewards/accuracies": 0.5141196250915527,
"eval_rewards/chosen": -0.013515940867364407,
"eval_rewards/margins": 0.005829531699419022,
"eval_rewards/rejected": -0.019345473498106003,
"eval_runtime": 145.1627,
"eval_samples_per_second": 2.363,
"eval_steps_per_second": 0.296,
"step": 500
},
{
"epoch": 1.32,
"grad_norm": 43.75,
"learning_rate": 4.22929424333435e-06,
"logits/chosen": -2.0122382640838623,
"logits/rejected": -2.0169613361358643,
"logps/chosen": -33.01191329956055,
"logps/rejected": -31.77316665649414,
"loss": 0.3867,
"rewards/accuracies": 0.800000011920929,
"rewards/chosen": 0.31180983781814575,
"rewards/margins": 0.40075111389160156,
"rewards/rejected": -0.08894126117229462,
"step": 510
},
{
"epoch": 1.35,
"grad_norm": 47.0,
"learning_rate": 4.1879307741372085e-06,
"logits/chosen": -2.045281171798706,
"logits/rejected": -2.0561985969543457,
"logps/chosen": -30.94211196899414,
"logps/rejected": -31.70987892150879,
"loss": 0.4115,
"rewards/accuracies": 0.862500011920929,
"rewards/chosen": 0.31381234526634216,
"rewards/margins": 0.455346018075943,
"rewards/rejected": -0.14153364300727844,
"step": 520
},
{
"epoch": 1.38,
"grad_norm": 69.0,
"learning_rate": 4.145700124802693e-06,
"logits/chosen": -1.9737331867218018,
"logits/rejected": -1.9702527523040771,
"logps/chosen": -31.990686416625977,
"logps/rejected": -32.61437225341797,
"loss": 0.4131,
"rewards/accuracies": 0.8999999761581421,
"rewards/chosen": 0.3008785843849182,
"rewards/margins": 0.4769964814186096,
"rewards/rejected": -0.1761178821325302,
"step": 530
},
{
"epoch": 1.4,
"grad_norm": 52.75,
"learning_rate": 4.102623991469562e-06,
"logits/chosen": -1.8443644046783447,
"logits/rejected": -1.853546142578125,
"logps/chosen": -32.251625061035156,
"logps/rejected": -32.053138732910156,
"loss": 0.3856,
"rewards/accuracies": 0.824999988079071,
"rewards/chosen": 0.22243666648864746,
"rewards/margins": 0.3604408800601959,
"rewards/rejected": -0.13800422847270966,
"step": 540
},
{
"epoch": 1.43,
"grad_norm": 58.75,
"learning_rate": 4.058724504646834e-06,
"logits/chosen": -1.9435665607452393,
"logits/rejected": -1.9371446371078491,
"logps/chosen": -33.34047317504883,
"logps/rejected": -31.121173858642578,
"loss": 0.3465,
"rewards/accuracies": 0.8125,
"rewards/chosen": 0.2574843168258667,
"rewards/margins": 0.34836047887802124,
"rewards/rejected": -0.09087616950273514,
"step": 550
},
{
"epoch": 1.45,
"grad_norm": 135.0,
"learning_rate": 4.014024217844167e-06,
"logits/chosen": -2.0361862182617188,
"logits/rejected": -2.0341010093688965,
"logps/chosen": -34.12422561645508,
"logps/rejected": -31.634075164794922,
"loss": 0.6034,
"rewards/accuracies": 0.8374999761581421,
"rewards/chosen": 0.3128809928894043,
"rewards/margins": 0.40856558084487915,
"rewards/rejected": -0.09568462520837784,
"step": 560
},
{
"epoch": 1.48,
"grad_norm": 52.75,
"learning_rate": 3.968546095984911e-06,
"logits/chosen": -1.8710432052612305,
"logits/rejected": -1.8688808679580688,
"logps/chosen": -32.49556350708008,
"logps/rejected": -31.1130428314209,
"loss": 0.4937,
"rewards/accuracies": 0.8125,
"rewards/chosen": 0.21059513092041016,
"rewards/margins": 0.2881118357181549,
"rewards/rejected": -0.07751675695180893,
"step": 570
},
{
"epoch": 1.51,
"grad_norm": 95.5,
"learning_rate": 3.922313503607806e-06,
"logits/chosen": -2.005981922149658,
"logits/rejected": -2.0027260780334473,
"logps/chosen": -30.789072036743164,
"logps/rejected": -34.820518493652344,
"loss": 0.4043,
"rewards/accuracies": 0.800000011920929,
"rewards/chosen": 0.22090163826942444,
"rewards/margins": 0.356947124004364,
"rewards/rejected": -0.13604548573493958,
"step": 580
},
{
"epoch": 1.53,
"grad_norm": 20.25,
"learning_rate": 3.875350192863368e-06,
"logits/chosen": -1.948760986328125,
"logits/rejected": -1.9524637460708618,
"logps/chosen": -29.25394630432129,
"logps/rejected": -30.857736587524414,
"loss": 0.3681,
"rewards/accuracies": 0.8125,
"rewards/chosen": 0.2527596354484558,
"rewards/margins": 0.3919641375541687,
"rewards/rejected": -0.1392044574022293,
"step": 590
},
{
"epoch": 1.56,
"grad_norm": 28.125,
"learning_rate": 3.8276802913111436e-06,
"logits/chosen": -1.9878416061401367,
"logits/rejected": -1.9875671863555908,
"logps/chosen": -31.794857025146484,
"logps/rejected": -31.33709716796875,
"loss": 0.433,
"rewards/accuracies": 0.887499988079071,
"rewards/chosen": 0.2986726462841034,
"rewards/margins": 0.3995324373245239,
"rewards/rejected": -0.10085982084274292,
"step": 600
},
{
"epoch": 1.56,
"eval_logits/chosen": -2.2469263076782227,
"eval_logits/rejected": -2.2420639991760254,
"eval_logps/chosen": -34.01606369018555,
"eval_logps/rejected": -37.50114822387695,
"eval_loss": 0.6143299341201782,
"eval_rewards/accuracies": 0.5245016813278198,
"eval_rewards/chosen": 0.012940882705152035,
"eval_rewards/margins": 0.002107805572450161,
"eval_rewards/rejected": 0.010833078064024448,
"eval_runtime": 145.3485,
"eval_samples_per_second": 2.36,
"eval_steps_per_second": 0.296,
"step": 600
},
{
"epoch": 1.58,
"grad_norm": 77.5,
"learning_rate": 3.7793282895240927e-06,
"logits/chosen": -2.05083966255188,
"logits/rejected": -2.0515644550323486,
"logps/chosen": -34.6113395690918,
"logps/rejected": -33.167572021484375,
"loss": 0.3998,
"rewards/accuracies": 0.8125,
"rewards/chosen": 0.29316502809524536,
"rewards/margins": 0.3515344560146332,
"rewards/rejected": -0.058369409292936325,
"step": 610
},
{
"epoch": 1.61,
"grad_norm": 91.5,
"learning_rate": 3.730319028506478e-06,
"logits/chosen": -2.01389741897583,
"logits/rejected": -2.0112407207489014,
"logps/chosen": -32.689491271972656,
"logps/rejected": -32.25727081298828,
"loss": 0.3825,
"rewards/accuracies": 0.862500011920929,
"rewards/chosen": 0.3051331341266632,
"rewards/margins": 0.4362831711769104,
"rewards/rejected": -0.1311500072479248,
"step": 620
},
{
"epoch": 1.64,
"grad_norm": 39.25,
"learning_rate": 3.6806776869317074e-06,
"logits/chosen": -2.028050184249878,
"logits/rejected": -2.018892765045166,
"logps/chosen": -32.39860153198242,
"logps/rejected": -30.971405029296875,
"loss": 0.5713,
"rewards/accuracies": 0.762499988079071,
"rewards/chosen": 0.26426073908805847,
"rewards/margins": 0.28948652744293213,
"rewards/rejected": -0.02522580698132515,
"step": 630
},
{
"epoch": 1.66,
"grad_norm": 32.25,
"learning_rate": 3.6304297682067146e-06,
"logits/chosen": -2.0308046340942383,
"logits/rejected": -2.027677059173584,
"logps/chosen": -31.706838607788086,
"logps/rejected": -32.60099792480469,
"loss": 0.2067,
"rewards/accuracies": 0.9624999761581421,
"rewards/chosen": 0.3441086411476135,
"rewards/margins": 0.4712149500846863,
"rewards/rejected": -0.12710630893707275,
"step": 640
},
{
"epoch": 1.69,
"grad_norm": 27.875,
"learning_rate": 3.579601087369492e-06,
"logits/chosen": -2.03548264503479,
"logits/rejected": -2.0378546714782715,
"logps/chosen": -33.151695251464844,
"logps/rejected": -33.8459587097168,
"loss": 0.3123,
"rewards/accuracies": 0.9125000238418579,
"rewards/chosen": 0.3514065742492676,
"rewards/margins": 0.465198814868927,
"rewards/rejected": -0.11379221826791763,
"step": 650
},
{
"epoch": 1.71,
"grad_norm": 47.5,
"learning_rate": 3.5282177578265295e-06,
"logits/chosen": -1.9493051767349243,
"logits/rejected": -1.9498169422149658,
"logps/chosen": -33.52117156982422,
"logps/rejected": -31.683589935302734,
"loss": 0.3393,
"rewards/accuracies": 0.8125,
"rewards/chosen": 0.2979589104652405,
"rewards/margins": 0.35139504075050354,
"rewards/rejected": -0.053436122834682465,
"step": 660
},
{
"epoch": 1.74,
"grad_norm": 60.25,
"learning_rate": 3.476306177936961e-06,
"logits/chosen": -1.9889039993286133,
"logits/rejected": -1.9790817499160767,
"logps/chosen": -33.58037567138672,
"logps/rejected": -32.429481506347656,
"loss": 0.2419,
"rewards/accuracies": 0.887499988079071,
"rewards/chosen": 0.2946351170539856,
"rewards/margins": 0.37981870770454407,
"rewards/rejected": -0.08518362790346146,
"step": 670
},
{
"epoch": 1.77,
"grad_norm": 58.75,
"learning_rate": 3.423893017450324e-06,
"logits/chosen": -1.8874107599258423,
"logits/rejected": -1.8842309713363647,
"logps/chosen": -30.960819244384766,
"logps/rejected": -34.40943145751953,
"loss": 0.2598,
"rewards/accuracies": 0.862500011920929,
"rewards/chosen": 0.35009804368019104,
"rewards/margins": 0.41369539499282837,
"rewards/rejected": -0.06359735876321793,
"step": 680
},
{
"epoch": 1.79,
"grad_norm": 114.5,
"learning_rate": 3.3710052038048794e-06,
"logits/chosen": -1.944596290588379,
"logits/rejected": -1.9437839984893799,
"logps/chosen": -34.76592254638672,
"logps/rejected": -35.450462341308594,
"loss": 0.3155,
"rewards/accuracies": 0.8999999761581421,
"rewards/chosen": 0.3530573844909668,
"rewards/margins": 0.42070499062538147,
"rewards/rejected": -0.06764759868383408,
"step": 690
},
{
"epoch": 1.82,
"grad_norm": 35.25,
"learning_rate": 3.3176699082935546e-06,
"logits/chosen": -1.9258012771606445,
"logits/rejected": -1.928860068321228,
"logps/chosen": -32.108642578125,
"logps/rejected": -35.734214782714844,
"loss": 0.3298,
"rewards/accuracies": 0.875,
"rewards/chosen": 0.3459445536136627,
"rewards/margins": 0.4107227325439453,
"rewards/rejected": -0.06477821618318558,
"step": 700
},
{
"epoch": 1.82,
"eval_logits/chosen": -2.244976043701172,
"eval_logits/rejected": -2.240100622177124,
"eval_logps/chosen": -33.94416046142578,
"eval_logps/rejected": -37.445308685302734,
"eval_loss": 0.579022228717804,
"eval_rewards/accuracies": 0.5195183157920837,
"eval_rewards/chosen": 0.06327207386493683,
"eval_rewards/margins": 0.013353652320802212,
"eval_rewards/rejected": 0.049918416887521744,
"eval_runtime": 145.2212,
"eval_samples_per_second": 2.362,
"eval_steps_per_second": 0.296,
"step": 700
},
{
"epoch": 1.84,
"grad_norm": 43.5,
"learning_rate": 3.2639145321045933e-06,
"logits/chosen": -2.0379347801208496,
"logits/rejected": -2.0407185554504395,
"logps/chosen": -34.414249420166016,
"logps/rejected": -34.340797424316406,
"loss": 0.4057,
"rewards/accuracies": 0.8125,
"rewards/chosen": 0.3369380831718445,
"rewards/margins": 0.38638633489608765,
"rewards/rejected": -0.04944825917482376,
"step": 710
},
{
"epoch": 1.87,
"grad_norm": 105.5,
"learning_rate": 3.2097666922441107e-06,
"logits/chosen": -1.8936197757720947,
"logits/rejected": -1.8877111673355103,
"logps/chosen": -34.38829040527344,
"logps/rejected": -32.45317459106445,
"loss": 0.3661,
"rewards/accuracies": 0.862500011920929,
"rewards/chosen": 0.31145918369293213,
"rewards/margins": 0.3823915123939514,
"rewards/rejected": -0.0709322914481163,
"step": 720
},
{
"epoch": 1.9,
"grad_norm": 34.75,
"learning_rate": 3.1552542073477554e-06,
"logits/chosen": -2.068476438522339,
"logits/rejected": -2.065448045730591,
"logps/chosen": -30.17218017578125,
"logps/rejected": -31.612863540649414,
"loss": 0.3787,
"rewards/accuracies": 0.9125000238418579,
"rewards/chosen": 0.3365642726421356,
"rewards/margins": 0.45183300971984863,
"rewards/rejected": -0.11526870727539062,
"step": 730
},
{
"epoch": 1.92,
"grad_norm": 77.0,
"learning_rate": 3.100405083388799e-06,
"logits/chosen": -1.911435842514038,
"logits/rejected": -1.9112478494644165,
"logps/chosen": -32.99172592163086,
"logps/rejected": -37.709041595458984,
"loss": 0.3501,
"rewards/accuracies": 0.8125,
"rewards/chosen": 0.3300129473209381,
"rewards/margins": 0.3868759870529175,
"rewards/rejected": -0.05686303228139877,
"step": 740
},
{
"epoch": 1.95,
"grad_norm": 93.0,
"learning_rate": 3.0452474992899645e-06,
"logits/chosen": -1.7933385372161865,
"logits/rejected": -1.7985591888427734,
"logps/chosen": -36.41147994995117,
"logps/rejected": -33.99577713012695,
"loss": 0.3683,
"rewards/accuracies": 0.8374999761581421,
"rewards/chosen": 0.364685595035553,
"rewards/margins": 0.38109368085861206,
"rewards/rejected": -0.016408095136284828,
"step": 750
},
{
"epoch": 1.97,
"grad_norm": 111.5,
"learning_rate": 2.989809792446417e-06,
"logits/chosen": -2.0044448375701904,
"logits/rejected": -2.0057899951934814,
"logps/chosen": -32.149024963378906,
"logps/rejected": -32.881797790527344,
"loss": 0.3432,
"rewards/accuracies": 0.862500011920929,
"rewards/chosen": 0.3196606934070587,
"rewards/margins": 0.39146822690963745,
"rewards/rejected": -0.07180755585432053,
"step": 760
},
{
"epoch": 2.0,
"grad_norm": 71.0,
"learning_rate": 2.9341204441673267e-06,
"logits/chosen": -1.9735772609710693,
"logits/rejected": -1.9727319478988647,
"logps/chosen": -31.752365112304688,
"logps/rejected": -35.02272033691406,
"loss": 0.3104,
"rewards/accuracies": 0.8125,
"rewards/chosen": 0.2962704002857208,
"rewards/margins": 0.29246559739112854,
"rewards/rejected": 0.0038047723937779665,
"step": 770
},
{
"epoch": 2.03,
"grad_norm": 27.0,
"learning_rate": 2.878208065043501e-06,
"logits/chosen": -1.9740774631500244,
"logits/rejected": -1.9732996225357056,
"logps/chosen": -34.172630310058594,
"logps/rejected": -31.525737762451172,
"loss": 0.2392,
"rewards/accuracies": 0.862500011920929,
"rewards/chosen": 0.33308273553848267,
"rewards/margins": 0.40743428468704224,
"rewards/rejected": -0.07435150444507599,
"step": 780
},
{
"epoch": 2.05,
"grad_norm": 32.0,
"learning_rate": 2.8221013802485974e-06,
"logits/chosen": -2.00411057472229,
"logits/rejected": -2.0026121139526367,
"logps/chosen": -29.090576171875,
"logps/rejected": -32.80150604248047,
"loss": 0.166,
"rewards/accuracies": 0.9624999761581421,
"rewards/chosen": 0.3308223783969879,
"rewards/margins": 0.4464591145515442,
"rewards/rejected": -0.11563672870397568,
"step": 790
},
{
"epoch": 2.08,
"grad_norm": 28.625,
"learning_rate": 2.76582921478147e-06,
"logits/chosen": -2.0544497966766357,
"logits/rejected": -2.05083966255188,
"logps/chosen": -31.90934181213379,
"logps/rejected": -34.358177185058594,
"loss": 0.14,
"rewards/accuracies": 0.925000011920929,
"rewards/chosen": 0.36250725388526917,
"rewards/margins": 0.4739067554473877,
"rewards/rejected": -0.11139953136444092,
"step": 800
},
{
"epoch": 2.08,
"eval_logits/chosen": -2.247178077697754,
"eval_logits/rejected": -2.2423007488250732,
"eval_logps/chosen": -33.95090103149414,
"eval_logps/rejected": -37.43887710571289,
"eval_loss": 0.5903980731964111,
"eval_rewards/accuracies": 0.5161960124969482,
"eval_rewards/chosen": 0.05855472758412361,
"eval_rewards/margins": 0.0041350084356963634,
"eval_rewards/rejected": 0.05441971868276596,
"eval_runtime": 145.1787,
"eval_samples_per_second": 2.363,
"eval_steps_per_second": 0.296,
"step": 800
},
{
"epoch": 2.1,
"grad_norm": 104.0,
"learning_rate": 2.7094204786572254e-06,
"logits/chosen": -1.8908456563949585,
"logits/rejected": -1.8829033374786377,
"logps/chosen": -33.954002380371094,
"logps/rejected": -34.61323165893555,
"loss": 0.3154,
"rewards/accuracies": 0.8374999761581421,
"rewards/chosen": 0.3515668511390686,
"rewards/margins": 0.38875049352645874,
"rewards/rejected": -0.03718366473913193,
"step": 810
},
{
"epoch": 2.13,
"grad_norm": 43.75,
"learning_rate": 2.6529041520546072e-06,
"logits/chosen": -1.9656623601913452,
"logits/rejected": -1.976292371749878,
"logps/chosen": -35.201332092285156,
"logps/rejected": -32.56475067138672,
"loss": 0.2191,
"rewards/accuracies": 0.925000011920929,
"rewards/chosen": 0.3460533618927002,
"rewards/margins": 0.4393738806247711,
"rewards/rejected": -0.09332051128149033,
"step": 820
},
{
"epoch": 2.16,
"grad_norm": 58.0,
"learning_rate": 2.5963092704273302e-06,
"logits/chosen": -2.0145046710968018,
"logits/rejected": -2.018724203109741,
"logps/chosen": -34.39170455932617,
"logps/rejected": -29.336902618408203,
"loss": 0.196,
"rewards/accuracies": 0.9125000238418579,
"rewards/chosen": 0.357754647731781,
"rewards/margins": 0.4058148264884949,
"rewards/rejected": -0.04806024581193924,
"step": 830
},
{
"epoch": 2.18,
"grad_norm": 28.5,
"learning_rate": 2.53966490958702e-06,
"logits/chosen": -2.012115240097046,
"logits/rejected": -2.0198771953582764,
"logps/chosen": -34.340667724609375,
"logps/rejected": -29.972625732421875,
"loss": 0.375,
"rewards/accuracies": 0.8999999761581421,
"rewards/chosen": 0.3131526708602905,
"rewards/margins": 0.40060001611709595,
"rewards/rejected": -0.08744733780622482,
"step": 840
},
{
"epoch": 2.21,
"grad_norm": 48.75,
"learning_rate": 2.4830001707654135e-06,
"logits/chosen": -1.942496657371521,
"logits/rejected": -1.932924509048462,
"logps/chosen": -31.318111419677734,
"logps/rejected": -32.194252014160156,
"loss": 0.2437,
"rewards/accuracies": 0.9375,
"rewards/chosen": 0.3325595259666443,
"rewards/margins": 0.4588742256164551,
"rewards/rejected": -0.1263146996498108,
"step": 850
},
{
"epoch": 2.23,
"grad_norm": 57.0,
"learning_rate": 2.4263441656635054e-06,
"logits/chosen": -2.0852370262145996,
"logits/rejected": -2.075453281402588,
"logps/chosen": -25.809616088867188,
"logps/rejected": -30.046661376953125,
"loss": 0.2,
"rewards/accuracies": 0.875,
"rewards/chosen": 0.29011884331703186,
"rewards/margins": 0.4121415615081787,
"rewards/rejected": -0.12202271074056625,
"step": 860
},
{
"epoch": 2.26,
"grad_norm": 62.0,
"learning_rate": 2.3697260014953107e-06,
"logits/chosen": -1.9381517171859741,
"logits/rejected": -1.9393552541732788,
"logps/chosen": -33.79442596435547,
"logps/rejected": -30.08365821838379,
"loss": 0.3179,
"rewards/accuracies": 0.8999999761581421,
"rewards/chosen": 0.37700217962265015,
"rewards/margins": 0.4478107988834381,
"rewards/rejected": -0.07080861181020737,
"step": 870
},
{
"epoch": 2.29,
"grad_norm": 25.25,
"learning_rate": 2.3131747660339396e-06,
"logits/chosen": -1.9556992053985596,
"logits/rejected": -1.9564440250396729,
"logps/chosen": -31.94332504272461,
"logps/rejected": -33.166770935058594,
"loss": 0.2801,
"rewards/accuracies": 0.887499988079071,
"rewards/chosen": 0.3121483623981476,
"rewards/margins": 0.3932458460330963,
"rewards/rejected": -0.08109745383262634,
"step": 880
},
{
"epoch": 2.31,
"grad_norm": 42.25,
"learning_rate": 2.256719512667651e-06,
"logits/chosen": -1.858231782913208,
"logits/rejected": -1.856436014175415,
"logps/chosen": -35.18943405151367,
"logps/rejected": -36.062679290771484,
"loss": 0.2634,
"rewards/accuracies": 0.9125000238418579,
"rewards/chosen": 0.3328378200531006,
"rewards/margins": 0.46265101432800293,
"rewards/rejected": -0.12981316447257996,
"step": 890
},
{
"epoch": 2.34,
"grad_norm": 79.0,
"learning_rate": 2.2003892454735786e-06,
"logits/chosen": -2.0039381980895996,
"logits/rejected": -1.9971100091934204,
"logps/chosen": -31.39545249938965,
"logps/rejected": -32.833343505859375,
"loss": 0.2302,
"rewards/accuracies": 0.9375,
"rewards/chosen": 0.4250277578830719,
"rewards/margins": 0.47772127389907837,
"rewards/rejected": -0.05269356444478035,
"step": 900
},
{
"epoch": 2.34,
"eval_logits/chosen": -2.2496516704559326,
"eval_logits/rejected": -2.2447705268859863,
"eval_logps/chosen": -33.913028717041016,
"eval_logps/rejected": -37.41090393066406,
"eval_loss": 0.5758183598518372,
"eval_rewards/accuracies": 0.5544019937515259,
"eval_rewards/chosen": 0.08506669104099274,
"eval_rewards/margins": 0.011066065169870853,
"eval_rewards/rejected": 0.07400061935186386,
"eval_runtime": 145.1902,
"eval_samples_per_second": 2.362,
"eval_steps_per_second": 0.296,
"step": 900
},
{
"epoch": 2.36,
"grad_norm": 21.875,
"learning_rate": 2.1442129043167877e-06,
"logits/chosen": -2.0508811473846436,
"logits/rejected": -2.0461061000823975,
"logps/chosen": -33.02284240722656,
"logps/rejected": -34.64262390136719,
"loss": 0.2017,
"rewards/accuracies": 0.9375,
"rewards/chosen": 0.3743865489959717,
"rewards/margins": 0.45418882369995117,
"rewards/rejected": -0.0798022598028183,
"step": 910
},
{
"epoch": 2.39,
"grad_norm": 31.875,
"learning_rate": 2.088219349982323e-06,
"logits/chosen": -1.9676059484481812,
"logits/rejected": -1.972545862197876,
"logps/chosen": -34.80748748779297,
"logps/rejected": -32.547122955322266,
"loss": 0.297,
"rewards/accuracies": 0.875,
"rewards/chosen": 0.3787701427936554,
"rewards/margins": 0.41182270646095276,
"rewards/rejected": -0.03305256366729736,
"step": 920
},
{
"epoch": 2.42,
"grad_norm": 45.0,
"learning_rate": 2.0324373493478803e-06,
"logits/chosen": -2.0639491081237793,
"logits/rejected": -2.054579257965088,
"logps/chosen": -31.739765167236328,
"logps/rejected": -34.55535125732422,
"loss": 0.1863,
"rewards/accuracies": 0.9750000238418579,
"rewards/chosen": 0.3816167712211609,
"rewards/margins": 0.4604073166847229,
"rewards/rejected": -0.07879054546356201,
"step": 930
},
{
"epoch": 2.44,
"grad_norm": 63.5,
"learning_rate": 1.976895560604729e-06,
"logits/chosen": -1.9907861948013306,
"logits/rejected": -1.987722635269165,
"logps/chosen": -30.749156951904297,
"logps/rejected": -32.56981658935547,
"loss": 0.2258,
"rewards/accuracies": 0.9125000238418579,
"rewards/chosen": 0.3775084614753723,
"rewards/margins": 0.4506160318851471,
"rewards/rejected": -0.07310755550861359,
"step": 940
},
{
"epoch": 2.47,
"grad_norm": 29.875,
"learning_rate": 1.921622518534466e-06,
"logits/chosen": -1.9317022562026978,
"logits/rejected": -1.9392095804214478,
"logps/chosen": -32.34041976928711,
"logps/rejected": -35.695228576660156,
"loss": 0.2412,
"rewards/accuracies": 0.887499988079071,
"rewards/chosen": 0.3714412748813629,
"rewards/margins": 0.47314995527267456,
"rewards/rejected": -0.10170867294073105,
"step": 950
},
{
"epoch": 2.49,
"grad_norm": 114.5,
"learning_rate": 1.8666466198491794e-06,
"logits/chosen": -1.9403190612792969,
"logits/rejected": -1.9334567785263062,
"logps/chosen": -33.3146858215332,
"logps/rejected": -36.74799346923828,
"loss": 0.3338,
"rewards/accuracies": 0.887499988079071,
"rewards/chosen": 0.3596806824207306,
"rewards/margins": 0.46411022543907166,
"rewards/rejected": -0.1044294610619545,
"step": 960
},
{
"epoch": 2.52,
"grad_norm": 30.375,
"learning_rate": 1.8119961086025376e-06,
"logits/chosen": -1.9422130584716797,
"logits/rejected": -1.942911148071289,
"logps/chosen": -30.72060203552246,
"logps/rejected": -33.139339447021484,
"loss": 0.2958,
"rewards/accuracies": 0.925000011920929,
"rewards/chosen": 0.3610595762729645,
"rewards/margins": 0.4717164933681488,
"rewards/rejected": -0.11065696179866791,
"step": 970
},
{
"epoch": 2.55,
"grad_norm": 49.75,
"learning_rate": 1.7576990616793139e-06,
"logits/chosen": -1.9468282461166382,
"logits/rejected": -1.9580564498901367,
"logps/chosen": -32.50783157348633,
"logps/rejected": -34.23973083496094,
"loss": 0.2541,
"rewards/accuracies": 0.8999999761581421,
"rewards/chosen": 0.3981999158859253,
"rewards/margins": 0.47416257858276367,
"rewards/rejected": -0.07596264779567719,
"step": 980
},
{
"epoch": 2.57,
"grad_norm": 83.5,
"learning_rate": 1.7037833743707892e-06,
"logits/chosen": -2.0208706855773926,
"logits/rejected": -2.018265962600708,
"logps/chosen": -35.62385940551758,
"logps/rejected": -32.450279235839844,
"loss": 0.3659,
"rewards/accuracies": 0.8374999761581421,
"rewards/chosen": 0.34803569316864014,
"rewards/margins": 0.38377222418785095,
"rewards/rejected": -0.03573652729392052,
"step": 990
},
{
"epoch": 2.6,
"grad_norm": 24.375,
"learning_rate": 1.6502767460434588e-06,
"logits/chosen": -1.9889123439788818,
"logits/rejected": -1.9926494359970093,
"logps/chosen": -34.00056457519531,
"logps/rejected": -34.569400787353516,
"loss": 0.2296,
"rewards/accuracies": 0.9125000238418579,
"rewards/chosen": 0.32390862703323364,
"rewards/margins": 0.38225558400154114,
"rewards/rejected": -0.05834692716598511,
"step": 1000
},
{
"epoch": 2.6,
"eval_logits/chosen": -2.248878240585327,
"eval_logits/rejected": -2.2439959049224854,
"eval_logps/chosen": -33.944374084472656,
"eval_logps/rejected": -37.437828063964844,
"eval_loss": 0.5750337243080139,
"eval_rewards/accuracies": 0.5074750781059265,
"eval_rewards/chosen": 0.06312327831983566,
"eval_rewards/margins": 0.007969892583787441,
"eval_rewards/rejected": 0.05515338107943535,
"eval_runtime": 145.3511,
"eval_samples_per_second": 2.36,
"eval_steps_per_second": 0.296,
"step": 1000
},
{
"epoch": 2.62,
"grad_norm": 75.0,
"learning_rate": 1.5972066659083796e-06,
"logits/chosen": -1.9976009130477905,
"logits/rejected": -2.0026943683624268,
"logps/chosen": -32.26424789428711,
"logps/rejected": -32.64023208618164,
"loss": 0.204,
"rewards/accuracies": 0.925000011920929,
"rewards/chosen": 0.3753224313259125,
"rewards/margins": 0.45860418677330017,
"rewards/rejected": -0.08328177034854889,
"step": 1010
},
{
"epoch": 2.65,
"grad_norm": 58.5,
"learning_rate": 1.5446003988985041e-06,
"logits/chosen": -1.9610779285430908,
"logits/rejected": -1.9637727737426758,
"logps/chosen": -29.9860897064209,
"logps/rejected": -31.74285316467285,
"loss": 0.2786,
"rewards/accuracies": 0.875,
"rewards/chosen": 0.32168638706207275,
"rewards/margins": 0.3522700071334839,
"rewards/rejected": -0.03058364987373352,
"step": 1020
},
{
"epoch": 2.68,
"grad_norm": 24.0,
"learning_rate": 1.4924849716612211e-06,
"logits/chosen": -2.0177597999572754,
"logits/rejected": -2.011871814727783,
"logps/chosen": -33.44524383544922,
"logps/rejected": -33.608463287353516,
"loss": 0.1846,
"rewards/accuracies": 0.949999988079071,
"rewards/chosen": 0.38070768117904663,
"rewards/margins": 0.44403451681137085,
"rewards/rejected": -0.06332685798406601,
"step": 1030
},
{
"epoch": 2.7,
"grad_norm": 46.25,
"learning_rate": 1.440887158673332e-06,
"logits/chosen": -1.945654273033142,
"logits/rejected": -1.9493560791015625,
"logps/chosen": -35.2274284362793,
"logps/rejected": -34.58420181274414,
"loss": 0.2056,
"rewards/accuracies": 0.9750000238418579,
"rewards/chosen": 0.36841046810150146,
"rewards/margins": 0.483646959066391,
"rewards/rejected": -0.11523648351430893,
"step": 1040
},
{
"epoch": 2.73,
"grad_norm": 17.375,
"learning_rate": 1.3898334684855647e-06,
"logits/chosen": -1.9452226161956787,
"logits/rejected": -1.958367109298706,
"logps/chosen": -31.376358032226562,
"logps/rejected": -33.11796951293945,
"loss": 0.2082,
"rewards/accuracies": 0.887499988079071,
"rewards/chosen": 0.34057068824768066,
"rewards/margins": 0.40912023186683655,
"rewards/rejected": -0.06854955106973648,
"step": 1050
},
{
"epoch": 2.75,
"grad_norm": 49.5,
"learning_rate": 1.3393501301037245e-06,
"logits/chosen": -1.9365898370742798,
"logits/rejected": -1.9300063848495483,
"logps/chosen": -31.5865478515625,
"logps/rejected": -33.410743713378906,
"loss": 0.1983,
"rewards/accuracies": 0.9375,
"rewards/chosen": 0.4009482264518738,
"rewards/margins": 0.46589574217796326,
"rewards/rejected": -0.06494755297899246,
"step": 1060
},
{
"epoch": 2.78,
"grad_norm": 30.125,
"learning_rate": 1.2894630795134454e-06,
"logits/chosen": -2.062471866607666,
"logits/rejected": -2.063298225402832,
"logps/chosen": -33.060279846191406,
"logps/rejected": -33.027076721191406,
"loss": 0.1608,
"rewards/accuracies": 0.9375,
"rewards/chosen": 0.3407577872276306,
"rewards/margins": 0.46842724084854126,
"rewards/rejected": -0.12766942381858826,
"step": 1070
},
{
"epoch": 2.81,
"grad_norm": 19.125,
"learning_rate": 1.2401979463554984e-06,
"logits/chosen": -1.9935426712036133,
"logits/rejected": -1.9923900365829468,
"logps/chosen": -33.55720901489258,
"logps/rejected": -33.266090393066406,
"loss": 0.2082,
"rewards/accuracies": 0.8999999761581421,
"rewards/chosen": 0.372835636138916,
"rewards/margins": 0.42481860518455505,
"rewards/rejected": -0.051982950419187546,
"step": 1080
},
{
"epoch": 2.83,
"grad_norm": 37.0,
"learning_rate": 1.1915800407584705e-06,
"logits/chosen": -2.021810531616211,
"logits/rejected": -2.01427960395813,
"logps/chosen": -33.71491622924805,
"logps/rejected": -31.135467529296875,
"loss": 0.1958,
"rewards/accuracies": 0.9375,
"rewards/chosen": 0.3576733469963074,
"rewards/margins": 0.42342647910118103,
"rewards/rejected": -0.06575315445661545,
"step": 1090
},
{
"epoch": 2.86,
"grad_norm": 67.5,
"learning_rate": 1.1436343403356019e-06,
"logits/chosen": -1.9883337020874023,
"logits/rejected": -1.9875253438949585,
"logps/chosen": -34.86940002441406,
"logps/rejected": -36.43136978149414,
"loss": 0.2798,
"rewards/accuracies": 0.925000011920929,
"rewards/chosen": 0.3843076229095459,
"rewards/margins": 0.4393698275089264,
"rewards/rejected": -0.055062174797058105,
"step": 1100
},
{
"epoch": 2.86,
"eval_logits/chosen": -2.2467923164367676,
"eval_logits/rejected": -2.2419159412384033,
"eval_logps/chosen": -33.93034744262695,
"eval_logps/rejected": -37.438724517822266,
"eval_loss": 0.5483447909355164,
"eval_rewards/accuracies": 0.5427741408348083,
"eval_rewards/chosen": 0.07294297218322754,
"eval_rewards/margins": 0.018412673845887184,
"eval_rewards/rejected": 0.05453029274940491,
"eval_runtime": 145.1205,
"eval_samples_per_second": 2.364,
"eval_steps_per_second": 0.296,
"step": 1100
},
{
"epoch": 2.88,
"grad_norm": 54.25,
"learning_rate": 1.0963854773524548e-06,
"logits/chosen": -2.0355682373046875,
"logits/rejected": -2.0409066677093506,
"logps/chosen": -34.98146057128906,
"logps/rejected": -35.63863754272461,
"loss": 0.2279,
"rewards/accuracies": 0.862500011920929,
"rewards/chosen": 0.3399600386619568,
"rewards/margins": 0.40649813413619995,
"rewards/rejected": -0.06653807312250137,
"step": 1110
},
{
"epoch": 2.91,
"grad_norm": 71.5,
"learning_rate": 1.049857726072005e-06,
"logits/chosen": -2.0084500312805176,
"logits/rejected": -2.006545066833496,
"logps/chosen": -32.13376235961914,
"logps/rejected": -33.14947509765625,
"loss": 0.2249,
"rewards/accuracies": 0.925000011920929,
"rewards/chosen": 0.33771127462387085,
"rewards/margins": 0.45441800355911255,
"rewards/rejected": -0.11670677363872528,
"step": 1120
},
{
"epoch": 2.94,
"grad_norm": 17.875,
"learning_rate": 1.0040749902836508e-06,
"logits/chosen": -1.9112991094589233,
"logits/rejected": -1.9141557216644287,
"logps/chosen": -29.258800506591797,
"logps/rejected": -30.369312286376953,
"loss": 0.2421,
"rewards/accuracies": 0.887499988079071,
"rewards/chosen": 0.3428861200809479,
"rewards/margins": 0.429875910282135,
"rewards/rejected": -0.08698974549770355,
"step": 1130
},
{
"epoch": 2.96,
"grad_norm": 42.25,
"learning_rate": 9.59060791022566e-07,
"logits/chosen": -2.0088257789611816,
"logits/rejected": -2.005708694458008,
"logps/chosen": -32.668514251708984,
"logps/rejected": -32.6502799987793,
"loss": 0.2007,
"rewards/accuracies": 0.8999999761581421,
"rewards/chosen": 0.37466686964035034,
"rewards/margins": 0.461048424243927,
"rewards/rejected": -0.08638156205415726,
"step": 1140
},
{
"epoch": 2.99,
"grad_norm": 29.25,
"learning_rate": 9.148382544856885e-07,
"logits/chosen": -1.930190086364746,
"logits/rejected": -1.9238803386688232,
"logps/chosen": -27.67626953125,
"logps/rejected": -32.34447479248047,
"loss": 0.1585,
"rewards/accuracies": 0.9624999761581421,
"rewards/chosen": 0.32320261001586914,
"rewards/margins": 0.4605270028114319,
"rewards/rejected": -0.13732439279556274,
"step": 1150
},
{
"epoch": 3.01,
"grad_norm": 35.75,
"learning_rate": 8.714301001505568e-07,
"logits/chosen": -2.04903507232666,
"logits/rejected": -2.0465400218963623,
"logps/chosen": -32.619815826416016,
"logps/rejected": -35.1733283996582,
"loss": 0.1505,
"rewards/accuracies": 0.925000011920929,
"rewards/chosen": 0.3333774507045746,
"rewards/margins": 0.39063215255737305,
"rewards/rejected": -0.05725465342402458,
"step": 1160
},
{
"epoch": 3.04,
"grad_norm": 15.625,
"learning_rate": 8.288586291031025e-07,
"logits/chosen": -2.0296478271484375,
"logits/rejected": -2.0277647972106934,
"logps/chosen": -31.087743759155273,
"logps/rejected": -32.51514434814453,
"loss": 0.0912,
"rewards/accuracies": 0.9624999761581421,
"rewards/chosen": 0.36866381764411926,
"rewards/margins": 0.4744475483894348,
"rewards/rejected": -0.10578374564647675,
"step": 1170
},
{
"epoch": 3.06,
"grad_norm": 35.75,
"learning_rate": 7.871457125803897e-07,
"logits/chosen": -2.0225229263305664,
"logits/rejected": -2.0101773738861084,
"logps/chosen": -35.537513732910156,
"logps/rejected": -34.152286529541016,
"loss": 0.183,
"rewards/accuracies": 0.9125000238418579,
"rewards/chosen": 0.35545071959495544,
"rewards/margins": 0.4120173454284668,
"rewards/rejected": -0.056566618382930756,
"step": 1180
},
{
"epoch": 3.09,
"grad_norm": 30.5,
"learning_rate": 7.463127807341966e-07,
"logits/chosen": -1.9124174118041992,
"logits/rejected": -1.907052755355835,
"logps/chosen": -34.80583953857422,
"logps/rejected": -34.315452575683594,
"loss": 0.1388,
"rewards/accuracies": 0.9375,
"rewards/chosen": 0.3806923031806946,
"rewards/margins": 0.43503332138061523,
"rewards/rejected": -0.05434093996882439,
"step": 1190
},
{
"epoch": 3.12,
"grad_norm": 30.75,
"learning_rate": 7.063808116212021e-07,
"logits/chosen": -1.9541196823120117,
"logits/rejected": -1.9571189880371094,
"logps/chosen": -32.08814239501953,
"logps/rejected": -31.591243743896484,
"loss": 0.1195,
"rewards/accuracies": 0.949999988079071,
"rewards/chosen": 0.3688638210296631,
"rewards/margins": 0.4424096941947937,
"rewards/rejected": -0.07354583591222763,
"step": 1200
},
{
"epoch": 3.12,
"eval_logits/chosen": -2.247305393218994,
"eval_logits/rejected": -2.242413282394409,
"eval_logps/chosen": -33.93861770629883,
"eval_logps/rejected": -37.42906188964844,
"eval_loss": 0.5758681297302246,
"eval_rewards/accuracies": 0.5137043595314026,
"eval_rewards/chosen": 0.06715311110019684,
"eval_rewards/margins": 0.005864266771823168,
"eval_rewards/rejected": 0.06128884479403496,
"eval_runtime": 145.3379,
"eval_samples_per_second": 2.36,
"eval_steps_per_second": 0.296,
"step": 1200
},
{
"epoch": 3.14,
"grad_norm": 32.75,
"learning_rate": 6.673703204254348e-07,
"logits/chosen": -1.986507773399353,
"logits/rejected": -1.9819329977035522,
"logps/chosen": -30.773900985717773,
"logps/rejected": -31.090429306030273,
"loss": 0.117,
"rewards/accuracies": 0.949999988079071,
"rewards/chosen": 0.349838525056839,
"rewards/margins": 0.4774731695652008,
"rewards/rejected": -0.12763459980487823,
"step": 1210
},
{
"epoch": 3.17,
"grad_norm": 33.75,
"learning_rate": 6.293013489185315e-07,
"logits/chosen": -1.9981073141098022,
"logits/rejected": -1.9925674200057983,
"logps/chosen": -34.77349090576172,
"logps/rejected": -34.63701248168945,
"loss": 0.1218,
"rewards/accuracies": 0.9624999761581421,
"rewards/chosen": 0.35031452775001526,
"rewards/margins": 0.3973073959350586,
"rewards/rejected": -0.046992845833301544,
"step": 1220
},
{
"epoch": 3.19,
"grad_norm": 172.0,
"learning_rate": 5.921934551632086e-07,
"logits/chosen": -1.9999059438705444,
"logits/rejected": -1.9867637157440186,
"logps/chosen": -32.57126998901367,
"logps/rejected": -34.21943283081055,
"loss": 0.1846,
"rewards/accuracies": 0.8999999761581421,
"rewards/chosen": 0.3659234941005707,
"rewards/margins": 0.4356323182582855,
"rewards/rejected": -0.06970882415771484,
"step": 1230
},
{
"epoch": 3.22,
"grad_norm": 36.25,
"learning_rate": 5.560657034652405e-07,
"logits/chosen": -2.0442328453063965,
"logits/rejected": -2.041350841522217,
"logps/chosen": -34.520362854003906,
"logps/rejected": -31.955829620361328,
"loss": 0.1368,
"rewards/accuracies": 0.925000011920929,
"rewards/chosen": 0.3587776720523834,
"rewards/margins": 0.4399491250514984,
"rewards/rejected": -0.08117148280143738,
"step": 1240
},
{
"epoch": 3.25,
"grad_norm": 95.5,
"learning_rate": 5.2093665457911e-07,
"logits/chosen": -1.97548508644104,
"logits/rejected": -1.9726585149765015,
"logps/chosen": -33.8781623840332,
"logps/rejected": -34.95212173461914,
"loss": 0.1618,
"rewards/accuracies": 0.925000011920929,
"rewards/chosen": 0.3625948429107666,
"rewards/margins": 0.47109127044677734,
"rewards/rejected": -0.10849642753601074,
"step": 1250
},
{
"epoch": 3.27,
"grad_norm": 26.25,
"learning_rate": 4.868243561723535e-07,
"logits/chosen": -2.038349151611328,
"logits/rejected": -2.033210277557373,
"logps/chosen": -30.266530990600586,
"logps/rejected": -32.47600555419922,
"loss": 0.1155,
"rewards/accuracies": 0.949999988079071,
"rewards/chosen": 0.3459872007369995,
"rewards/margins": 0.4114041328430176,
"rewards/rejected": -0.06541696935892105,
"step": 1260
},
{
"epoch": 3.3,
"grad_norm": 17.75,
"learning_rate": 4.537463335535161e-07,
"logits/chosen": -2.0647287368774414,
"logits/rejected": -2.069408416748047,
"logps/chosen": -32.192989349365234,
"logps/rejected": -31.8017578125,
"loss": 0.126,
"rewards/accuracies": 0.949999988079071,
"rewards/chosen": 0.3626050055027008,
"rewards/margins": 0.446880578994751,
"rewards/rejected": -0.08427558094263077,
"step": 1270
},
{
"epoch": 3.32,
"grad_norm": 42.0,
"learning_rate": 4.217195806684629e-07,
"logits/chosen": -1.9595832824707031,
"logits/rejected": -1.966700792312622,
"logps/chosen": -34.40987777709961,
"logps/rejected": -32.77952575683594,
"loss": 0.1119,
"rewards/accuracies": 0.9624999761581421,
"rewards/chosen": 0.3481543958187103,
"rewards/margins": 0.4750334620475769,
"rewards/rejected": -0.126879021525383,
"step": 1280
},
{
"epoch": 3.35,
"grad_norm": 30.375,
"learning_rate": 3.907605513696808e-07,
"logits/chosen": -1.8622219562530518,
"logits/rejected": -1.8643817901611328,
"logps/chosen": -32.59587860107422,
"logps/rejected": -36.642635345458984,
"loss": 0.1373,
"rewards/accuracies": 0.9624999761581421,
"rewards/chosen": 0.3752126097679138,
"rewards/margins": 0.4370684027671814,
"rewards/rejected": -0.06185578554868698,
"step": 1290
},
{
"epoch": 3.38,
"grad_norm": 89.5,
"learning_rate": 3.6088515096305675e-07,
"logits/chosen": -1.9182116985321045,
"logits/rejected": -1.9216159582138062,
"logps/chosen": -32.71792221069336,
"logps/rejected": -32.585205078125,
"loss": 0.1371,
"rewards/accuracies": 0.9375,
"rewards/chosen": 0.3491230309009552,
"rewards/margins": 0.4338512420654297,
"rewards/rejected": -0.08472825586795807,
"step": 1300
},
{
"epoch": 3.38,
"eval_logits/chosen": -2.2483434677124023,
"eval_logits/rejected": -2.243445634841919,
"eval_logps/chosen": -33.929874420166016,
"eval_logps/rejected": -37.43461990356445,
"eval_loss": 0.5592050552368164,
"eval_rewards/accuracies": 0.5494186282157898,
"eval_rewards/chosen": 0.0732741430401802,
"eval_rewards/margins": 0.01587507873773575,
"eval_rewards/rejected": 0.05739906057715416,
"eval_runtime": 145.2128,
"eval_samples_per_second": 2.362,
"eval_steps_per_second": 0.296,
"step": 1300
},
{
"epoch": 3.4,
"grad_norm": 22.125,
"learning_rate": 3.321087280364757e-07,
"logits/chosen": -2.0270915031433105,
"logits/rejected": -2.006920337677002,
"logps/chosen": -30.607580184936523,
"logps/rejected": -35.26466369628906,
"loss": 0.14,
"rewards/accuracies": 0.925000011920929,
"rewards/chosen": 0.35407310724258423,
"rewards/margins": 0.42285603284835815,
"rewards/rejected": -0.0687829852104187,
"step": 1310
},
{
"epoch": 3.43,
"grad_norm": 22.875,
"learning_rate": 3.044460665744284e-07,
"logits/chosen": -2.006091594696045,
"logits/rejected": -2.0118560791015625,
"logps/chosen": -32.49515914916992,
"logps/rejected": -32.58808517456055,
"loss": 0.1516,
"rewards/accuracies": 0.9375,
"rewards/chosen": 0.37704116106033325,
"rewards/margins": 0.4451761245727539,
"rewards/rejected": -0.06813497841358185,
"step": 1320
},
{
"epoch": 3.45,
"grad_norm": 58.25,
"learning_rate": 2.779113783626916e-07,
"logits/chosen": -1.9745457172393799,
"logits/rejected": -1.969630479812622,
"logps/chosen": -32.551475524902344,
"logps/rejected": -34.6931266784668,
"loss": 0.1414,
"rewards/accuracies": 0.925000011920929,
"rewards/chosen": 0.3297252357006073,
"rewards/margins": 0.4143126606941223,
"rewards/rejected": -0.08458739519119263,
"step": 1330
},
{
"epoch": 3.48,
"grad_norm": 38.5,
"learning_rate": 2.5251829568697204e-07,
"logits/chosen": -1.840356469154358,
"logits/rejected": -1.8500515222549438,
"logps/chosen": -33.240806579589844,
"logps/rejected": -31.120624542236328,
"loss": 0.1088,
"rewards/accuracies": 0.9624999761581421,
"rewards/chosen": 0.391147643327713,
"rewards/margins": 0.4517039358615875,
"rewards/rejected": -0.0605563148856163,
"step": 1340
},
{
"epoch": 3.51,
"grad_norm": 25.75,
"learning_rate": 2.2827986432927774e-07,
"logits/chosen": -1.9017702341079712,
"logits/rejected": -1.906445860862732,
"logps/chosen": -33.377037048339844,
"logps/rejected": -33.292083740234375,
"loss": 0.0844,
"rewards/accuracies": 0.9750000238418579,
"rewards/chosen": 0.4017297625541687,
"rewards/margins": 0.4628194272518158,
"rewards/rejected": -0.06108973175287247,
"step": 1350
},
{
"epoch": 3.53,
"grad_norm": 70.5,
"learning_rate": 2.0520853686560177e-07,
"logits/chosen": -1.9277992248535156,
"logits/rejected": -1.9333375692367554,
"logps/chosen": -33.33270263671875,
"logps/rejected": -34.68195343017578,
"loss": 0.13,
"rewards/accuracies": 0.925000011920929,
"rewards/chosen": 0.3492715060710907,
"rewards/margins": 0.43249478936195374,
"rewards/rejected": -0.08322323858737946,
"step": 1360
},
{
"epoch": 3.56,
"grad_norm": 45.0,
"learning_rate": 1.833161662683672e-07,
"logits/chosen": -2.017637252807617,
"logits/rejected": -2.0131232738494873,
"logps/chosen": -32.15598678588867,
"logps/rejected": -30.987085342407227,
"loss": 0.1109,
"rewards/accuracies": 0.9375,
"rewards/chosen": 0.335787296295166,
"rewards/margins": 0.38841742277145386,
"rewards/rejected": -0.052630096673965454,
"step": 1370
},
{
"epoch": 3.58,
"grad_norm": 23.75,
"learning_rate": 1.626139998169246e-07,
"logits/chosen": -1.9399824142456055,
"logits/rejected": -1.9417117834091187,
"logps/chosen": -29.205978393554688,
"logps/rejected": -30.845836639404297,
"loss": 0.1135,
"rewards/accuracies": 0.925000011920929,
"rewards/chosen": 0.3173268437385559,
"rewards/margins": 0.3884861469268799,
"rewards/rejected": -0.07115931063890457,
"step": 1380
},
{
"epoch": 3.61,
"grad_norm": 36.5,
"learning_rate": 1.4311267331922535e-07,
"logits/chosen": -1.9257915019989014,
"logits/rejected": -1.9303070306777954,
"logps/chosen": -31.455486297607422,
"logps/rejected": -33.11508560180664,
"loss": 0.0911,
"rewards/accuracies": 0.9750000238418579,
"rewards/chosen": 0.39480385184288025,
"rewards/margins": 0.4837164878845215,
"rewards/rejected": -0.08891263604164124,
"step": 1390
},
{
"epoch": 3.64,
"grad_norm": 60.0,
"learning_rate": 1.2482220564763669e-07,
"logits/chosen": -1.99787175655365,
"logits/rejected": -2.0011351108551025,
"logps/chosen": -34.36697006225586,
"logps/rejected": -33.57072448730469,
"loss": 0.0993,
"rewards/accuracies": 0.9750000238418579,
"rewards/chosen": 0.3945205807685852,
"rewards/margins": 0.4975336194038391,
"rewards/rejected": -0.10301297903060913,
"step": 1400
},
{
"epoch": 3.64,
"eval_logits/chosen": -2.247082471847534,
"eval_logits/rejected": -2.242189884185791,
"eval_logps/chosen": -33.95657730102539,
"eval_logps/rejected": -37.431129455566406,
"eval_loss": 0.6129570007324219,
"eval_rewards/accuracies": 0.4871262311935425,
"eval_rewards/chosen": 0.05458100512623787,
"eval_rewards/margins": -0.005263464525341988,
"eval_rewards/rejected": 0.05984446778893471,
"eval_runtime": 145.3553,
"eval_samples_per_second": 2.36,
"eval_steps_per_second": 0.296,
"step": 1400
},
{
"epoch": 3.66,
"grad_norm": 64.5,
"learning_rate": 1.0775199359171346e-07,
"logits/chosen": -2.0669209957122803,
"logits/rejected": -2.0600974559783936,
"logps/chosen": -33.28879928588867,
"logps/rejected": -34.330623626708984,
"loss": 0.14,
"rewards/accuracies": 0.9375,
"rewards/chosen": 0.3596227765083313,
"rewards/margins": 0.4198623299598694,
"rewards/rejected": -0.06023954600095749,
"step": 1410
},
{
"epoch": 3.69,
"grad_norm": 25.625,
"learning_rate": 9.191080703056604e-08,
"logits/chosen": -1.9519445896148682,
"logits/rejected": -1.9626754522323608,
"logps/chosen": -33.65924072265625,
"logps/rejected": -33.730491638183594,
"loss": 0.0849,
"rewards/accuracies": 0.9750000238418579,
"rewards/chosen": 0.34304356575012207,
"rewards/margins": 0.4406338632106781,
"rewards/rejected": -0.09759029000997543,
"step": 1420
},
{
"epoch": 3.71,
"grad_norm": 58.0,
"learning_rate": 7.730678442730539e-08,
"logits/chosen": -2.0385050773620605,
"logits/rejected": -2.0503756999969482,
"logps/chosen": -34.54597473144531,
"logps/rejected": -33.77598571777344,
"loss": 0.1741,
"rewards/accuracies": 0.9125000238418579,
"rewards/chosen": 0.36038076877593994,
"rewards/margins": 0.4567357897758484,
"rewards/rejected": -0.09635499119758606,
"step": 1430
},
{
"epoch": 3.74,
"grad_norm": 18.875,
"learning_rate": 6.394742864787806e-08,
"logits/chosen": -2.0281505584716797,
"logits/rejected": -2.030089855194092,
"logps/chosen": -32.70575714111328,
"logps/rejected": -33.9802360534668,
"loss": 0.1205,
"rewards/accuracies": 0.9624999761581421,
"rewards/chosen": 0.35183992981910706,
"rewards/margins": 0.4360295832157135,
"rewards/rejected": -0.08418963849544525,
"step": 1440
},
{
"epoch": 3.77,
"grad_norm": 20.5,
"learning_rate": 5.183960310644748e-08,
"logits/chosen": -2.0094611644744873,
"logits/rejected": -2.0022339820861816,
"logps/chosen": -35.01443099975586,
"logps/rejected": -33.88014221191406,
"loss": 0.1175,
"rewards/accuracies": 0.9624999761581421,
"rewards/chosen": 0.38772544264793396,
"rewards/margins": 0.464539110660553,
"rewards/rejected": -0.07681362330913544,
"step": 1450
},
{
"epoch": 3.79,
"grad_norm": 25.625,
"learning_rate": 4.098952823928693e-08,
"logits/chosen": -1.9764325618743896,
"logits/rejected": -1.9825315475463867,
"logps/chosen": -30.76479148864746,
"logps/rejected": -33.833831787109375,
"loss": 0.1383,
"rewards/accuracies": 0.9375,
"rewards/chosen": 0.31386876106262207,
"rewards/margins": 0.4285518229007721,
"rewards/rejected": -0.11468305438756943,
"step": 1460
},
{
"epoch": 3.82,
"grad_norm": 21.125,
"learning_rate": 3.1402778309014284e-08,
"logits/chosen": -1.923027753829956,
"logits/rejected": -1.9207102060317993,
"logps/chosen": -30.280017852783203,
"logps/rejected": -30.681386947631836,
"loss": 0.1347,
"rewards/accuracies": 0.925000011920929,
"rewards/chosen": 0.3958771824836731,
"rewards/margins": 0.4568881392478943,
"rewards/rejected": -0.061010997742414474,
"step": 1470
},
{
"epoch": 3.84,
"grad_norm": 18.25,
"learning_rate": 2.3084278540791427e-08,
"logits/chosen": -2.0964748859405518,
"logits/rejected": -2.090756893157959,
"logps/chosen": -34.767372131347656,
"logps/rejected": -32.20579528808594,
"loss": 0.1015,
"rewards/accuracies": 0.987500011920929,
"rewards/chosen": 0.3625258803367615,
"rewards/margins": 0.4668874740600586,
"rewards/rejected": -0.1043616384267807,
"step": 1480
},
{
"epoch": 3.87,
"grad_norm": 21.375,
"learning_rate": 1.6038302591975807e-08,
"logits/chosen": -1.9879157543182373,
"logits/rejected": -1.9899780750274658,
"logps/chosen": -27.98676109313965,
"logps/rejected": -28.119998931884766,
"loss": 0.1002,
"rewards/accuracies": 0.9375,
"rewards/chosen": 0.34273290634155273,
"rewards/margins": 0.41113558411598206,
"rewards/rejected": -0.06840268522500992,
"step": 1490
},
{
"epoch": 3.9,
"grad_norm": 59.75,
"learning_rate": 1.0268470356514237e-08,
"logits/chosen": -1.9974746704101562,
"logits/rejected": -1.9917857646942139,
"logps/chosen": -32.35822677612305,
"logps/rejected": -32.09123611450195,
"loss": 0.18,
"rewards/accuracies": 0.9375,
"rewards/chosen": 0.3097951412200928,
"rewards/margins": 0.3953918516635895,
"rewards/rejected": -0.0855967253446579,
"step": 1500
},
{
"epoch": 3.9,
"eval_logits/chosen": -2.2471930980682373,
"eval_logits/rejected": -2.242299795150757,
"eval_logps/chosen": -33.923362731933594,
"eval_logps/rejected": -37.430606842041016,
"eval_loss": 0.556613028049469,
"eval_rewards/accuracies": 0.5049834251403809,
"eval_rewards/chosen": 0.07783377170562744,
"eval_rewards/margins": 0.01762736216187477,
"eval_rewards/rejected": 0.06020640954375267,
"eval_runtime": 145.2496,
"eval_samples_per_second": 2.361,
"eval_steps_per_second": 0.296,
"step": 1500
},
{
"epoch": 3.92,
"grad_norm": 31.25,
"learning_rate": 5.777746105209147e-09,
"logits/chosen": -1.9237276315689087,
"logits/rejected": -1.9281084537506104,
"logps/chosen": -33.68144989013672,
"logps/rejected": -34.5217399597168,
"loss": 0.1599,
"rewards/accuracies": 0.8999999761581421,
"rewards/chosen": 0.34268397092819214,
"rewards/margins": 0.4206327497959137,
"rewards/rejected": -0.07794882357120514,
"step": 1510
},
{
"epoch": 3.95,
"grad_norm": 44.25,
"learning_rate": 2.5684369628148352e-09,
"logits/chosen": -1.9838800430297852,
"logits/rejected": -1.9821369647979736,
"logps/chosen": -30.222864151000977,
"logps/rejected": -32.720550537109375,
"loss": 0.1456,
"rewards/accuracies": 0.949999988079071,
"rewards/chosen": 0.3579188287258148,
"rewards/margins": 0.44949913024902344,
"rewards/rejected": -0.09158027172088623,
"step": 1520
},
{
"epoch": 3.97,
"grad_norm": 40.75,
"learning_rate": 6.421917227455999e-10,
"logits/chosen": -2.0813286304473877,
"logits/rejected": -2.0737102031707764,
"logps/chosen": -27.602453231811523,
"logps/rejected": -28.810745239257812,
"loss": 0.091,
"rewards/accuracies": 0.987500011920929,
"rewards/chosen": 0.32234111428260803,
"rewards/margins": 0.41973695158958435,
"rewards/rejected": -0.09739579260349274,
"step": 1530
},
{
"epoch": 4.0,
"grad_norm": 19.0,
"learning_rate": 0.0,
"logits/chosen": -1.979434609413147,
"logits/rejected": -1.9692825078964233,
"logps/chosen": -32.49375534057617,
"logps/rejected": -35.5590705871582,
"loss": 0.1005,
"rewards/accuracies": 0.987500011920929,
"rewards/chosen": 0.40070921182632446,
"rewards/margins": 0.5042387247085571,
"rewards/rejected": -0.10352955758571625,
"step": 1540
},
{
"epoch": 4.0,
"step": 1540,
"total_flos": 0.0,
"train_loss": 0.20226844080856868,
"train_runtime": 10768.956,
"train_samples_per_second": 1.144,
"train_steps_per_second": 0.143
}
],
"logging_steps": 10,
"max_steps": 1540,
"num_input_tokens_seen": 0,
"num_train_epochs": 4,
"save_steps": 100,
"total_flos": 0.0,
"train_batch_size": 4,
"trial_name": null,
"trial_params": null
}