phi-2-ipo-renew1 / trainer_state.json
BraylonDash's picture
Model save
70a5420 verified
{
"best_metric": null,
"best_model_checkpoint": null,
"epoch": 1.9994765768123528,
"eval_steps": 100,
"global_step": 3820,
"is_hyper_param_search": false,
"is_local_process_zero": true,
"is_world_process_zero": true,
"log_history": [
{
"epoch": 0.0,
"learning_rate": 1.3089005235602096e-08,
"logits/chosen": 0.896942138671875,
"logits/rejected": 0.9175108075141907,
"logps/chosen": -192.32028198242188,
"logps/rejected": -193.69876098632812,
"loss": 2500.0,
"rewards/accuracies": 0.0,
"rewards/chosen": 0.0,
"rewards/margins": 0.0,
"rewards/rejected": 0.0,
"step": 1
},
{
"epoch": 0.01,
"learning_rate": 1.3089005235602095e-07,
"logits/chosen": 0.903715968132019,
"logits/rejected": 0.9309377670288086,
"logps/chosen": -253.598876953125,
"logps/rejected": -228.25482177734375,
"loss": 2504.6897,
"rewards/accuracies": 0.3819444477558136,
"rewards/chosen": -0.0001807510998332873,
"rewards/margins": -0.0004412428825162351,
"rewards/rejected": 0.00026049179723486304,
"step": 10
},
{
"epoch": 0.01,
"learning_rate": 2.617801047120419e-07,
"logits/chosen": 0.8256899118423462,
"logits/rejected": 0.9293961524963379,
"logps/chosen": -252.84963989257812,
"logps/rejected": -214.4913330078125,
"loss": 2511.0686,
"rewards/accuracies": 0.4625000059604645,
"rewards/chosen": -0.0004701187717728317,
"rewards/margins": -0.0010407656664028764,
"rewards/rejected": 0.0005706468946300447,
"step": 20
},
{
"epoch": 0.02,
"learning_rate": 3.926701570680629e-07,
"logits/chosen": 0.8985889554023743,
"logits/rejected": 0.8785662651062012,
"logps/chosen": -236.40536499023438,
"logps/rejected": -219.20285034179688,
"loss": 2494.8072,
"rewards/accuracies": 0.512499988079071,
"rewards/chosen": 0.0004474873130675405,
"rewards/margins": 0.0005769692361354828,
"rewards/rejected": -0.00012948190851602703,
"step": 30
},
{
"epoch": 0.02,
"learning_rate": 5.235602094240838e-07,
"logits/chosen": 0.819919228553772,
"logits/rejected": 0.9144619703292847,
"logps/chosen": -252.99588012695312,
"logps/rejected": -225.9224853515625,
"loss": 2504.7604,
"rewards/accuracies": 0.4749999940395355,
"rewards/chosen": -0.0003411176148802042,
"rewards/margins": -0.0004235326196067035,
"rewards/rejected": 8.241502655437216e-05,
"step": 40
},
{
"epoch": 0.03,
"learning_rate": 6.544502617801048e-07,
"logits/chosen": 0.7974398136138916,
"logits/rejected": 0.8803712725639343,
"logps/chosen": -254.3247528076172,
"logps/rejected": -243.318603515625,
"loss": 2498.3947,
"rewards/accuracies": 0.45625001192092896,
"rewards/chosen": -0.00023676609271205962,
"rewards/margins": 0.0002204025659011677,
"rewards/rejected": -0.00045716846943832934,
"step": 50
},
{
"epoch": 0.03,
"learning_rate": 7.853403141361258e-07,
"logits/chosen": 0.8626053929328918,
"logits/rejected": 0.8485649824142456,
"logps/chosen": -262.6585693359375,
"logps/rejected": -248.63272094726562,
"loss": 2500.4902,
"rewards/accuracies": 0.4937500059604645,
"rewards/chosen": -0.0002292692952323705,
"rewards/margins": 2.0124425645917654e-05,
"rewards/rejected": -0.0002493937499821186,
"step": 60
},
{
"epoch": 0.04,
"learning_rate": 9.162303664921466e-07,
"logits/chosen": 0.8897444605827332,
"logits/rejected": 0.8922082185745239,
"logps/chosen": -232.531005859375,
"logps/rejected": -234.0869903564453,
"loss": 2496.8041,
"rewards/accuracies": 0.53125,
"rewards/chosen": 3.7896970752626657e-06,
"rewards/margins": 0.00037148987757973373,
"rewards/rejected": -0.00036770018050447106,
"step": 70
},
{
"epoch": 0.04,
"learning_rate": 1.0471204188481676e-06,
"logits/chosen": 0.8788009881973267,
"logits/rejected": 0.8891068696975708,
"logps/chosen": -242.5009765625,
"logps/rejected": -229.5125732421875,
"loss": 2508.6898,
"rewards/accuracies": 0.4437499940395355,
"rewards/chosen": -0.0007867829990573227,
"rewards/margins": -0.0008174808463081717,
"rewards/rejected": 3.069788363063708e-05,
"step": 80
},
{
"epoch": 0.05,
"learning_rate": 1.1780104712041885e-06,
"logits/chosen": 0.8606799840927124,
"logits/rejected": 0.9575719833374023,
"logps/chosen": -232.0597686767578,
"logps/rejected": -218.4732666015625,
"loss": 2496.8559,
"rewards/accuracies": 0.574999988079071,
"rewards/chosen": 0.0003882866003550589,
"rewards/margins": 0.000359431782271713,
"rewards/rejected": 2.8854870834038593e-05,
"step": 90
},
{
"epoch": 0.05,
"learning_rate": 1.3089005235602096e-06,
"logits/chosen": 0.8833224177360535,
"logits/rejected": 0.8661258816719055,
"logps/chosen": -245.799072265625,
"logps/rejected": -249.2645721435547,
"loss": 2496.843,
"rewards/accuracies": 0.5,
"rewards/chosen": -0.0005279771285131574,
"rewards/margins": 0.00037464461638592184,
"rewards/rejected": -0.0009026216575875878,
"step": 100
},
{
"epoch": 0.05,
"eval_logits/chosen": 0.8318074345588684,
"eval_logits/rejected": 0.8888298273086548,
"eval_logps/chosen": -256.65057373046875,
"eval_logps/rejected": -233.56494140625,
"eval_loss": 2502.266845703125,
"eval_rewards/accuracies": 0.5005000233650208,
"eval_rewards/chosen": -0.0003313073539175093,
"eval_rewards/margins": -0.00017098072567023337,
"eval_rewards/rejected": -0.00016032661369536072,
"eval_runtime": 416.835,
"eval_samples_per_second": 4.798,
"eval_steps_per_second": 1.2,
"step": 100
},
{
"epoch": 0.06,
"learning_rate": 1.4397905759162306e-06,
"logits/chosen": 0.9012953042984009,
"logits/rejected": 0.8766192197799683,
"logps/chosen": -229.46292114257812,
"logps/rejected": -210.2642364501953,
"loss": 2501.3449,
"rewards/accuracies": 0.5,
"rewards/chosen": -0.00010574392217677087,
"rewards/margins": -7.803810149198398e-05,
"rewards/rejected": -2.7705809770850465e-05,
"step": 110
},
{
"epoch": 0.06,
"learning_rate": 1.5706806282722515e-06,
"logits/chosen": 0.8108441233634949,
"logits/rejected": 0.8906086087226868,
"logps/chosen": -273.14385986328125,
"logps/rejected": -259.1924133300781,
"loss": 2494.2678,
"rewards/accuracies": 0.5562499761581421,
"rewards/chosen": 0.00021679741621483117,
"rewards/margins": 0.0006280258530750871,
"rewards/rejected": -0.00041122836410067976,
"step": 120
},
{
"epoch": 0.07,
"learning_rate": 1.7015706806282726e-06,
"logits/chosen": 0.8712674975395203,
"logits/rejected": 0.9049458503723145,
"logps/chosen": -277.8616943359375,
"logps/rejected": -222.53662109375,
"loss": 2489.5006,
"rewards/accuracies": 0.5687500238418579,
"rewards/chosen": 0.0006732499459758401,
"rewards/margins": 0.001107201213017106,
"rewards/rejected": -0.0004339513252489269,
"step": 130
},
{
"epoch": 0.07,
"learning_rate": 1.8324607329842933e-06,
"logits/chosen": 0.7955681085586548,
"logits/rejected": 0.8811987638473511,
"logps/chosen": -248.83865356445312,
"logps/rejected": -246.317138671875,
"loss": 2504.0979,
"rewards/accuracies": 0.46875,
"rewards/chosen": -0.00031032637343741953,
"rewards/margins": -0.00034084441722370684,
"rewards/rejected": 3.0518032872350886e-05,
"step": 140
},
{
"epoch": 0.08,
"learning_rate": 1.9633507853403143e-06,
"logits/chosen": 0.7933157086372375,
"logits/rejected": 0.8591764569282532,
"logps/chosen": -257.7363586425781,
"logps/rejected": -217.54580688476562,
"loss": 2507.8082,
"rewards/accuracies": 0.46875,
"rewards/chosen": -0.0007291415822692215,
"rewards/margins": -0.0007318807765841484,
"rewards/rejected": 2.739173851296073e-06,
"step": 150
},
{
"epoch": 0.08,
"learning_rate": 2.094240837696335e-06,
"logits/chosen": 0.8041954040527344,
"logits/rejected": 0.8887465596199036,
"logps/chosen": -276.43304443359375,
"logps/rejected": -250.4193572998047,
"loss": 2504.2807,
"rewards/accuracies": 0.5,
"rewards/chosen": -0.00010556764755165204,
"rewards/margins": -0.0003692187019623816,
"rewards/rejected": 0.00026365104713477194,
"step": 160
},
{
"epoch": 0.09,
"learning_rate": 2.2251308900523565e-06,
"logits/chosen": 0.8059272766113281,
"logits/rejected": 0.8950363993644714,
"logps/chosen": -274.240234375,
"logps/rejected": -247.8701171875,
"loss": 2501.8248,
"rewards/accuracies": 0.512499988079071,
"rewards/chosen": -0.001084670191630721,
"rewards/margins": -0.0001141707762144506,
"rewards/rejected": -0.0009704994154162705,
"step": 170
},
{
"epoch": 0.09,
"learning_rate": 2.356020942408377e-06,
"logits/chosen": 0.8783141374588013,
"logits/rejected": 0.8253491520881653,
"logps/chosen": -242.3585968017578,
"logps/rejected": -221.0929718017578,
"loss": 2496.3063,
"rewards/accuracies": 0.518750011920929,
"rewards/chosen": -0.0004814372514374554,
"rewards/margins": 0.0004220888367854059,
"rewards/rejected": -0.0009035261464305222,
"step": 180
},
{
"epoch": 0.1,
"learning_rate": 2.4869109947643982e-06,
"logits/chosen": 0.8767743110656738,
"logits/rejected": 0.8822822570800781,
"logps/chosen": -246.2511444091797,
"logps/rejected": -224.4364471435547,
"loss": 2498.3156,
"rewards/accuracies": 0.518750011920929,
"rewards/chosen": 0.00032182232826016843,
"rewards/margins": 0.00022237170196603984,
"rewards/rejected": 9.945056081051007e-05,
"step": 190
},
{
"epoch": 0.1,
"learning_rate": 2.617801047120419e-06,
"logits/chosen": 0.855573296546936,
"logits/rejected": 0.9106731414794922,
"logps/chosen": -258.25885009765625,
"logps/rejected": -236.6140594482422,
"loss": 2499.2807,
"rewards/accuracies": 0.48750001192092896,
"rewards/chosen": -0.00018535128037910908,
"rewards/margins": 0.0001189738031825982,
"rewards/rejected": -0.0003043250762857497,
"step": 200
},
{
"epoch": 0.1,
"eval_logits/chosen": 0.8310006856918335,
"eval_logits/rejected": 0.8882209062576294,
"eval_logps/chosen": -256.6106262207031,
"eval_logps/rejected": -233.5994873046875,
"eval_loss": 2494.83544921875,
"eval_rewards/accuracies": 0.5189999938011169,
"eval_rewards/chosen": 6.786447193007916e-05,
"eval_rewards/margins": 0.0005738017498515546,
"eval_rewards/rejected": -0.0005059372633695602,
"eval_runtime": 416.4863,
"eval_samples_per_second": 4.802,
"eval_steps_per_second": 1.201,
"step": 200
},
{
"epoch": 0.11,
"learning_rate": 2.74869109947644e-06,
"logits/chosen": 0.9517833590507507,
"logits/rejected": 0.910740852355957,
"logps/chosen": -244.80032348632812,
"logps/rejected": -232.45321655273438,
"loss": 2500.5746,
"rewards/accuracies": 0.4749999940395355,
"rewards/chosen": -9.258640056941658e-05,
"rewards/margins": -6.444106020353502e-06,
"rewards/rejected": -8.61423322930932e-05,
"step": 210
},
{
"epoch": 0.12,
"learning_rate": 2.8795811518324613e-06,
"logits/chosen": 0.8369059562683105,
"logits/rejected": 0.8937622904777527,
"logps/chosen": -267.40948486328125,
"logps/rejected": -209.13290405273438,
"loss": 2487.1783,
"rewards/accuracies": 0.5874999761581421,
"rewards/chosen": 0.0004562476242426783,
"rewards/margins": 0.0013524172827601433,
"rewards/rejected": -0.0008961696876212955,
"step": 220
},
{
"epoch": 0.12,
"learning_rate": 3.010471204188482e-06,
"logits/chosen": 0.8930699229240417,
"logits/rejected": 0.9343907237052917,
"logps/chosen": -258.6376037597656,
"logps/rejected": -221.6857452392578,
"loss": 2493.877,
"rewards/accuracies": 0.5062500238418579,
"rewards/chosen": 0.0002062669227598235,
"rewards/margins": 0.0006668218411505222,
"rewards/rejected": -0.00046055493294261396,
"step": 230
},
{
"epoch": 0.13,
"learning_rate": 3.141361256544503e-06,
"logits/chosen": 0.9376400113105774,
"logits/rejected": 0.8995400667190552,
"logps/chosen": -228.9315185546875,
"logps/rejected": -242.112548828125,
"loss": 2480.1844,
"rewards/accuracies": 0.59375,
"rewards/chosen": 0.0004402367048896849,
"rewards/margins": 0.002044759690761566,
"rewards/rejected": -0.001604523160494864,
"step": 240
},
{
"epoch": 0.13,
"learning_rate": 3.2722513089005235e-06,
"logits/chosen": 0.9079924821853638,
"logits/rejected": 0.8767238855361938,
"logps/chosen": -242.71121215820312,
"logps/rejected": -233.9228973388672,
"loss": 2493.2939,
"rewards/accuracies": 0.543749988079071,
"rewards/chosen": -6.936644058441743e-05,
"rewards/margins": 0.0007372990949079394,
"rewards/rejected": -0.0008066653972491622,
"step": 250
},
{
"epoch": 0.14,
"learning_rate": 3.403141361256545e-06,
"logits/chosen": 0.8814166784286499,
"logits/rejected": 0.9410937428474426,
"logps/chosen": -236.88095092773438,
"logps/rejected": -223.930419921875,
"loss": 2496.3828,
"rewards/accuracies": 0.550000011920929,
"rewards/chosen": -0.00017086375737562776,
"rewards/margins": 0.0004164519195910543,
"rewards/rejected": -0.0005873156478628516,
"step": 260
},
{
"epoch": 0.14,
"learning_rate": 3.534031413612566e-06,
"logits/chosen": 0.8372557759284973,
"logits/rejected": 0.8741558194160461,
"logps/chosen": -212.303466796875,
"logps/rejected": -239.15158081054688,
"loss": 2483.2088,
"rewards/accuracies": 0.5562499761581421,
"rewards/chosen": -0.00031574201420880854,
"rewards/margins": 0.0017516377847641706,
"rewards/rejected": -0.0020673798862844706,
"step": 270
},
{
"epoch": 0.15,
"learning_rate": 3.6649214659685865e-06,
"logits/chosen": 0.8835927844047546,
"logits/rejected": 0.9312320947647095,
"logps/chosen": -250.3417510986328,
"logps/rejected": -263.51531982421875,
"loss": 2492.1223,
"rewards/accuracies": 0.5249999761581421,
"rewards/chosen": -0.0005325328675098717,
"rewards/margins": 0.000863347842823714,
"rewards/rejected": -0.0013958807103335857,
"step": 280
},
{
"epoch": 0.15,
"learning_rate": 3.7958115183246074e-06,
"logits/chosen": 0.8323150873184204,
"logits/rejected": 0.8896921277046204,
"logps/chosen": -250.1656951904297,
"logps/rejected": -234.757568359375,
"loss": 2483.249,
"rewards/accuracies": 0.581250011920929,
"rewards/chosen": -0.00013801059685647488,
"rewards/margins": 0.0017679758602753282,
"rewards/rejected": -0.001905986457131803,
"step": 290
},
{
"epoch": 0.16,
"learning_rate": 3.926701570680629e-06,
"logits/chosen": 0.8511127233505249,
"logits/rejected": 0.8209661245346069,
"logps/chosen": -273.7172546386719,
"logps/rejected": -250.82748413085938,
"loss": 2477.7609,
"rewards/accuracies": 0.5625,
"rewards/chosen": -0.0011967200553044677,
"rewards/margins": 0.0023162723518908024,
"rewards/rejected": -0.0035129922907799482,
"step": 300
},
{
"epoch": 0.16,
"eval_logits/chosen": 0.8318725824356079,
"eval_logits/rejected": 0.8892252445220947,
"eval_logps/chosen": -256.7284851074219,
"eval_logps/rejected": -233.8547821044922,
"eval_loss": 2481.50146484375,
"eval_rewards/accuracies": 0.559499979019165,
"eval_rewards/chosen": -0.001110685057938099,
"eval_rewards/margins": 0.0019479345064610243,
"eval_rewards/rejected": -0.003058619564399123,
"eval_runtime": 416.6935,
"eval_samples_per_second": 4.8,
"eval_steps_per_second": 1.2,
"step": 300
},
{
"epoch": 0.16,
"learning_rate": 4.05759162303665e-06,
"logits/chosen": 0.7432538866996765,
"logits/rejected": 0.817090630531311,
"logps/chosen": -274.85931396484375,
"logps/rejected": -236.4228057861328,
"loss": 2469.5742,
"rewards/accuracies": 0.6312500238418579,
"rewards/chosen": -0.001123375492170453,
"rewards/margins": 0.003155052661895752,
"rewards/rejected": -0.004278427921235561,
"step": 310
},
{
"epoch": 0.17,
"learning_rate": 4.18848167539267e-06,
"logits/chosen": 0.9147623777389526,
"logits/rejected": 0.9334859848022461,
"logps/chosen": -233.425537109375,
"logps/rejected": -214.6092071533203,
"loss": 2457.892,
"rewards/accuracies": 0.6812499761581421,
"rewards/chosen": 0.00017569802002981305,
"rewards/margins": 0.004348465241491795,
"rewards/rejected": -0.004172767512500286,
"step": 320
},
{
"epoch": 0.17,
"learning_rate": 4.319371727748692e-06,
"logits/chosen": 0.8749006390571594,
"logits/rejected": 0.9252738952636719,
"logps/chosen": -247.8308563232422,
"logps/rejected": -218.9490203857422,
"loss": 2463.7824,
"rewards/accuracies": 0.581250011920929,
"rewards/chosen": -0.0017152890795841813,
"rewards/margins": 0.0037990615237504244,
"rewards/rejected": -0.0055143507197499275,
"step": 330
},
{
"epoch": 0.18,
"learning_rate": 4.450261780104713e-06,
"logits/chosen": 0.8781224489212036,
"logits/rejected": 0.9259663820266724,
"logps/chosen": -253.4806365966797,
"logps/rejected": -239.67434692382812,
"loss": 2474.3055,
"rewards/accuracies": 0.5687500238418579,
"rewards/chosen": -0.0034939595498144627,
"rewards/margins": 0.0027302266098558903,
"rewards/rejected": -0.00622418662533164,
"step": 340
},
{
"epoch": 0.18,
"learning_rate": 4.5811518324607335e-06,
"logits/chosen": 0.7855554223060608,
"logits/rejected": 0.9314893484115601,
"logps/chosen": -255.0915985107422,
"logps/rejected": -206.8708038330078,
"loss": 2432.458,
"rewards/accuracies": 0.6625000238418579,
"rewards/chosen": -0.0009129707468673587,
"rewards/margins": 0.007046517916023731,
"rewards/rejected": -0.007959488779306412,
"step": 350
},
{
"epoch": 0.19,
"learning_rate": 4.712041884816754e-06,
"logits/chosen": 0.8957219123840332,
"logits/rejected": 0.8874330520629883,
"logps/chosen": -257.02764892578125,
"logps/rejected": -230.2834014892578,
"loss": 2441.5672,
"rewards/accuracies": 0.5687500238418579,
"rewards/chosen": -0.001695218845270574,
"rewards/margins": 0.006147631909698248,
"rewards/rejected": -0.007842850871384144,
"step": 360
},
{
"epoch": 0.19,
"learning_rate": 4.842931937172775e-06,
"logits/chosen": 0.9125442504882812,
"logits/rejected": 0.8982815742492676,
"logps/chosen": -230.16940307617188,
"logps/rejected": -211.3495635986328,
"loss": 2439.5756,
"rewards/accuracies": 0.637499988079071,
"rewards/chosen": -0.004968739114701748,
"rewards/margins": 0.006435071583837271,
"rewards/rejected": -0.011403810232877731,
"step": 370
},
{
"epoch": 0.2,
"learning_rate": 4.9738219895287965e-06,
"logits/chosen": 0.9134615063667297,
"logits/rejected": 0.8667083978652954,
"logps/chosen": -267.7635192871094,
"logps/rejected": -219.404541015625,
"loss": 2438.1262,
"rewards/accuracies": 0.637499988079071,
"rewards/chosen": -0.004255574196577072,
"rewards/margins": 0.006601777859032154,
"rewards/rejected": -0.0108573529869318,
"step": 380
},
{
"epoch": 0.2,
"learning_rate": 4.999933200062888e-06,
"logits/chosen": 0.8681972622871399,
"logits/rejected": 0.8684479594230652,
"logps/chosen": -253.1089324951172,
"logps/rejected": -232.1811981201172,
"loss": 2414.8473,
"rewards/accuracies": 0.643750011920929,
"rewards/chosen": -0.0038361139595508575,
"rewards/margins": 0.009047028608620167,
"rewards/rejected": -0.012883143499493599,
"step": 390
},
{
"epoch": 0.21,
"learning_rate": 4.999661831436499e-06,
"logits/chosen": 0.9156022071838379,
"logits/rejected": 0.9197471737861633,
"logps/chosen": -260.40093994140625,
"logps/rejected": -238.50961303710938,
"loss": 2428.4195,
"rewards/accuracies": 0.6187499761581421,
"rewards/chosen": -0.005848343018442392,
"rewards/margins": 0.007789201103150845,
"rewards/rejected": -0.013637542724609375,
"step": 400
},
{
"epoch": 0.21,
"eval_logits/chosen": 0.8404272794723511,
"eval_logits/rejected": 0.8983384966850281,
"eval_logps/chosen": -257.29510498046875,
"eval_logps/rejected": -235.11265563964844,
"eval_loss": 2419.1044921875,
"eval_rewards/accuracies": 0.6495000123977661,
"eval_rewards/chosen": -0.006776793394237757,
"eval_rewards/margins": 0.008860657922923565,
"eval_rewards/rejected": -0.01563744992017746,
"eval_runtime": 416.4578,
"eval_samples_per_second": 4.802,
"eval_steps_per_second": 1.201,
"step": 400
},
{
"epoch": 0.21,
"learning_rate": 4.999181741766532e-06,
"logits/chosen": 0.8992105722427368,
"logits/rejected": 0.8969219923019409,
"logps/chosen": -252.7702178955078,
"logps/rejected": -249.8018798828125,
"loss": 2438.1242,
"rewards/accuracies": 0.643750011920929,
"rewards/chosen": -0.007479649968445301,
"rewards/margins": 0.006951476447284222,
"rewards/rejected": -0.014431129209697247,
"step": 410
},
{
"epoch": 0.22,
"learning_rate": 4.9984929711403395e-06,
"logits/chosen": 0.913814902305603,
"logits/rejected": 0.9069592356681824,
"logps/chosen": -255.3079833984375,
"logps/rejected": -251.0774383544922,
"loss": 2417.1992,
"rewards/accuracies": 0.6000000238418579,
"rewards/chosen": -0.011085378006100655,
"rewards/margins": 0.009236546233296394,
"rewards/rejected": -0.020321926102042198,
"step": 420
},
{
"epoch": 0.23,
"learning_rate": 4.997595577070068e-06,
"logits/chosen": 0.8943805694580078,
"logits/rejected": 0.8994030952453613,
"logps/chosen": -235.75009155273438,
"logps/rejected": -232.7864990234375,
"loss": 2429.1924,
"rewards/accuracies": 0.6312500238418579,
"rewards/chosen": -0.012942090630531311,
"rewards/margins": 0.007976246066391468,
"rewards/rejected": -0.020918335765600204,
"step": 430
},
{
"epoch": 0.23,
"learning_rate": 4.996489634487865e-06,
"logits/chosen": 0.8081871867179871,
"logits/rejected": 0.8945296406745911,
"logps/chosen": -231.6891632080078,
"logps/rejected": -253.6363067626953,
"loss": 2484.0605,
"rewards/accuracies": 0.574999988079071,
"rewards/chosen": -0.012701654806733131,
"rewards/margins": 0.003084682859480381,
"rewards/rejected": -0.015786338597536087,
"step": 440
},
{
"epoch": 0.24,
"learning_rate": 4.995175235739619e-06,
"logits/chosen": 0.8565770983695984,
"logits/rejected": 0.8623224496841431,
"logps/chosen": -264.3777770996094,
"logps/rejected": -267.23712158203125,
"loss": 2392.5188,
"rewards/accuracies": 0.6812499761581421,
"rewards/chosen": -0.011126170866191387,
"rewards/margins": 0.011980591341853142,
"rewards/rejected": -0.023106763139367104,
"step": 450
},
{
"epoch": 0.24,
"learning_rate": 4.9936524905772466e-06,
"logits/chosen": 0.78350430727005,
"logits/rejected": 0.8708200454711914,
"logps/chosen": -267.3443298339844,
"logps/rejected": -236.05459594726562,
"loss": 2404.3129,
"rewards/accuracies": 0.612500011920929,
"rewards/chosen": -0.015727100893855095,
"rewards/margins": 0.010795451700687408,
"rewards/rejected": -0.026522550731897354,
"step": 460
},
{
"epoch": 0.25,
"learning_rate": 4.991921526149529e-06,
"logits/chosen": 0.9162321090698242,
"logits/rejected": 0.9280640482902527,
"logps/chosen": -256.3532409667969,
"logps/rejected": -247.89315795898438,
"loss": 2386.3984,
"rewards/accuracies": 0.668749988079071,
"rewards/chosen": -0.01587381586432457,
"rewards/margins": 0.01281227171421051,
"rewards/rejected": -0.02868608757853508,
"step": 470
},
{
"epoch": 0.25,
"learning_rate": 4.9899824869915e-06,
"logits/chosen": 0.8048622012138367,
"logits/rejected": 0.8283928036689758,
"logps/chosen": -246.1285858154297,
"logps/rejected": -252.7605743408203,
"loss": 2396.8398,
"rewards/accuracies": 0.6499999761581421,
"rewards/chosen": -0.021037602797150612,
"rewards/margins": 0.012100132182240486,
"rewards/rejected": -0.0331377312541008,
"step": 480
},
{
"epoch": 0.26,
"learning_rate": 4.987835535012371e-06,
"logits/chosen": 0.8453197479248047,
"logits/rejected": 0.86089026927948,
"logps/chosen": -240.1754913330078,
"logps/rejected": -229.0949249267578,
"loss": 2350.6213,
"rewards/accuracies": 0.625,
"rewards/chosen": -0.019980577751994133,
"rewards/margins": 0.01681477203965187,
"rewards/rejected": -0.03679535537958145,
"step": 490
},
{
"epoch": 0.26,
"learning_rate": 4.985480849482012e-06,
"logits/chosen": 0.8507258296012878,
"logits/rejected": 0.8831195831298828,
"logps/chosen": -264.3097839355469,
"logps/rejected": -267.3841552734375,
"loss": 2296.8842,
"rewards/accuracies": 0.6875,
"rewards/chosen": -0.019409244880080223,
"rewards/margins": 0.023494381457567215,
"rewards/rejected": -0.04290362820029259,
"step": 500
},
{
"epoch": 0.26,
"eval_logits/chosen": 0.8214389681816101,
"eval_logits/rejected": 0.8805551528930664,
"eval_logps/chosen": -259.0124206542969,
"eval_logps/rejected": -237.73793029785156,
"eval_loss": 2349.435791015625,
"eval_rewards/accuracies": 0.656499981880188,
"eval_rewards/chosen": -0.023950034752488136,
"eval_rewards/margins": 0.017940117046236992,
"eval_rewards/rejected": -0.04189015179872513,
"eval_runtime": 416.5178,
"eval_samples_per_second": 4.802,
"eval_steps_per_second": 1.2,
"step": 500
},
{
"epoch": 0.27,
"learning_rate": 4.98291862701599e-06,
"logits/chosen": 0.8344039916992188,
"logits/rejected": 0.8795874714851379,
"logps/chosen": -254.190673828125,
"logps/rejected": -214.09396362304688,
"loss": 2307.1967,
"rewards/accuracies": 0.675000011920929,
"rewards/chosen": -0.02996075712144375,
"rewards/margins": 0.022381700575351715,
"rewards/rejected": -0.052342455834150314,
"step": 510
},
{
"epoch": 0.27,
"learning_rate": 4.980149081559142e-06,
"logits/chosen": 0.8595240712165833,
"logits/rejected": 0.903663158416748,
"logps/chosen": -237.7533416748047,
"logps/rejected": -241.7561492919922,
"loss": 2330.1758,
"rewards/accuracies": 0.6499999761581421,
"rewards/chosen": -0.02604847028851509,
"rewards/margins": 0.020816484466195107,
"rewards/rejected": -0.04686495289206505,
"step": 520
},
{
"epoch": 0.28,
"learning_rate": 4.977172444367718e-06,
"logits/chosen": 0.8232777714729309,
"logits/rejected": 0.8955798149108887,
"logps/chosen": -248.8101806640625,
"logps/rejected": -226.21786499023438,
"loss": 2284.3357,
"rewards/accuracies": 0.675000011920929,
"rewards/chosen": -0.0257116612046957,
"rewards/margins": 0.026299094781279564,
"rewards/rejected": -0.052010755985975266,
"step": 530
},
{
"epoch": 0.28,
"learning_rate": 4.9739889639900655e-06,
"logits/chosen": 0.8850505948066711,
"logits/rejected": 0.9008530378341675,
"logps/chosen": -260.18963623046875,
"logps/rejected": -228.6940155029297,
"loss": 2314.4127,
"rewards/accuracies": 0.6312500238418579,
"rewards/chosen": -0.028372693806886673,
"rewards/margins": 0.02354586310684681,
"rewards/rejected": -0.051918547600507736,
"step": 540
},
{
"epoch": 0.29,
"learning_rate": 4.9705989062458805e-06,
"logits/chosen": 0.8566417694091797,
"logits/rejected": 0.8558026552200317,
"logps/chosen": -242.9883270263672,
"logps/rejected": -247.76370239257812,
"loss": 2339.4164,
"rewards/accuracies": 0.606249988079071,
"rewards/chosen": -0.03327028453350067,
"rewards/margins": 0.02076330967247486,
"rewards/rejected": -0.054033588618040085,
"step": 550
},
{
"epoch": 0.29,
"learning_rate": 4.967002554204009e-06,
"logits/chosen": 0.7901058793067932,
"logits/rejected": 0.8357075452804565,
"logps/chosen": -263.3518371582031,
"logps/rejected": -250.97958374023438,
"loss": 2302.1379,
"rewards/accuracies": 0.6812499761581421,
"rewards/chosen": -0.04014524444937706,
"rewards/margins": 0.025143718346953392,
"rewards/rejected": -0.0652889683842659,
"step": 560
},
{
"epoch": 0.3,
"learning_rate": 4.963200208158811e-06,
"logits/chosen": 0.8461757898330688,
"logits/rejected": 0.9372328519821167,
"logps/chosen": -223.61373901367188,
"logps/rejected": -212.1987762451172,
"loss": 2320.3783,
"rewards/accuracies": 0.65625,
"rewards/chosen": -0.04337463900446892,
"rewards/margins": 0.021725038066506386,
"rewards/rejected": -0.06509967893362045,
"step": 570
},
{
"epoch": 0.3,
"learning_rate": 4.959192185605089e-06,
"logits/chosen": 0.7897135615348816,
"logits/rejected": 0.9126697778701782,
"logps/chosen": -294.5254821777344,
"logps/rejected": -232.0474395751953,
"loss": 2167.2896,
"rewards/accuracies": 0.737500011920929,
"rewards/chosen": -0.03270890563726425,
"rewards/margins": 0.041936445981264114,
"rewards/rejected": -0.07464535534381866,
"step": 580
},
{
"epoch": 0.31,
"learning_rate": 4.95497882121157e-06,
"logits/chosen": 0.8067277669906616,
"logits/rejected": 0.8418477773666382,
"logps/chosen": -240.95639038085938,
"logps/rejected": -217.8979949951172,
"loss": 2249.4051,
"rewards/accuracies": 0.7250000238418579,
"rewards/chosen": -0.04393559694290161,
"rewards/margins": 0.03141217678785324,
"rewards/rejected": -0.07534776628017426,
"step": 590
},
{
"epoch": 0.31,
"learning_rate": 4.950560466792969e-06,
"logits/chosen": 0.8411356806755066,
"logits/rejected": 0.8524805307388306,
"logps/chosen": -241.7827606201172,
"logps/rejected": -234.87985229492188,
"loss": 2254.5846,
"rewards/accuracies": 0.6499999761581421,
"rewards/chosen": -0.04195866733789444,
"rewards/margins": 0.03153757005929947,
"rewards/rejected": -0.07349623739719391,
"step": 600
},
{
"epoch": 0.31,
"eval_logits/chosen": 0.7868022322654724,
"eval_logits/rejected": 0.8478493094444275,
"eval_logps/chosen": -261.86590576171875,
"eval_logps/rejected": -241.83828735351562,
"eval_loss": 2273.499267578125,
"eval_rewards/accuracies": 0.6570000052452087,
"eval_rewards/chosen": -0.05248467996716499,
"eval_rewards/margins": 0.03040897473692894,
"eval_rewards/rejected": -0.08289366215467453,
"eval_runtime": 416.6239,
"eval_samples_per_second": 4.8,
"eval_steps_per_second": 1.2,
"step": 600
},
{
"epoch": 0.32,
"learning_rate": 4.945937491280611e-06,
"logits/chosen": 0.7756252288818359,
"logits/rejected": 0.8814484477043152,
"logps/chosen": -245.0117950439453,
"logps/rejected": -217.2014923095703,
"loss": 2348.3988,
"rewards/accuracies": 0.6187499761581421,
"rewards/chosen": -0.05819234997034073,
"rewards/margins": 0.021564457565546036,
"rewards/rejected": -0.07975681126117706,
"step": 610
},
{
"epoch": 0.32,
"learning_rate": 4.9411102806916185e-06,
"logits/chosen": 0.7903825640678406,
"logits/rejected": 0.8663345575332642,
"logps/chosen": -270.52239990234375,
"logps/rejected": -254.25369262695312,
"loss": 2181.5219,
"rewards/accuracies": 0.699999988079071,
"rewards/chosen": -0.05034894496202469,
"rewards/margins": 0.042683206498622894,
"rewards/rejected": -0.09303215146064758,
"step": 620
},
{
"epoch": 0.33,
"learning_rate": 4.9360792380966875e-06,
"logits/chosen": 0.8880133628845215,
"logits/rejected": 0.8636928796768188,
"logps/chosen": -241.08279418945312,
"logps/rejected": -220.8309783935547,
"loss": 2263.642,
"rewards/accuracies": 0.706250011920929,
"rewards/chosen": -0.05800148844718933,
"rewards/margins": 0.0320889875292778,
"rewards/rejected": -0.09009047597646713,
"step": 630
},
{
"epoch": 0.33,
"learning_rate": 4.930844783586424e-06,
"logits/chosen": 0.8702048063278198,
"logits/rejected": 0.8985433578491211,
"logps/chosen": -240.00460815429688,
"logps/rejected": -238.61227416992188,
"loss": 2244.0928,
"rewards/accuracies": 0.668749988079071,
"rewards/chosen": -0.05921437591314316,
"rewards/margins": 0.032827965915203094,
"rewards/rejected": -0.09204234182834625,
"step": 640
},
{
"epoch": 0.34,
"learning_rate": 4.925407354236279e-06,
"logits/chosen": 0.8151038885116577,
"logits/rejected": 0.8790351152420044,
"logps/chosen": -241.45755004882812,
"logps/rejected": -226.37667846679688,
"loss": 2236.8199,
"rewards/accuracies": 0.65625,
"rewards/chosen": -0.06360708922147751,
"rewards/margins": 0.03408312052488327,
"rewards/rejected": -0.09769020974636078,
"step": 650
},
{
"epoch": 0.35,
"learning_rate": 4.919767404070033e-06,
"logits/chosen": 0.8537635803222656,
"logits/rejected": 0.8909260034561157,
"logps/chosen": -221.7104034423828,
"logps/rejected": -206.9849395751953,
"loss": 2317.7387,
"rewards/accuracies": 0.625,
"rewards/chosen": -0.07372693717479706,
"rewards/margins": 0.02632719837129116,
"rewards/rejected": -0.10005412995815277,
"step": 660
},
{
"epoch": 0.35,
"learning_rate": 4.913925404021905e-06,
"logits/chosen": 0.8039971590042114,
"logits/rejected": 0.8006811141967773,
"logps/chosen": -240.97213745117188,
"logps/rejected": -209.97219848632812,
"loss": 2208.3297,
"rewards/accuracies": 0.668749988079071,
"rewards/chosen": -0.07874181121587753,
"rewards/margins": 0.03945617750287056,
"rewards/rejected": -0.11819799244403839,
"step": 670
},
{
"epoch": 0.36,
"learning_rate": 4.907881841897216e-06,
"logits/chosen": 0.8469578623771667,
"logits/rejected": 0.8435947299003601,
"logps/chosen": -257.61810302734375,
"logps/rejected": -245.639892578125,
"loss": 2260.2275,
"rewards/accuracies": 0.6875,
"rewards/chosen": -0.07468362152576447,
"rewards/margins": 0.03678290545940399,
"rewards/rejected": -0.11146652698516846,
"step": 680
},
{
"epoch": 0.36,
"learning_rate": 4.901637222331665e-06,
"logits/chosen": 0.7657278776168823,
"logits/rejected": 0.7471415996551514,
"logps/chosen": -259.5301513671875,
"logps/rejected": -236.86032104492188,
"loss": 2287.06,
"rewards/accuracies": 0.643750011920929,
"rewards/chosen": -0.08623397350311279,
"rewards/margins": 0.03296409547328949,
"rewards/rejected": -0.11919806897640228,
"step": 690
},
{
"epoch": 0.37,
"learning_rate": 4.89519206674919e-06,
"logits/chosen": 0.7977254390716553,
"logits/rejected": 0.8500420451164246,
"logps/chosen": -244.900634765625,
"logps/rejected": -251.4091339111328,
"loss": 2330.7787,
"rewards/accuracies": 0.612500011920929,
"rewards/chosen": -0.08788236975669861,
"rewards/margins": 0.02753649279475212,
"rewards/rejected": -0.11541886627674103,
"step": 700
},
{
"epoch": 0.37,
"eval_logits/chosen": 0.7516666650772095,
"eval_logits/rejected": 0.812827467918396,
"eval_logps/chosen": -264.809326171875,
"eval_logps/rejected": -245.7631378173828,
"eval_loss": 2224.3349609375,
"eval_rewards/accuracies": 0.6629999876022339,
"eval_rewards/chosen": -0.08191882818937302,
"eval_rewards/margins": 0.0402236245572567,
"eval_rewards/rejected": -0.12214244902133942,
"eval_runtime": 416.6908,
"eval_samples_per_second": 4.8,
"eval_steps_per_second": 1.2,
"step": 700
},
{
"epoch": 0.37,
"learning_rate": 4.8885469133184235e-06,
"logits/chosen": 0.8586422204971313,
"logits/rejected": 0.8278988003730774,
"logps/chosen": -244.2412109375,
"logps/rejected": -248.913818359375,
"loss": 2265.9631,
"rewards/accuracies": 0.6187499761581421,
"rewards/chosen": -0.0877029225230217,
"rewards/margins": 0.03201908990740776,
"rewards/rejected": -0.11972200870513916,
"step": 710
},
{
"epoch": 0.38,
"learning_rate": 4.881702316907769e-06,
"logits/chosen": 0.853478729724884,
"logits/rejected": 0.9013971090316772,
"logps/chosen": -240.85440063476562,
"logps/rejected": -233.39761352539062,
"loss": 2238.1352,
"rewards/accuracies": 0.625,
"rewards/chosen": -0.08841414749622345,
"rewards/margins": 0.03821689262986183,
"rewards/rejected": -0.12663105130195618,
"step": 720
},
{
"epoch": 0.38,
"learning_rate": 4.874658849039054e-06,
"logits/chosen": 0.6829933524131775,
"logits/rejected": 0.7707113027572632,
"logps/chosen": -274.42095947265625,
"logps/rejected": -232.1612091064453,
"loss": 2119.9938,
"rewards/accuracies": 0.6625000238418579,
"rewards/chosen": -0.07465031743049622,
"rewards/margins": 0.049049459397792816,
"rewards/rejected": -0.12369978427886963,
"step": 730
},
{
"epoch": 0.39,
"learning_rate": 4.86741709783982e-06,
"logits/chosen": 0.7617892026901245,
"logits/rejected": 0.8164576292037964,
"logps/chosen": -267.9073486328125,
"logps/rejected": -240.621826171875,
"loss": 2243.2557,
"rewards/accuracies": 0.6812499761581421,
"rewards/chosen": -0.085872121155262,
"rewards/margins": 0.042114924639463425,
"rewards/rejected": -0.1279870569705963,
"step": 740
},
{
"epoch": 0.39,
"learning_rate": 4.859977667994209e-06,
"logits/chosen": 0.74756920337677,
"logits/rejected": 0.8244425654411316,
"logps/chosen": -255.57754516601562,
"logps/rejected": -242.3626251220703,
"loss": 2231.4938,
"rewards/accuracies": 0.606249988079071,
"rewards/chosen": -0.09324190765619278,
"rewards/margins": 0.04055342823266983,
"rewards/rejected": -0.1337953507900238,
"step": 750
},
{
"epoch": 0.4,
"learning_rate": 4.852341180692471e-06,
"logits/chosen": 0.7698923945426941,
"logits/rejected": 0.7992275953292847,
"logps/chosen": -256.96868896484375,
"logps/rejected": -271.1941833496094,
"loss": 2239.7266,
"rewards/accuracies": 0.668749988079071,
"rewards/chosen": -0.08798633515834808,
"rewards/margins": 0.0419507697224617,
"rewards/rejected": -0.1299370974302292,
"step": 760
},
{
"epoch": 0.4,
"learning_rate": 4.844508273579097e-06,
"logits/chosen": 0.803545355796814,
"logits/rejected": 0.7743754982948303,
"logps/chosen": -249.5584716796875,
"logps/rejected": -238.66683959960938,
"loss": 2204.2545,
"rewards/accuracies": 0.6625000238418579,
"rewards/chosen": -0.08735480904579163,
"rewards/margins": 0.04183940216898918,
"rewards/rejected": -0.1291942000389099,
"step": 770
},
{
"epoch": 0.41,
"learning_rate": 4.836479600699579e-06,
"logits/chosen": 0.7211157083511353,
"logits/rejected": 0.7573873400688171,
"logps/chosen": -251.00076293945312,
"logps/rejected": -248.5784454345703,
"loss": 2175.5199,
"rewards/accuracies": 0.625,
"rewards/chosen": -0.09977405518293381,
"rewards/margins": 0.047359712421894073,
"rewards/rejected": -0.14713376760482788,
"step": 780
},
{
"epoch": 0.41,
"learning_rate": 4.82825583244579e-06,
"logits/chosen": 0.7303954362869263,
"logits/rejected": 0.7423623204231262,
"logps/chosen": -271.29888916015625,
"logps/rejected": -255.95645141601562,
"loss": 2097.0797,
"rewards/accuracies": 0.6625000238418579,
"rewards/chosen": -0.09688698500394821,
"rewards/margins": 0.060234714299440384,
"rewards/rejected": -0.1571216881275177,
"step": 790
},
{
"epoch": 0.42,
"learning_rate": 4.819837655500014e-06,
"logits/chosen": 0.7324298620223999,
"logits/rejected": 0.8285747766494751,
"logps/chosen": -259.47650146484375,
"logps/rejected": -261.0966491699219,
"loss": 2223.6863,
"rewards/accuracies": 0.6812499761581421,
"rewards/chosen": -0.1077154278755188,
"rewards/margins": 0.04667884111404419,
"rewards/rejected": -0.154394268989563,
"step": 800
},
{
"epoch": 0.42,
"eval_logits/chosen": 0.6992308497428894,
"eval_logits/rejected": 0.7610952854156494,
"eval_logps/chosen": -266.70574951171875,
"eval_logps/rejected": -248.42222595214844,
"eval_loss": 2196.09912109375,
"eval_rewards/accuracies": 0.6675000190734863,
"eval_rewards/chosen": -0.1008833572268486,
"eval_rewards/margins": 0.04784964770078659,
"eval_rewards/rejected": -0.1487330049276352,
"eval_runtime": 416.5458,
"eval_samples_per_second": 4.801,
"eval_steps_per_second": 1.2,
"step": 800
},
{
"epoch": 0.42,
"learning_rate": 4.811225772777603e-06,
"logits/chosen": 0.8175959587097168,
"logits/rejected": 0.7778623700141907,
"logps/chosen": -281.6056823730469,
"logps/rejected": -235.4732666015625,
"loss": 2154.2184,
"rewards/accuracies": 0.675000011920929,
"rewards/chosen": -0.10602346807718277,
"rewards/margins": 0.0522245354950428,
"rewards/rejected": -0.15824799239635468,
"step": 810
},
{
"epoch": 0.43,
"learning_rate": 4.802420903368286e-06,
"logits/chosen": 0.6645683646202087,
"logits/rejected": 0.7504470944404602,
"logps/chosen": -262.6244812011719,
"logps/rejected": -246.62728881835938,
"loss": 2070.7916,
"rewards/accuracies": 0.699999988079071,
"rewards/chosen": -0.08415599167346954,
"rewards/margins": 0.06298204511404037,
"rewards/rejected": -0.14713802933692932,
"step": 820
},
{
"epoch": 0.43,
"learning_rate": 4.793423782476125e-06,
"logits/chosen": 0.7014611959457397,
"logits/rejected": 0.7595884203910828,
"logps/chosen": -261.1951599121094,
"logps/rejected": -237.8996124267578,
"loss": 2303.9654,
"rewards/accuracies": 0.65625,
"rewards/chosen": -0.11634109169244766,
"rewards/margins": 0.03935299813747406,
"rewards/rejected": -0.15569409728050232,
"step": 830
},
{
"epoch": 0.44,
"learning_rate": 4.784235161358124e-06,
"logits/chosen": 0.7220847010612488,
"logits/rejected": 0.8208295702934265,
"logps/chosen": -274.5125427246094,
"logps/rejected": -267.01275634765625,
"loss": 2374.0922,
"rewards/accuracies": 0.643750011920929,
"rewards/chosen": -0.11991143226623535,
"rewards/margins": 0.03418232128024101,
"rewards/rejected": -0.15409375727176666,
"step": 840
},
{
"epoch": 0.44,
"learning_rate": 4.774855807261504e-06,
"logits/chosen": 0.771617591381073,
"logits/rejected": 0.7759231925010681,
"logps/chosen": -266.54156494140625,
"logps/rejected": -232.45114135742188,
"loss": 2085.9527,
"rewards/accuracies": 0.7124999761581421,
"rewards/chosen": -0.09387587755918503,
"rewards/margins": 0.059171438217163086,
"rewards/rejected": -0.1530473232269287,
"step": 850
},
{
"epoch": 0.45,
"learning_rate": 4.765286503359632e-06,
"logits/chosen": 0.7542043924331665,
"logits/rejected": 0.7199236154556274,
"logps/chosen": -247.95004272460938,
"logps/rejected": -234.8477325439453,
"loss": 2140.2543,
"rewards/accuracies": 0.71875,
"rewards/chosen": -0.09680913388729095,
"rewards/margins": 0.05759376287460327,
"rewards/rejected": -0.15440289676189423,
"step": 860
},
{
"epoch": 0.46,
"learning_rate": 4.755528048686629e-06,
"logits/chosen": 0.7054905891418457,
"logits/rejected": 0.7627168297767639,
"logps/chosen": -269.0238342285156,
"logps/rejected": -231.31594848632812,
"loss": 2018.9971,
"rewards/accuracies": 0.65625,
"rewards/chosen": -0.09635048359632492,
"rewards/margins": 0.07080944627523422,
"rewards/rejected": -0.16715992987155914,
"step": 870
},
{
"epoch": 0.46,
"learning_rate": 4.745581258070654e-06,
"logits/chosen": 0.7587330937385559,
"logits/rejected": 0.7288376092910767,
"logps/chosen": -279.3045959472656,
"logps/rejected": -248.2842559814453,
"loss": 2182.2826,
"rewards/accuracies": 0.643750011920929,
"rewards/chosen": -0.10375384986400604,
"rewards/margins": 0.04985477030277252,
"rewards/rejected": -0.15360862016677856,
"step": 880
},
{
"epoch": 0.47,
"learning_rate": 4.73544696206586e-06,
"logits/chosen": 0.7099634408950806,
"logits/rejected": 0.7651978731155396,
"logps/chosen": -241.7110137939453,
"logps/rejected": -226.1469268798828,
"loss": 2313.518,
"rewards/accuracies": 0.612500011920929,
"rewards/chosen": -0.11171738058328629,
"rewards/margins": 0.033566057682037354,
"rewards/rejected": -0.14528343081474304,
"step": 890
},
{
"epoch": 0.47,
"learning_rate": 4.725126006883047e-06,
"logits/chosen": 0.6832414865493774,
"logits/rejected": 0.6996358633041382,
"logps/chosen": -287.02655029296875,
"logps/rejected": -271.8457946777344,
"loss": 2066.7418,
"rewards/accuracies": 0.6875,
"rewards/chosen": -0.1082179993391037,
"rewards/margins": 0.06335000693798065,
"rewards/rejected": -0.17156800627708435,
"step": 900
},
{
"epoch": 0.47,
"eval_logits/chosen": 0.6917389035224915,
"eval_logits/rejected": 0.7518260478973389,
"eval_logps/chosen": -267.73968505859375,
"eval_logps/rejected": -250.13189697265625,
"eval_loss": 2166.0732421875,
"eval_rewards/accuracies": 0.6700000166893005,
"eval_rewards/chosen": -0.11122233420610428,
"eval_rewards/margins": 0.0546073243021965,
"eval_rewards/rejected": -0.16582968831062317,
"eval_runtime": 416.784,
"eval_samples_per_second": 4.799,
"eval_steps_per_second": 1.2,
"step": 900
},
{
"epoch": 0.48,
"learning_rate": 4.7146192543190005e-06,
"logits/chosen": 0.7068012952804565,
"logits/rejected": 0.7604703307151794,
"logps/chosen": -301.75897216796875,
"logps/rejected": -260.5961608886719,
"loss": 2114.2752,
"rewards/accuracies": 0.699999988079071,
"rewards/chosen": -0.10466556251049042,
"rewards/margins": 0.06352122128009796,
"rewards/rejected": -0.16818679869174957,
"step": 910
},
{
"epoch": 0.48,
"learning_rate": 4.70392758168454e-06,
"logits/chosen": 0.7013599872589111,
"logits/rejected": 0.7524459362030029,
"logps/chosen": -280.307861328125,
"logps/rejected": -256.722900390625,
"loss": 2081.7857,
"rewards/accuracies": 0.699999988079071,
"rewards/chosen": -0.11534447968006134,
"rewards/margins": 0.06791369616985321,
"rewards/rejected": -0.18325819075107574,
"step": 920
},
{
"epoch": 0.49,
"learning_rate": 4.693051881731251e-06,
"logits/chosen": 0.6879482269287109,
"logits/rejected": 0.7315651178359985,
"logps/chosen": -267.4771728515625,
"logps/rejected": -269.62249755859375,
"loss": 2219.8021,
"rewards/accuracies": 0.6625000238418579,
"rewards/chosen": -0.11604435741901398,
"rewards/margins": 0.04904730245471001,
"rewards/rejected": -0.16509169340133667,
"step": 930
},
{
"epoch": 0.49,
"learning_rate": 4.68199306257695e-06,
"logits/chosen": 0.7221347093582153,
"logits/rejected": 0.8089338541030884,
"logps/chosen": -277.0709533691406,
"logps/rejected": -288.2813415527344,
"loss": 2066.718,
"rewards/accuracies": 0.7124999761581421,
"rewards/chosen": -0.09484682977199554,
"rewards/margins": 0.066488116979599,
"rewards/rejected": -0.16133496165275574,
"step": 940
},
{
"epoch": 0.5,
"learning_rate": 4.670752047629855e-06,
"logits/chosen": 0.7649358510971069,
"logits/rejected": 0.8068546056747437,
"logps/chosen": -289.7987976074219,
"logps/rejected": -257.411865234375,
"loss": 1989.5814,
"rewards/accuracies": 0.6875,
"rewards/chosen": -0.09663524478673935,
"rewards/margins": 0.07417033612728119,
"rewards/rejected": -0.17080560326576233,
"step": 950
},
{
"epoch": 0.5,
"learning_rate": 4.659329775511478e-06,
"logits/chosen": 0.6801126599311829,
"logits/rejected": 0.7201008796691895,
"logps/chosen": -275.82244873046875,
"logps/rejected": -263.333251953125,
"loss": 2137.0553,
"rewards/accuracies": 0.6812499761581421,
"rewards/chosen": -0.10409847646951675,
"rewards/margins": 0.06283750385046005,
"rewards/rejected": -0.1669359654188156,
"step": 960
},
{
"epoch": 0.51,
"learning_rate": 4.647727199978255e-06,
"logits/chosen": 0.675479531288147,
"logits/rejected": 0.755820095539093,
"logps/chosen": -281.660888671875,
"logps/rejected": -264.6354675292969,
"loss": 2290.1266,
"rewards/accuracies": 0.606249988079071,
"rewards/chosen": -0.11869573593139648,
"rewards/margins": 0.0478622205555439,
"rewards/rejected": -0.1665579378604889,
"step": 970
},
{
"epoch": 0.51,
"learning_rate": 4.635945289841902e-06,
"logits/chosen": 0.6294044852256775,
"logits/rejected": 0.701261043548584,
"logps/chosen": -249.8705596923828,
"logps/rejected": -245.52346801757812,
"loss": 2214.392,
"rewards/accuracies": 0.643750011920929,
"rewards/chosen": -0.12245059013366699,
"rewards/margins": 0.05106619745492935,
"rewards/rejected": -0.17351679503917694,
"step": 980
},
{
"epoch": 0.52,
"learning_rate": 4.623985028888527e-06,
"logits/chosen": 0.7620214223861694,
"logits/rejected": 0.799843430519104,
"logps/chosen": -236.2934112548828,
"logps/rejected": -222.5919189453125,
"loss": 2149.6912,
"rewards/accuracies": 0.637499988079071,
"rewards/chosen": -0.12090835720300674,
"rewards/margins": 0.05957023426890373,
"rewards/rejected": -0.18047860264778137,
"step": 990
},
{
"epoch": 0.52,
"learning_rate": 4.611847415796476e-06,
"logits/chosen": 0.7124743461608887,
"logits/rejected": 0.6904253363609314,
"logps/chosen": -265.0976257324219,
"logps/rejected": -254.4890594482422,
"loss": 2119.2691,
"rewards/accuracies": 0.6499999761581421,
"rewards/chosen": -0.11531393229961395,
"rewards/margins": 0.0636182576417923,
"rewards/rejected": -0.17893218994140625,
"step": 1000
},
{
"epoch": 0.52,
"eval_logits/chosen": 0.6619382500648499,
"eval_logits/rejected": 0.721328854560852,
"eval_logps/chosen": -268.7693176269531,
"eval_logps/rejected": -251.76100158691406,
"eval_loss": 2138.93115234375,
"eval_rewards/accuracies": 0.671500027179718,
"eval_rewards/chosen": -0.12151883542537689,
"eval_rewards/margins": 0.060602057725191116,
"eval_rewards/rejected": -0.1821209043264389,
"eval_runtime": 416.4897,
"eval_samples_per_second": 4.802,
"eval_steps_per_second": 1.201,
"step": 1000
},
{
"epoch": 0.53,
"learning_rate": 4.599533464052951e-06,
"logits/chosen": 0.7095866203308105,
"logits/rejected": 0.7142434120178223,
"logps/chosen": -285.8958740234375,
"logps/rejected": -269.59906005859375,
"loss": 2002.4428,
"rewards/accuracies": 0.6625000238418579,
"rewards/chosen": -0.09995652735233307,
"rewards/margins": 0.08161594718694687,
"rewards/rejected": -0.18157246708869934,
"step": 1010
},
{
"epoch": 0.53,
"learning_rate": 4.587044201869378e-06,
"logits/chosen": 0.6786423921585083,
"logits/rejected": 0.7168447375297546,
"logps/chosen": -285.7081298828125,
"logps/rejected": -245.5774688720703,
"loss": 2104.4162,
"rewards/accuracies": 0.6875,
"rewards/chosen": -0.10604969412088394,
"rewards/margins": 0.06783930957317352,
"rewards/rejected": -0.17388899624347687,
"step": 1020
},
{
"epoch": 0.54,
"learning_rate": 4.574380672095555e-06,
"logits/chosen": 0.6884575486183167,
"logits/rejected": 0.7298802733421326,
"logps/chosen": -223.19393920898438,
"logps/rejected": -240.6363983154297,
"loss": 2218.8305,
"rewards/accuracies": 0.637499988079071,
"rewards/chosen": -0.1402120441198349,
"rewards/margins": 0.049252741038799286,
"rewards/rejected": -0.1894647777080536,
"step": 1030
},
{
"epoch": 0.54,
"learning_rate": 4.561543932132574e-06,
"logits/chosen": 0.6861045360565186,
"logits/rejected": 0.7231858968734741,
"logps/chosen": -282.3648376464844,
"logps/rejected": -247.057373046875,
"loss": 2149.1324,
"rewards/accuracies": 0.6499999761581421,
"rewards/chosen": -0.1236349567770958,
"rewards/margins": 0.05487797409296036,
"rewards/rejected": -0.17851293087005615,
"step": 1040
},
{
"epoch": 0.55,
"learning_rate": 4.548535053844527e-06,
"logits/chosen": 0.6396581530570984,
"logits/rejected": 0.7092006206512451,
"logps/chosen": -280.1047058105469,
"logps/rejected": -267.3292541503906,
"loss": 2057.9182,
"rewards/accuracies": 0.6937500238418579,
"rewards/chosen": -0.12618432939052582,
"rewards/margins": 0.07123459875583649,
"rewards/rejected": -0.1974189579486847,
"step": 1050
},
{
"epoch": 0.55,
"learning_rate": 4.535355123469009e-06,
"logits/chosen": 0.7520751357078552,
"logits/rejected": 0.7533235549926758,
"logps/chosen": -246.0561065673828,
"logps/rejected": -210.5857391357422,
"loss": 2157.4354,
"rewards/accuracies": 0.643750011920929,
"rewards/chosen": -0.11811725795269012,
"rewards/margins": 0.05882970616221428,
"rewards/rejected": -0.1769469678401947,
"step": 1060
},
{
"epoch": 0.56,
"learning_rate": 4.522005241526411e-06,
"logits/chosen": 0.670494019985199,
"logits/rejected": 0.7469106912612915,
"logps/chosen": -281.83013916015625,
"logps/rejected": -235.8207244873047,
"loss": 2224.4785,
"rewards/accuracies": 0.6499999761581421,
"rewards/chosen": -0.12622186541557312,
"rewards/margins": 0.04977993294596672,
"rewards/rejected": -0.17600181698799133,
"step": 1070
},
{
"epoch": 0.57,
"learning_rate": 4.508486522728037e-06,
"logits/chosen": 0.6910241842269897,
"logits/rejected": 0.7257175445556641,
"logps/chosen": -276.518798828125,
"logps/rejected": -260.69781494140625,
"loss": 2003.6445,
"rewards/accuracies": 0.6812499761581421,
"rewards/chosen": -0.1153578907251358,
"rewards/margins": 0.07534319162368774,
"rewards/rejected": -0.19070109724998474,
"step": 1080
},
{
"epoch": 0.57,
"learning_rate": 4.494800095883014e-06,
"logits/chosen": 0.6132059097290039,
"logits/rejected": 0.6958727836608887,
"logps/chosen": -290.1937255859375,
"logps/rejected": -244.7795867919922,
"loss": 1914.4395,
"rewards/accuracies": 0.6937500238418579,
"rewards/chosen": -0.1075458973646164,
"rewards/margins": 0.09307406842708588,
"rewards/rejected": -0.20061998069286346,
"step": 1090
},
{
"epoch": 0.58,
"learning_rate": 4.480947103804044e-06,
"logits/chosen": 0.5848616361618042,
"logits/rejected": 0.6844476461410522,
"logps/chosen": -286.0004577636719,
"logps/rejected": -235.7235565185547,
"loss": 2191.7109,
"rewards/accuracies": 0.668749988079071,
"rewards/chosen": -0.12465153634548187,
"rewards/margins": 0.05475841090083122,
"rewards/rejected": -0.1794099658727646,
"step": 1100
},
{
"epoch": 0.58,
"eval_logits/chosen": 0.6583799123764038,
"eval_logits/rejected": 0.7175658345222473,
"eval_logps/chosen": -269.19097900390625,
"eval_logps/rejected": -252.60589599609375,
"eval_loss": 2121.8115234375,
"eval_rewards/accuracies": 0.6694999933242798,
"eval_rewards/chosen": -0.12573528289794922,
"eval_rewards/margins": 0.06483451277017593,
"eval_rewards/rejected": -0.19056977331638336,
"eval_runtime": 416.4568,
"eval_samples_per_second": 4.802,
"eval_steps_per_second": 1.201,
"step": 1100
},
{
"epoch": 0.58,
"learning_rate": 4.466928703211981e-06,
"logits/chosen": 0.6980951428413391,
"logits/rejected": 0.6862035989761353,
"logps/chosen": -281.24700927734375,
"logps/rejected": -248.4276123046875,
"loss": 2063.3184,
"rewards/accuracies": 0.6812499761581421,
"rewards/chosen": -0.11639384180307388,
"rewards/margins": 0.07433497160673141,
"rewards/rejected": -0.1907288283109665,
"step": 1110
},
{
"epoch": 0.59,
"learning_rate": 4.452746064639239e-06,
"logits/chosen": 0.678636372089386,
"logits/rejected": 0.6571283936500549,
"logps/chosen": -258.97674560546875,
"logps/rejected": -254.3134307861328,
"loss": 2036.2967,
"rewards/accuracies": 0.7124999761581421,
"rewards/chosen": -0.10998505353927612,
"rewards/margins": 0.06968870759010315,
"rewards/rejected": -0.17967377603054047,
"step": 1120
},
{
"epoch": 0.59,
"learning_rate": 4.438400372332058e-06,
"logits/chosen": 0.7093490958213806,
"logits/rejected": 0.7673132419586182,
"logps/chosen": -265.0453796386719,
"logps/rejected": -250.15982055664062,
"loss": 1899.6914,
"rewards/accuracies": 0.7437499761581421,
"rewards/chosen": -0.10779963433742523,
"rewards/margins": 0.08556055277585983,
"rewards/rejected": -0.19336020946502686,
"step": 1130
},
{
"epoch": 0.6,
"learning_rate": 4.423892824151617e-06,
"logits/chosen": 0.6779216527938843,
"logits/rejected": 0.7495394945144653,
"logps/chosen": -276.6440734863281,
"logps/rejected": -247.07150268554688,
"loss": 2002.1414,
"rewards/accuracies": 0.71875,
"rewards/chosen": -0.11242429912090302,
"rewards/margins": 0.08376909792423248,
"rewards/rejected": -0.19619342684745789,
"step": 1140
},
{
"epoch": 0.6,
"learning_rate": 4.409224631474014e-06,
"logits/chosen": 0.6950255632400513,
"logits/rejected": 0.7308493256568909,
"logps/chosen": -258.3533020019531,
"logps/rejected": -235.54483032226562,
"loss": 1995.5852,
"rewards/accuracies": 0.7250000238418579,
"rewards/chosen": -0.12844698131084442,
"rewards/margins": 0.07693418860435486,
"rewards/rejected": -0.20538118481636047,
"step": 1150
},
{
"epoch": 0.61,
"learning_rate": 4.3943970190891164e-06,
"logits/chosen": 0.6389755010604858,
"logits/rejected": 0.6700756549835205,
"logps/chosen": -264.4709167480469,
"logps/rejected": -256.2208557128906,
"loss": 2016.508,
"rewards/accuracies": 0.6937500238418579,
"rewards/chosen": -0.13102933764457703,
"rewards/margins": 0.07655525207519531,
"rewards/rejected": -0.20758457481861115,
"step": 1160
},
{
"epoch": 0.61,
"learning_rate": 4.379411225098292e-06,
"logits/chosen": 0.6980705261230469,
"logits/rejected": 0.7923838496208191,
"logps/chosen": -283.1957702636719,
"logps/rejected": -267.30938720703125,
"loss": 2012.2674,
"rewards/accuracies": 0.731249988079071,
"rewards/chosen": -0.138292133808136,
"rewards/margins": 0.08078579604625702,
"rewards/rejected": -0.2190779149532318,
"step": 1170
},
{
"epoch": 0.62,
"learning_rate": 4.364268500811025e-06,
"logits/chosen": 0.6915451288223267,
"logits/rejected": 0.6770834922790527,
"logps/chosen": -258.2509460449219,
"logps/rejected": -266.7464294433594,
"loss": 2101.8338,
"rewards/accuracies": 0.675000011920929,
"rewards/chosen": -0.13359954953193665,
"rewards/margins": 0.07587826251983643,
"rewards/rejected": -0.20947781205177307,
"step": 1180
},
{
"epoch": 0.62,
"learning_rate": 4.348970110640437e-06,
"logits/chosen": 0.6509718298912048,
"logits/rejected": 0.7129366993904114,
"logps/chosen": -258.56280517578125,
"logps/rejected": -237.4566192626953,
"loss": 2027.6717,
"rewards/accuracies": 0.6937500238418579,
"rewards/chosen": -0.14995795488357544,
"rewards/margins": 0.07759587466716766,
"rewards/rejected": -0.2275538146495819,
"step": 1190
},
{
"epoch": 0.63,
"learning_rate": 4.333517331997704e-06,
"logits/chosen": 0.5978332161903381,
"logits/rejected": 0.6565033793449402,
"logps/chosen": -272.02166748046875,
"logps/rejected": -271.3028259277344,
"loss": 2308.1883,
"rewards/accuracies": 0.5874999761581421,
"rewards/chosen": -0.1387997567653656,
"rewards/margins": 0.04619471728801727,
"rewards/rejected": -0.18499447405338287,
"step": 1200
},
{
"epoch": 0.63,
"eval_logits/chosen": 0.6329967379570007,
"eval_logits/rejected": 0.6920445561408997,
"eval_logps/chosen": -270.7044372558594,
"eval_logps/rejected": -254.78115844726562,
"eval_loss": 2110.306884765625,
"eval_rewards/accuracies": 0.6664999723434448,
"eval_rewards/chosen": -0.14087003469467163,
"eval_rewards/margins": 0.07145243883132935,
"eval_rewards/rejected": -0.21232248842716217,
"eval_runtime": 416.6934,
"eval_samples_per_second": 4.8,
"eval_steps_per_second": 1.2,
"step": 1200
},
{
"epoch": 0.63,
"learning_rate": 4.317911455185396e-06,
"logits/chosen": 0.6959893703460693,
"logits/rejected": 0.7259203791618347,
"logps/chosen": -266.06829833984375,
"logps/rejected": -238.3871307373047,
"loss": 2262.5527,
"rewards/accuracies": 0.6187499761581421,
"rewards/chosen": -0.15047064423561096,
"rewards/margins": 0.04972488805651665,
"rewards/rejected": -0.2001955509185791,
"step": 1210
},
{
"epoch": 0.64,
"learning_rate": 4.302153783289737e-06,
"logits/chosen": 0.6275352239608765,
"logits/rejected": 0.7193800806999207,
"logps/chosen": -266.05908203125,
"logps/rejected": -258.8971252441406,
"loss": 2233.5896,
"rewards/accuracies": 0.6187499761581421,
"rewards/chosen": -0.1438441276550293,
"rewards/margins": 0.05622429400682449,
"rewards/rejected": -0.2000684291124344,
"step": 1220
},
{
"epoch": 0.64,
"learning_rate": 4.286245632071791e-06,
"logits/chosen": 0.6443454623222351,
"logits/rejected": 0.6870865225791931,
"logps/chosen": -257.45611572265625,
"logps/rejected": -253.72756958007812,
"loss": 2126.4154,
"rewards/accuracies": 0.668749988079071,
"rewards/chosen": -0.15624599158763885,
"rewards/margins": 0.06528286635875702,
"rewards/rejected": -0.22152885794639587,
"step": 1230
},
{
"epoch": 0.65,
"learning_rate": 4.270188329857613e-06,
"logits/chosen": 0.7713927626609802,
"logits/rejected": 0.7753847241401672,
"logps/chosen": -263.57708740234375,
"logps/rejected": -270.3426513671875,
"loss": 2108.1736,
"rewards/accuracies": 0.7124999761581421,
"rewards/chosen": -0.12420948594808578,
"rewards/margins": 0.06607901304960251,
"rewards/rejected": -0.1902884989976883,
"step": 1240
},
{
"epoch": 0.65,
"learning_rate": 4.253983217427313e-06,
"logits/chosen": 0.6878337860107422,
"logits/rejected": 0.7090884447097778,
"logps/chosen": -271.42657470703125,
"logps/rejected": -288.615478515625,
"loss": 2128.0695,
"rewards/accuracies": 0.643750011920929,
"rewards/chosen": -0.13797307014465332,
"rewards/margins": 0.06890544295310974,
"rewards/rejected": -0.20687851309776306,
"step": 1250
},
{
"epoch": 0.66,
"learning_rate": 4.237631647903115e-06,
"logits/chosen": 0.6635148525238037,
"logits/rejected": 0.6455484628677368,
"logps/chosen": -266.98248291015625,
"logps/rejected": -250.23403930664062,
"loss": 2209.6877,
"rewards/accuracies": 0.606249988079071,
"rewards/chosen": -0.13750961422920227,
"rewards/margins": 0.05613657087087631,
"rewards/rejected": -0.1936461478471756,
"step": 1260
},
{
"epoch": 0.66,
"learning_rate": 4.221134986636371e-06,
"logits/chosen": 0.6171488761901855,
"logits/rejected": 0.6567360758781433,
"logps/chosen": -273.8824157714844,
"logps/rejected": -249.103515625,
"loss": 1885.2906,
"rewards/accuracies": 0.7124999761581421,
"rewards/chosen": -0.11038468778133392,
"rewards/margins": 0.10038020461797714,
"rewards/rejected": -0.21076488494873047,
"step": 1270
},
{
"epoch": 0.67,
"learning_rate": 4.204494611093548e-06,
"logits/chosen": 0.7011705636978149,
"logits/rejected": 0.6791177988052368,
"logps/chosen": -251.37478637695312,
"logps/rejected": -261.1529846191406,
"loss": 2258.1799,
"rewards/accuracies": 0.6312500238418579,
"rewards/chosen": -0.14569328725337982,
"rewards/margins": 0.0609690323472023,
"rewards/rejected": -0.20666229724884033,
"step": 1280
},
{
"epoch": 0.68,
"learning_rate": 4.1877119107412165e-06,
"logits/chosen": 0.6343793869018555,
"logits/rejected": 0.6927725672721863,
"logps/chosen": -237.8370819091797,
"logps/rejected": -256.54254150390625,
"loss": 2061.6873,
"rewards/accuracies": 0.6812499761581421,
"rewards/chosen": -0.1375429332256317,
"rewards/margins": 0.07506345212459564,
"rewards/rejected": -0.21260638535022736,
"step": 1290
},
{
"epoch": 0.68,
"learning_rate": 4.170788286930024e-06,
"logits/chosen": 0.6356396675109863,
"logits/rejected": 0.7518913149833679,
"logps/chosen": -275.9583435058594,
"logps/rejected": -253.1591033935547,
"loss": 1996.7178,
"rewards/accuracies": 0.6812499761581421,
"rewards/chosen": -0.1259593516588211,
"rewards/margins": 0.09192151576280594,
"rewards/rejected": -0.21788087487220764,
"step": 1300
},
{
"epoch": 0.68,
"eval_logits/chosen": 0.6140788793563843,
"eval_logits/rejected": 0.6721699833869934,
"eval_logps/chosen": -269.7620544433594,
"eval_logps/rejected": -253.97259521484375,
"eval_loss": 2095.31298828125,
"eval_rewards/accuracies": 0.6754999756813049,
"eval_rewards/chosen": -0.1314462274312973,
"eval_rewards/margins": 0.07279053330421448,
"eval_rewards/rejected": -0.20423679053783417,
"eval_runtime": 416.679,
"eval_samples_per_second": 4.8,
"eval_steps_per_second": 1.2,
"step": 1300
},
{
"epoch": 0.69,
"learning_rate": 4.15372515277769e-06,
"logits/chosen": 0.6244436502456665,
"logits/rejected": 0.6609630584716797,
"logps/chosen": -280.32794189453125,
"logps/rejected": -248.7582244873047,
"loss": 2011.6244,
"rewards/accuracies": 0.6812499761581421,
"rewards/chosen": -0.11511021852493286,
"rewards/margins": 0.08765153586864471,
"rewards/rejected": -0.20276173949241638,
"step": 1310
},
{
"epoch": 0.69,
"learning_rate": 4.136523933051005e-06,
"logits/chosen": 0.6894992589950562,
"logits/rejected": 0.6712801456451416,
"logps/chosen": -263.05767822265625,
"logps/rejected": -234.242431640625,
"loss": 1956.5369,
"rewards/accuracies": 0.737500011920929,
"rewards/chosen": -0.11226965487003326,
"rewards/margins": 0.09646574407815933,
"rewards/rejected": -0.208735391497612,
"step": 1320
},
{
"epoch": 0.7,
"learning_rate": 4.119186064046868e-06,
"logits/chosen": 0.6183528900146484,
"logits/rejected": 0.644507110118866,
"logps/chosen": -274.1941223144531,
"logps/rejected": -245.35562133789062,
"loss": 2166.2785,
"rewards/accuracies": 0.6937500238418579,
"rewards/chosen": -0.12718230485916138,
"rewards/margins": 0.06536950916051865,
"rewards/rejected": -0.19255182147026062,
"step": 1330
},
{
"epoch": 0.7,
"learning_rate": 4.101712993472348e-06,
"logits/chosen": 0.6670488715171814,
"logits/rejected": 0.6754225492477417,
"logps/chosen": -278.2078552246094,
"logps/rejected": -241.5122528076172,
"loss": 1883.2881,
"rewards/accuracies": 0.75,
"rewards/chosen": -0.119564950466156,
"rewards/margins": 0.09150619804859161,
"rewards/rejected": -0.2110711634159088,
"step": 1340
},
{
"epoch": 0.71,
"learning_rate": 4.084106180323813e-06,
"logits/chosen": 0.6214176416397095,
"logits/rejected": 0.66867595911026,
"logps/chosen": -261.7587890625,
"logps/rejected": -251.6807098388672,
"loss": 2017.3074,
"rewards/accuracies": 0.699999988079071,
"rewards/chosen": -0.1294572651386261,
"rewards/margins": 0.08191975206136703,
"rewards/rejected": -0.21137702465057373,
"step": 1350
},
{
"epoch": 0.71,
"learning_rate": 4.066367094765091e-06,
"logits/chosen": 0.6692546606063843,
"logits/rejected": 0.6942587494850159,
"logps/chosen": -259.8294982910156,
"logps/rejected": -268.8597412109375,
"loss": 2097.6473,
"rewards/accuracies": 0.65625,
"rewards/chosen": -0.12980519235134125,
"rewards/margins": 0.07286903262138367,
"rewards/rejected": -0.20267422497272491,
"step": 1360
},
{
"epoch": 0.72,
"learning_rate": 4.048497218004724e-06,
"logits/chosen": 0.5632964968681335,
"logits/rejected": 0.6666015386581421,
"logps/chosen": -265.3409729003906,
"logps/rejected": -252.78958129882812,
"loss": 2076.2803,
"rewards/accuracies": 0.6499999761581421,
"rewards/chosen": -0.13252495229244232,
"rewards/margins": 0.08360429853200912,
"rewards/rejected": -0.21612922847270966,
"step": 1370
},
{
"epoch": 0.72,
"learning_rate": 4.030498042172277e-06,
"logits/chosen": 0.6174412369728088,
"logits/rejected": 0.6783226728439331,
"logps/chosen": -249.213134765625,
"logps/rejected": -241.413330078125,
"loss": 2285.0457,
"rewards/accuracies": 0.6625000238418579,
"rewards/chosen": -0.14881116151809692,
"rewards/margins": 0.05003712326288223,
"rewards/rejected": -0.19884827733039856,
"step": 1380
},
{
"epoch": 0.73,
"learning_rate": 4.012371070193753e-06,
"logits/chosen": 0.6269063353538513,
"logits/rejected": 0.6346549391746521,
"logps/chosen": -241.5870361328125,
"logps/rejected": -245.96456909179688,
"loss": 2133.8158,
"rewards/accuracies": 0.6625000238418579,
"rewards/chosen": -0.1412040889263153,
"rewards/margins": 0.06111832335591316,
"rewards/rejected": -0.20232239365577698,
"step": 1390
},
{
"epoch": 0.73,
"learning_rate": 3.994117815666095e-06,
"logits/chosen": 0.6533576250076294,
"logits/rejected": 0.6740087866783142,
"logps/chosen": -283.4432678222656,
"logps/rejected": -267.79071044921875,
"loss": 2038.3844,
"rewards/accuracies": 0.65625,
"rewards/chosen": -0.1330070197582245,
"rewards/margins": 0.07623559981584549,
"rewards/rejected": -0.20924265682697296,
"step": 1400
},
{
"epoch": 0.73,
"eval_logits/chosen": 0.5933060050010681,
"eval_logits/rejected": 0.6512511968612671,
"eval_logps/chosen": -270.4488220214844,
"eval_logps/rejected": -254.944091796875,
"eval_loss": 2085.085205078125,
"eval_rewards/accuracies": 0.6800000071525574,
"eval_rewards/chosen": -0.13831348717212677,
"eval_rewards/margins": 0.07563827186822891,
"eval_rewards/rejected": -0.21395176649093628,
"eval_runtime": 416.609,
"eval_samples_per_second": 4.801,
"eval_steps_per_second": 1.2,
"step": 1400
},
{
"epoch": 0.74,
"learning_rate": 3.975739802730805e-06,
"logits/chosen": 0.5807250738143921,
"logits/rejected": 0.6861320734024048,
"logps/chosen": -298.25604248046875,
"logps/rejected": -273.8665466308594,
"loss": 2016.2207,
"rewards/accuracies": 0.6875,
"rewards/chosen": -0.12952394783496857,
"rewards/margins": 0.08187790215015411,
"rewards/rejected": -0.21140184998512268,
"step": 1410
},
{
"epoch": 0.74,
"learning_rate": 3.957238565946672e-06,
"logits/chosen": 0.6601132750511169,
"logits/rejected": 0.6705759763717651,
"logps/chosen": -266.2096252441406,
"logps/rejected": -249.5722198486328,
"loss": 2228.1479,
"rewards/accuracies": 0.6499999761581421,
"rewards/chosen": -0.1489478349685669,
"rewards/margins": 0.05832044407725334,
"rewards/rejected": -0.20726828277111053,
"step": 1420
},
{
"epoch": 0.75,
"learning_rate": 3.938615650161645e-06,
"logits/chosen": 0.6056556701660156,
"logits/rejected": 0.5954689979553223,
"logps/chosen": -244.7415771484375,
"logps/rejected": -234.8352813720703,
"loss": 2023.5078,
"rewards/accuracies": 0.706250011920929,
"rewards/chosen": -0.11950768530368805,
"rewards/margins": 0.0772583931684494,
"rewards/rejected": -0.19676607847213745,
"step": 1430
},
{
"epoch": 0.75,
"learning_rate": 3.919872610383831e-06,
"logits/chosen": 0.5716265439987183,
"logits/rejected": 0.6326289176940918,
"logps/chosen": -257.7333984375,
"logps/rejected": -234.2956085205078,
"loss": 2199.8311,
"rewards/accuracies": 0.6312500238418579,
"rewards/chosen": -0.14616943895816803,
"rewards/margins": 0.055809132754802704,
"rewards/rejected": -0.20197856426239014,
"step": 1440
},
{
"epoch": 0.76,
"learning_rate": 3.9010110116516595e-06,
"logits/chosen": 0.655591607093811,
"logits/rejected": 0.7094139456748962,
"logps/chosen": -266.137939453125,
"logps/rejected": -247.2853240966797,
"loss": 1965.8004,
"rewards/accuracies": 0.762499988079071,
"rewards/chosen": -0.13736537098884583,
"rewards/margins": 0.0867539569735527,
"rewards/rejected": -0.22411933541297913,
"step": 1450
},
{
"epoch": 0.76,
"learning_rate": 3.882032428903195e-06,
"logits/chosen": 0.6148696541786194,
"logits/rejected": 0.6896382570266724,
"logps/chosen": -256.70904541015625,
"logps/rejected": -245.3892822265625,
"loss": 2095.5068,
"rewards/accuracies": 0.637499988079071,
"rewards/chosen": -0.12945787608623505,
"rewards/margins": 0.07017168402671814,
"rewards/rejected": -0.19962957501411438,
"step": 1460
},
{
"epoch": 0.77,
"learning_rate": 3.8629384468446365e-06,
"logits/chosen": 0.5744356513023376,
"logits/rejected": 0.5954487919807434,
"logps/chosen": -250.98178100585938,
"logps/rejected": -272.93701171875,
"loss": 2119.459,
"rewards/accuracies": 0.6812499761581421,
"rewards/chosen": -0.13649822771549225,
"rewards/margins": 0.07962769269943237,
"rewards/rejected": -0.21612592041492462,
"step": 1470
},
{
"epoch": 0.77,
"learning_rate": 3.84373065981799e-06,
"logits/chosen": 0.6630114912986755,
"logits/rejected": 0.6675506830215454,
"logps/chosen": -265.52447509765625,
"logps/rejected": -247.32455444335938,
"loss": 1991.6379,
"rewards/accuracies": 0.706250011920929,
"rewards/chosen": -0.12121538817882538,
"rewards/margins": 0.08006526529788971,
"rewards/rejected": -0.20128066837787628,
"step": 1480
},
{
"epoch": 0.78,
"learning_rate": 3.824410671667948e-06,
"logits/chosen": 0.6106497645378113,
"logits/rejected": 0.680738091468811,
"logps/chosen": -260.89288330078125,
"logps/rejected": -252.52017211914062,
"loss": 1942.9977,
"rewards/accuracies": 0.7250000238418579,
"rewards/chosen": -0.12056465446949005,
"rewards/margins": 0.09015407413244247,
"rewards/rejected": -0.21071875095367432,
"step": 1490
},
{
"epoch": 0.79,
"learning_rate": 3.8049800956079552e-06,
"logits/chosen": 0.5932056903839111,
"logits/rejected": 0.6287232637405396,
"logps/chosen": -291.2415771484375,
"logps/rejected": -279.5646057128906,
"loss": 2094.2182,
"rewards/accuracies": 0.668749988079071,
"rewards/chosen": -0.13307559490203857,
"rewards/margins": 0.0750352293252945,
"rewards/rejected": -0.20811080932617188,
"step": 1500
},
{
"epoch": 0.79,
"eval_logits/chosen": 0.5898318886756897,
"eval_logits/rejected": 0.6474130749702454,
"eval_logps/chosen": -270.5129089355469,
"eval_logps/rejected": -255.21328735351562,
"eval_loss": 2076.30419921875,
"eval_rewards/accuracies": 0.6790000200271606,
"eval_rewards/chosen": -0.13895468413829803,
"eval_rewards/margins": 0.07768914848566055,
"eval_rewards/rejected": -0.2166438102722168,
"eval_runtime": 416.7121,
"eval_samples_per_second": 4.799,
"eval_steps_per_second": 1.2,
"step": 1500
},
{
"epoch": 0.79,
"learning_rate": 3.7854405540855268e-06,
"logits/chosen": 0.580877959728241,
"logits/rejected": 0.6030541658401489,
"logps/chosen": -255.82693481445312,
"logps/rejected": -249.1620635986328,
"loss": 2165.7623,
"rewards/accuracies": 0.706250011920929,
"rewards/chosen": -0.1440289467573166,
"rewards/margins": 0.06519783288240433,
"rewards/rejected": -0.2092268019914627,
"step": 1510
},
{
"epoch": 0.8,
"learning_rate": 3.765793678646753e-06,
"logits/chosen": 0.612065851688385,
"logits/rejected": 0.6108434200286865,
"logps/chosen": -236.6591796875,
"logps/rejected": -245.4730224609375,
"loss": 2151.6375,
"rewards/accuracies": 0.6812499761581421,
"rewards/chosen": -0.13691964745521545,
"rewards/margins": 0.06442641466856003,
"rewards/rejected": -0.20134606957435608,
"step": 1520
},
{
"epoch": 0.8,
"learning_rate": 3.7460411098000804e-06,
"logits/chosen": 0.620397686958313,
"logits/rejected": 0.6705790758132935,
"logps/chosen": -279.47003173828125,
"logps/rejected": -242.50320434570312,
"loss": 2097.6518,
"rewards/accuracies": 0.6625000238418579,
"rewards/chosen": -0.13826757669448853,
"rewards/margins": 0.06730998307466507,
"rewards/rejected": -0.2055775672197342,
"step": 1530
},
{
"epoch": 0.81,
"learning_rate": 3.726184496879323e-06,
"logits/chosen": 0.5731703042984009,
"logits/rejected": 0.6038475036621094,
"logps/chosen": -273.57684326171875,
"logps/rejected": -263.6417541503906,
"loss": 1980.56,
"rewards/accuracies": 0.6875,
"rewards/chosen": -0.13297039270401,
"rewards/margins": 0.08892510086297989,
"rewards/rejected": -0.2218955010175705,
"step": 1540
},
{
"epoch": 0.81,
"learning_rate": 3.706225497905946e-06,
"logits/chosen": 0.5495398640632629,
"logits/rejected": 0.6184272170066833,
"logps/chosen": -278.1634521484375,
"logps/rejected": -250.0457763671875,
"loss": 1925.0881,
"rewards/accuracies": 0.6937500238418579,
"rewards/chosen": -0.14073289930820465,
"rewards/margins": 0.08853240311145782,
"rewards/rejected": -0.22926530241966248,
"step": 1550
},
{
"epoch": 0.82,
"learning_rate": 3.686165779450619e-06,
"logits/chosen": 0.6478545069694519,
"logits/rejected": 0.6362086534500122,
"logps/chosen": -267.02618408203125,
"logps/rejected": -239.1699676513672,
"loss": 2063.3338,
"rewards/accuracies": 0.65625,
"rewards/chosen": -0.13111211359500885,
"rewards/margins": 0.07986196875572205,
"rewards/rejected": -0.2109740674495697,
"step": 1560
},
{
"epoch": 0.82,
"learning_rate": 3.6660070164940614e-06,
"logits/chosen": 0.6316484808921814,
"logits/rejected": 0.686813473701477,
"logps/chosen": -254.73837280273438,
"logps/rejected": -249.4886016845703,
"loss": 1958.6893,
"rewards/accuracies": 0.737500011920929,
"rewards/chosen": -0.1321488916873932,
"rewards/margins": 0.09855500608682632,
"rewards/rejected": -0.23070387542247772,
"step": 1570
},
{
"epoch": 0.83,
"learning_rate": 3.645750892287178e-06,
"logits/chosen": 0.6227657794952393,
"logits/rejected": 0.642948567867279,
"logps/chosen": -254.3902587890625,
"logps/rejected": -239.3006134033203,
"loss": 2093.4068,
"rewards/accuracies": 0.675000011920929,
"rewards/chosen": -0.14098116755485535,
"rewards/margins": 0.07492861151695251,
"rewards/rejected": -0.21590976417064667,
"step": 1580
},
{
"epoch": 0.83,
"learning_rate": 3.6253990982105114e-06,
"logits/chosen": 0.5823426842689514,
"logits/rejected": 0.6044851541519165,
"logps/chosen": -282.6208801269531,
"logps/rejected": -284.2301330566406,
"loss": 2308.56,
"rewards/accuracies": 0.625,
"rewards/chosen": -0.1489720195531845,
"rewards/margins": 0.050694145262241364,
"rewards/rejected": -0.19966615736484528,
"step": 1590
},
{
"epoch": 0.84,
"learning_rate": 3.604953333633009e-06,
"logits/chosen": 0.6414502859115601,
"logits/rejected": 0.6938506960868835,
"logps/chosen": -254.117431640625,
"logps/rejected": -234.0909423828125,
"loss": 2171.3457,
"rewards/accuracies": 0.637499988079071,
"rewards/chosen": -0.12779875099658966,
"rewards/margins": 0.06595613807439804,
"rewards/rejected": -0.1937548816204071,
"step": 1600
},
{
"epoch": 0.84,
"eval_logits/chosen": 0.5817673802375793,
"eval_logits/rejected": 0.6391910910606384,
"eval_logps/chosen": -270.3594970703125,
"eval_logps/rejected": -255.21298217773438,
"eval_loss": 2069.375732421875,
"eval_rewards/accuracies": 0.6809999942779541,
"eval_rewards/chosen": -0.137420654296875,
"eval_rewards/margins": 0.07922003418207169,
"eval_rewards/rejected": -0.2166406810283661,
"eval_runtime": 416.6975,
"eval_samples_per_second": 4.8,
"eval_steps_per_second": 1.2,
"step": 1600
},
{
"epoch": 0.84,
"learning_rate": 3.5844153057701303e-06,
"logits/chosen": 0.6806268095970154,
"logits/rejected": 0.6613883376121521,
"logps/chosen": -293.35455322265625,
"logps/rejected": -249.47317504882812,
"loss": 2235.1336,
"rewards/accuracies": 0.637499988079071,
"rewards/chosen": -0.13428549468517303,
"rewards/margins": 0.06987977027893066,
"rewards/rejected": -0.2041652649641037,
"step": 1610
},
{
"epoch": 0.85,
"learning_rate": 3.56378672954129e-06,
"logits/chosen": 0.5934259295463562,
"logits/rejected": 0.6393053531646729,
"logps/chosen": -263.8625183105469,
"logps/rejected": -268.93646240234375,
"loss": 2115.3395,
"rewards/accuracies": 0.6625000238418579,
"rewards/chosen": -0.12568698823451996,
"rewards/margins": 0.06675116717815399,
"rewards/rejected": -0.19243815541267395,
"step": 1620
},
{
"epoch": 0.85,
"learning_rate": 3.5430693274266694e-06,
"logits/chosen": 0.6212111711502075,
"logits/rejected": 0.6776979565620422,
"logps/chosen": -265.48065185546875,
"logps/rejected": -242.78189086914062,
"loss": 1885.1145,
"rewards/accuracies": 0.731249988079071,
"rewards/chosen": -0.12569081783294678,
"rewards/margins": 0.09405811876058578,
"rewards/rejected": -0.21974892914295197,
"step": 1630
},
{
"epoch": 0.86,
"learning_rate": 3.5222648293233806e-06,
"logits/chosen": 0.5869291424751282,
"logits/rejected": 0.614780843257904,
"logps/chosen": -309.3984069824219,
"logps/rejected": -281.76800537109375,
"loss": 2295.801,
"rewards/accuracies": 0.625,
"rewards/chosen": -0.13158050179481506,
"rewards/margins": 0.05296233296394348,
"rewards/rejected": -0.18454284965991974,
"step": 1640
},
{
"epoch": 0.86,
"learning_rate": 3.5013749724010298e-06,
"logits/chosen": 0.6291738152503967,
"logits/rejected": 0.6847606897354126,
"logps/chosen": -269.4018249511719,
"logps/rejected": -248.9547576904297,
"loss": 1985.4633,
"rewards/accuracies": 0.706250011920929,
"rewards/chosen": -0.12610626220703125,
"rewards/margins": 0.08754386752843857,
"rewards/rejected": -0.21365013718605042,
"step": 1650
},
{
"epoch": 0.87,
"learning_rate": 3.4804015009566573e-06,
"logits/chosen": 0.6193640828132629,
"logits/rejected": 0.6583009958267212,
"logps/chosen": -260.786865234375,
"logps/rejected": -246.417724609375,
"loss": 2067.8201,
"rewards/accuracies": 0.675000011920929,
"rewards/chosen": -0.13031774759292603,
"rewards/margins": 0.07650937139987946,
"rewards/rejected": -0.20682711899280548,
"step": 1660
},
{
"epoch": 0.87,
"learning_rate": 3.459346166269093e-06,
"logits/chosen": 0.569218635559082,
"logits/rejected": 0.6178910136222839,
"logps/chosen": -286.010498046875,
"logps/rejected": -284.7559509277344,
"loss": 2057.5484,
"rewards/accuracies": 0.6812499761581421,
"rewards/chosen": -0.11479117721319199,
"rewards/margins": 0.08598540723323822,
"rewards/rejected": -0.20077654719352722,
"step": 1670
},
{
"epoch": 0.88,
"learning_rate": 3.4382107264527244e-06,
"logits/chosen": 0.6346784234046936,
"logits/rejected": 0.7338213920593262,
"logps/chosen": -300.38739013671875,
"logps/rejected": -259.44525146484375,
"loss": 1994.4248,
"rewards/accuracies": 0.7250000238418579,
"rewards/chosen": -0.13437870144844055,
"rewards/margins": 0.08268047124147415,
"rewards/rejected": -0.2170591652393341,
"step": 1680
},
{
"epoch": 0.88,
"learning_rate": 3.416996946310694e-06,
"logits/chosen": 0.5468164086341858,
"logits/rejected": 0.5939579010009766,
"logps/chosen": -299.57061767578125,
"logps/rejected": -265.4569396972656,
"loss": 1896.8961,
"rewards/accuracies": 0.7124999761581421,
"rewards/chosen": -0.11913790553808212,
"rewards/margins": 0.10197613388299942,
"rewards/rejected": -0.22111406922340393,
"step": 1690
},
{
"epoch": 0.89,
"learning_rate": 3.3957065971875387e-06,
"logits/chosen": 0.5587860345840454,
"logits/rejected": 0.6276572346687317,
"logps/chosen": -253.8291473388672,
"logps/rejected": -239.05685424804688,
"loss": 2189.3863,
"rewards/accuracies": 0.637499988079071,
"rewards/chosen": -0.14831021428108215,
"rewards/margins": 0.06738562881946564,
"rewards/rejected": -0.2156958281993866,
"step": 1700
},
{
"epoch": 0.89,
"eval_logits/chosen": 0.5722830295562744,
"eval_logits/rejected": 0.629075288772583,
"eval_logps/chosen": -270.473876953125,
"eval_logps/rejected": -255.46749877929688,
"eval_loss": 2062.199462890625,
"eval_rewards/accuracies": 0.6779999732971191,
"eval_rewards/chosen": -0.13856419920921326,
"eval_rewards/margins": 0.08062165975570679,
"eval_rewards/rejected": -0.21918585896492004,
"eval_runtime": 416.4246,
"eval_samples_per_second": 4.803,
"eval_steps_per_second": 1.201,
"step": 1700
},
{
"epoch": 0.9,
"learning_rate": 3.3743414568212828e-06,
"logits/chosen": 0.6158628463745117,
"logits/rejected": 0.6673662066459656,
"logps/chosen": -296.7020263671875,
"logps/rejected": -242.6101837158203,
"loss": 2082.5406,
"rewards/accuracies": 0.6625000238418579,
"rewards/chosen": -0.13253936171531677,
"rewards/margins": 0.07927088439464569,
"rewards/rejected": -0.21181027591228485,
"step": 1710
},
{
"epoch": 0.9,
"learning_rate": 3.352903309194999e-06,
"logits/chosen": 0.6274576187133789,
"logits/rejected": 0.6294026374816895,
"logps/chosen": -293.9241638183594,
"logps/rejected": -253.40036010742188,
"loss": 2083.2709,
"rewards/accuracies": 0.7124999761581421,
"rewards/chosen": -0.13250832259655,
"rewards/margins": 0.08279639482498169,
"rewards/rejected": -0.21530470252037048,
"step": 1720
},
{
"epoch": 0.91,
"learning_rate": 3.331393944387845e-06,
"logits/chosen": 0.5965205430984497,
"logits/rejected": 0.70032799243927,
"logps/chosen": -291.1014099121094,
"logps/rejected": -274.6158752441406,
"loss": 2108.5279,
"rewards/accuracies": 0.6312500238418579,
"rewards/chosen": -0.12708380818367004,
"rewards/margins": 0.08121255040168762,
"rewards/rejected": -0.20829637348651886,
"step": 1730
},
{
"epoch": 0.91,
"learning_rate": 3.309815158425591e-06,
"logits/chosen": 0.588997483253479,
"logits/rejected": 0.6009566783905029,
"logps/chosen": -244.7908172607422,
"logps/rejected": -238.26651000976562,
"loss": 2017.7098,
"rewards/accuracies": 0.699999988079071,
"rewards/chosen": -0.11969755589962006,
"rewards/margins": 0.07758014649152756,
"rewards/rejected": -0.19727769494056702,
"step": 1740
},
{
"epoch": 0.92,
"learning_rate": 3.288168753130657e-06,
"logits/chosen": 0.6095719933509827,
"logits/rejected": 0.6279308199882507,
"logps/chosen": -250.91116333007812,
"logps/rejected": -265.10302734375,
"loss": 1975.5611,
"rewards/accuracies": 0.7250000238418579,
"rewards/chosen": -0.1158142238855362,
"rewards/margins": 0.08399386703968048,
"rewards/rejected": -0.19980809092521667,
"step": 1750
},
{
"epoch": 0.92,
"learning_rate": 3.266456535971654e-06,
"logits/chosen": 0.5891221165657043,
"logits/rejected": 0.5675392746925354,
"logps/chosen": -283.29901123046875,
"logps/rejected": -258.32562255859375,
"loss": 1991.3586,
"rewards/accuracies": 0.737500011920929,
"rewards/chosen": -0.1158837080001831,
"rewards/margins": 0.09360859543085098,
"rewards/rejected": -0.2094922959804535,
"step": 1760
},
{
"epoch": 0.93,
"learning_rate": 3.2446803199124666e-06,
"logits/chosen": 0.542614221572876,
"logits/rejected": 0.5660384893417358,
"logps/chosen": -260.6263427734375,
"logps/rejected": -240.3424835205078,
"loss": 2044.5988,
"rewards/accuracies": 0.706250011920929,
"rewards/chosen": -0.12066911160945892,
"rewards/margins": 0.0799749344587326,
"rewards/rejected": -0.20064406096935272,
"step": 1770
},
{
"epoch": 0.93,
"learning_rate": 3.2228419232608692e-06,
"logits/chosen": 0.5963379144668579,
"logits/rejected": 0.6299723386764526,
"logps/chosen": -248.21920776367188,
"logps/rejected": -235.8214874267578,
"loss": 2095.258,
"rewards/accuracies": 0.7124999761581421,
"rewards/chosen": -0.12164044380187988,
"rewards/margins": 0.07495652884244919,
"rewards/rejected": -0.19659698009490967,
"step": 1780
},
{
"epoch": 0.94,
"learning_rate": 3.2009431695166985e-06,
"logits/chosen": 0.5749480724334717,
"logits/rejected": 0.627223014831543,
"logps/chosen": -239.6404266357422,
"logps/rejected": -239.62014770507812,
"loss": 1970.9955,
"rewards/accuracies": 0.71875,
"rewards/chosen": -0.11313848197460175,
"rewards/margins": 0.08247244358062744,
"rewards/rejected": -0.19561094045639038,
"step": 1790
},
{
"epoch": 0.94,
"learning_rate": 3.1789858872195888e-06,
"logits/chosen": 0.6324980854988098,
"logits/rejected": 0.6260117888450623,
"logps/chosen": -244.56362915039062,
"logps/rejected": -245.474609375,
"loss": 2292.8938,
"rewards/accuracies": 0.6625000238418579,
"rewards/chosen": -0.14712049067020416,
"rewards/margins": 0.05482936650514603,
"rewards/rejected": -0.2019498646259308,
"step": 1800
},
{
"epoch": 0.94,
"eval_logits/chosen": 0.5703141689300537,
"eval_logits/rejected": 0.627535343170166,
"eval_logps/chosen": -268.5789489746094,
"eval_logps/rejected": -253.6024627685547,
"eval_loss": 2053.1298828125,
"eval_rewards/accuracies": 0.6830000281333923,
"eval_rewards/chosen": -0.11961515992879868,
"eval_rewards/margins": 0.08092045783996582,
"eval_rewards/rejected": -0.2005356103181839,
"eval_runtime": 416.7248,
"eval_samples_per_second": 4.799,
"eval_steps_per_second": 1.2,
"step": 1800
},
{
"epoch": 0.95,
"learning_rate": 3.156971909796295e-06,
"logits/chosen": 0.6370185613632202,
"logits/rejected": 0.7445378303527832,
"logps/chosen": -265.6059265136719,
"logps/rejected": -232.7034454345703,
"loss": 1958.5383,
"rewards/accuracies": 0.71875,
"rewards/chosen": -0.11271758377552032,
"rewards/margins": 0.08874475955963135,
"rewards/rejected": -0.20146234333515167,
"step": 1810
},
{
"epoch": 0.95,
"learning_rate": 3.1349030754075945e-06,
"logits/chosen": 0.623261034488678,
"logits/rejected": 0.6591364741325378,
"logps/chosen": -263.49993896484375,
"logps/rejected": -241.6349334716797,
"loss": 1940.6969,
"rewards/accuracies": 0.699999988079071,
"rewards/chosen": -0.1059044599533081,
"rewards/margins": 0.09299680590629578,
"rewards/rejected": -0.19890126585960388,
"step": 1820
},
{
"epoch": 0.96,
"learning_rate": 3.1127812267948095e-06,
"logits/chosen": 0.6355741024017334,
"logits/rejected": 0.6655168533325195,
"logps/chosen": -264.20062255859375,
"logps/rejected": -258.43310546875,
"loss": 1993.0014,
"rewards/accuracies": 0.6937500238418579,
"rewards/chosen": -0.12248332798480988,
"rewards/margins": 0.08727528899908066,
"rewards/rejected": -0.20975859463214874,
"step": 1830
},
{
"epoch": 0.96,
"learning_rate": 3.0906082111259313e-06,
"logits/chosen": 0.548941433429718,
"logits/rejected": 0.5715293884277344,
"logps/chosen": -277.3153076171875,
"logps/rejected": -248.1029510498047,
"loss": 2108.9805,
"rewards/accuracies": 0.6625000238418579,
"rewards/chosen": -0.10435888916254044,
"rewards/margins": 0.07835109531879425,
"rewards/rejected": -0.18270999193191528,
"step": 1840
},
{
"epoch": 0.97,
"learning_rate": 3.068385879841389e-06,
"logits/chosen": 0.6165980100631714,
"logits/rejected": 0.6937299966812134,
"logps/chosen": -233.5325469970703,
"logps/rejected": -246.7562255859375,
"loss": 2168.8945,
"rewards/accuracies": 0.675000011920929,
"rewards/chosen": -0.10994801670312881,
"rewards/margins": 0.06389383226633072,
"rewards/rejected": -0.17384183406829834,
"step": 1850
},
{
"epoch": 0.97,
"learning_rate": 3.046116088499449e-06,
"logits/chosen": 0.6379483938217163,
"logits/rejected": 0.6270259618759155,
"logps/chosen": -266.16009521484375,
"logps/rejected": -259.60919189453125,
"loss": 2145.4496,
"rewards/accuracies": 0.6312500238418579,
"rewards/chosen": -0.11401374638080597,
"rewards/margins": 0.07185572385787964,
"rewards/rejected": -0.1858694702386856,
"step": 1860
},
{
"epoch": 0.98,
"learning_rate": 3.02380069662128e-06,
"logits/chosen": 0.623966634273529,
"logits/rejected": 0.5938777327537537,
"logps/chosen": -252.69869995117188,
"logps/rejected": -245.094482421875,
"loss": 2034.7914,
"rewards/accuracies": 0.706250011920929,
"rewards/chosen": -0.10203299671411514,
"rewards/margins": 0.07526172697544098,
"rewards/rejected": -0.17729471623897552,
"step": 1870
},
{
"epoch": 0.98,
"learning_rate": 3.0014415675356813e-06,
"logits/chosen": 0.6284725069999695,
"logits/rejected": 0.6297743916511536,
"logps/chosen": -270.23333740234375,
"logps/rejected": -252.12704467773438,
"loss": 2150.0496,
"rewards/accuracies": 0.625,
"rewards/chosen": -0.10082075744867325,
"rewards/margins": 0.06684517115354538,
"rewards/rejected": -0.16766592860221863,
"step": 1880
},
{
"epoch": 0.99,
"learning_rate": 2.979040568223498e-06,
"logits/chosen": 0.5534299612045288,
"logits/rejected": 0.675399661064148,
"logps/chosen": -263.3745422363281,
"logps/rejected": -266.8883972167969,
"loss": 2205.4939,
"rewards/accuracies": 0.6812499761581421,
"rewards/chosen": -0.11857882887125015,
"rewards/margins": 0.06667140126228333,
"rewards/rejected": -0.18525022268295288,
"step": 1890
},
{
"epoch": 0.99,
"learning_rate": 2.9565995691617242e-06,
"logits/chosen": 0.6073340773582458,
"logits/rejected": 0.6487486362457275,
"logps/chosen": -292.90704345703125,
"logps/rejected": -238.2965545654297,
"loss": 2085.5805,
"rewards/accuracies": 0.706250011920929,
"rewards/chosen": -0.09591428190469742,
"rewards/margins": 0.08266115188598633,
"rewards/rejected": -0.17857542634010315,
"step": 1900
},
{
"epoch": 0.99,
"eval_logits/chosen": 0.5747328996658325,
"eval_logits/rejected": 0.6318737864494324,
"eval_logps/chosen": -267.4730224609375,
"eval_logps/rejected": -252.61306762695312,
"eval_loss": 2052.32373046875,
"eval_rewards/accuracies": 0.6899999976158142,
"eval_rewards/chosen": -0.10855603218078613,
"eval_rewards/margins": 0.08208546042442322,
"eval_rewards/rejected": -0.19064147770404816,
"eval_runtime": 416.81,
"eval_samples_per_second": 4.798,
"eval_steps_per_second": 1.2,
"step": 1900
},
{
"epoch": 1.0,
"learning_rate": 2.9341204441673267e-06,
"logits/chosen": 0.5886205434799194,
"logits/rejected": 0.6063315272331238,
"logps/chosen": -281.51129150390625,
"logps/rejected": -263.0057678222656,
"loss": 1992.2906,
"rewards/accuracies": 0.699999988079071,
"rewards/chosen": -0.10820697247982025,
"rewards/margins": 0.08012684434652328,
"rewards/rejected": -0.18833380937576294,
"step": 1910
},
{
"epoch": 1.0,
"learning_rate": 2.9116050702407706e-06,
"logits/chosen": 0.6380060315132141,
"logits/rejected": 0.6841104030609131,
"logps/chosen": -267.55145263671875,
"logps/rejected": -248.828369140625,
"loss": 2045.3801,
"rewards/accuracies": 0.6604167222976685,
"rewards/chosen": -0.10555033385753632,
"rewards/margins": 0.08623509109020233,
"rewards/rejected": -0.19178542494773865,
"step": 1920
},
{
"epoch": 1.01,
"learning_rate": 2.889055327409301e-06,
"logits/chosen": 0.5285671353340149,
"logits/rejected": 0.5704804062843323,
"logps/chosen": -263.24725341796875,
"logps/rejected": -248.02395629882812,
"loss": 2023.134,
"rewards/accuracies": 0.7124999761581421,
"rewards/chosen": -0.10273389518260956,
"rewards/margins": 0.08054333180189133,
"rewards/rejected": -0.1832772046327591,
"step": 1930
},
{
"epoch": 1.02,
"learning_rate": 2.8664730985699537e-06,
"logits/chosen": 0.5331718325614929,
"logits/rejected": 0.5988043546676636,
"logps/chosen": -242.79061889648438,
"logps/rejected": -238.0269012451172,
"loss": 1960.2693,
"rewards/accuracies": 0.75,
"rewards/chosen": -0.09900447726249695,
"rewards/margins": 0.0868852287530899,
"rewards/rejected": -0.18588972091674805,
"step": 1940
},
{
"epoch": 1.02,
"learning_rate": 2.843860269332339e-06,
"logits/chosen": 0.6072074174880981,
"logits/rejected": 0.631058394908905,
"logps/chosen": -273.4151306152344,
"logps/rejected": -246.41238403320312,
"loss": 1955.907,
"rewards/accuracies": 0.7250000238418579,
"rewards/chosen": -0.09934862703084946,
"rewards/margins": 0.09240168333053589,
"rewards/rejected": -0.19175033271312714,
"step": 1950
},
{
"epoch": 1.03,
"learning_rate": 2.8212187278611907e-06,
"logits/chosen": 0.6683967113494873,
"logits/rejected": 0.6856907606124878,
"logps/chosen": -257.2086181640625,
"logps/rejected": -247.91683959960938,
"loss": 1924.8703,
"rewards/accuracies": 0.706250011920929,
"rewards/chosen": -0.1071348562836647,
"rewards/margins": 0.09893598407506943,
"rewards/rejected": -0.20607082545757294,
"step": 1960
},
{
"epoch": 1.03,
"learning_rate": 2.7985503647187063e-06,
"logits/chosen": 0.5825555920600891,
"logits/rejected": 0.6476297378540039,
"logps/chosen": -288.1867980957031,
"logps/rejected": -254.5727996826172,
"loss": 1907.2662,
"rewards/accuracies": 0.706250011920929,
"rewards/chosen": -0.08619461953639984,
"rewards/margins": 0.10769355297088623,
"rewards/rejected": -0.19388815760612488,
"step": 1970
},
{
"epoch": 1.04,
"learning_rate": 2.7758570727066843e-06,
"logits/chosen": 0.5205335021018982,
"logits/rejected": 0.6433119177818298,
"logps/chosen": -261.37982177734375,
"logps/rejected": -240.10952758789062,
"loss": 2241.4863,
"rewards/accuracies": 0.65625,
"rewards/chosen": -0.12351379543542862,
"rewards/margins": 0.05891970917582512,
"rewards/rejected": -0.18243351578712463,
"step": 1980
},
{
"epoch": 1.04,
"learning_rate": 2.753140746708477e-06,
"logits/chosen": 0.6216637492179871,
"logits/rejected": 0.669810950756073,
"logps/chosen": -282.500244140625,
"logps/rejected": -273.12310791015625,
"loss": 1961.7119,
"rewards/accuracies": 0.65625,
"rewards/chosen": -0.09472217410802841,
"rewards/margins": 0.09439438581466675,
"rewards/rejected": -0.18911656737327576,
"step": 1990
},
{
"epoch": 1.05,
"learning_rate": 2.730403283530767e-06,
"logits/chosen": 0.638060986995697,
"logits/rejected": 0.7034865617752075,
"logps/chosen": -258.02447509765625,
"logps/rejected": -246.9062957763672,
"loss": 1847.759,
"rewards/accuracies": 0.75,
"rewards/chosen": -0.09892908483743668,
"rewards/margins": 0.10315445810556412,
"rewards/rejected": -0.2020835429430008,
"step": 2000
},
{
"epoch": 1.05,
"eval_logits/chosen": 0.5763067603111267,
"eval_logits/rejected": 0.6332587599754333,
"eval_logps/chosen": -267.7949523925781,
"eval_logps/rejected": -253.0826873779297,
"eval_loss": 2050.417724609375,
"eval_rewards/accuracies": 0.6850000023841858,
"eval_rewards/chosen": -0.11177488416433334,
"eval_rewards/margins": 0.0835629403591156,
"eval_rewards/rejected": -0.19533783197402954,
"eval_runtime": 416.585,
"eval_samples_per_second": 4.801,
"eval_steps_per_second": 1.2,
"step": 2000
},
{
"epoch": 1.05,
"learning_rate": 2.707646581745188e-06,
"logits/chosen": 0.6024230718612671,
"logits/rejected": 0.6550949811935425,
"logps/chosen": -275.99761962890625,
"logps/rejected": -272.4120178222656,
"loss": 1981.8529,
"rewards/accuracies": 0.6937500238418579,
"rewards/chosen": -0.0975821316242218,
"rewards/margins": 0.08785782754421234,
"rewards/rejected": -0.18543997406959534,
"step": 2010
},
{
"epoch": 1.06,
"learning_rate": 2.6848725415297888e-06,
"logits/chosen": 0.629960298538208,
"logits/rejected": 0.6249616742134094,
"logps/chosen": -256.87603759765625,
"logps/rejected": -253.21109008789062,
"loss": 2136.5217,
"rewards/accuracies": 0.6499999761581421,
"rewards/chosen": -0.10265711694955826,
"rewards/margins": 0.0642293393611908,
"rewards/rejected": -0.16688646376132965,
"step": 2020
},
{
"epoch": 1.06,
"learning_rate": 2.6620830645103753e-06,
"logits/chosen": 0.6109344959259033,
"logits/rejected": 0.6072026491165161,
"logps/chosen": -266.4075012207031,
"logps/rejected": -258.20208740234375,
"loss": 1938.8361,
"rewards/accuracies": 0.71875,
"rewards/chosen": -0.09916529804468155,
"rewards/margins": 0.09021677076816559,
"rewards/rejected": -0.18938204646110535,
"step": 2030
},
{
"epoch": 1.07,
"learning_rate": 2.639280053601719e-06,
"logits/chosen": 0.566746711730957,
"logits/rejected": 0.6063026189804077,
"logps/chosen": -261.76739501953125,
"logps/rejected": -270.6283874511719,
"loss": 2085.5938,
"rewards/accuracies": 0.65625,
"rewards/chosen": -0.12018795311450958,
"rewards/margins": 0.08117975294589996,
"rewards/rejected": -0.20136770606040955,
"step": 2040
},
{
"epoch": 1.07,
"learning_rate": 2.6164654128486683e-06,
"logits/chosen": 0.5058253407478333,
"logits/rejected": 0.6028685569763184,
"logps/chosen": -267.67376708984375,
"logps/rejected": -230.2966766357422,
"loss": 2055.1498,
"rewards/accuracies": 0.6499999761581421,
"rewards/chosen": -0.11003967374563217,
"rewards/margins": 0.08447955548763275,
"rewards/rejected": -0.19451923668384552,
"step": 2050
},
{
"epoch": 1.08,
"learning_rate": 2.59364104726716e-06,
"logits/chosen": 0.5947778820991516,
"logits/rejected": 0.5901384353637695,
"logps/chosen": -278.0001525878906,
"logps/rejected": -242.13583374023438,
"loss": 1877.0174,
"rewards/accuracies": 0.699999988079071,
"rewards/chosen": -0.11745290458202362,
"rewards/margins": 0.10013137012720108,
"rewards/rejected": -0.2175842821598053,
"step": 2060
},
{
"epoch": 1.08,
"learning_rate": 2.5708088626851546e-06,
"logits/chosen": 0.5502884387969971,
"logits/rejected": 0.603992760181427,
"logps/chosen": -269.38360595703125,
"logps/rejected": -244.87619018554688,
"loss": 2015.0283,
"rewards/accuracies": 0.637499988079071,
"rewards/chosen": -0.12011172622442245,
"rewards/margins": 0.08565986156463623,
"rewards/rejected": -0.20577159523963928,
"step": 2070
},
{
"epoch": 1.09,
"learning_rate": 2.547970765583491e-06,
"logits/chosen": 0.5619412064552307,
"logits/rejected": 0.6468341946601868,
"logps/chosen": -252.68115234375,
"logps/rejected": -252.705322265625,
"loss": 1855.9893,
"rewards/accuracies": 0.7250000238418579,
"rewards/chosen": -0.10235454142093658,
"rewards/margins": 0.10716482251882553,
"rewards/rejected": -0.20951935648918152,
"step": 2080
},
{
"epoch": 1.09,
"learning_rate": 2.525128662936707e-06,
"logits/chosen": 0.512058436870575,
"logits/rejected": 0.5677643418312073,
"logps/chosen": -270.7825012207031,
"logps/rejected": -260.822509765625,
"loss": 1831.3346,
"rewards/accuracies": 0.7562500238418579,
"rewards/chosen": -0.1064097136259079,
"rewards/margins": 0.10622493177652359,
"rewards/rejected": -0.21263465285301208,
"step": 2090
},
{
"epoch": 1.1,
"learning_rate": 2.502284462053799e-06,
"logits/chosen": 0.620409369468689,
"logits/rejected": 0.6358670592308044,
"logps/chosen": -258.42706298828125,
"logps/rejected": -258.72161865234375,
"loss": 2024.9559,
"rewards/accuracies": 0.6875,
"rewards/chosen": -0.10622759163379669,
"rewards/margins": 0.08639432489871979,
"rewards/rejected": -0.19262190163135529,
"step": 2100
},
{
"epoch": 1.1,
"eval_logits/chosen": 0.5589507818222046,
"eval_logits/rejected": 0.6156801581382751,
"eval_logps/chosen": -268.8072814941406,
"eval_logps/rejected": -254.37991333007812,
"eval_loss": 2046.75927734375,
"eval_rewards/accuracies": 0.6899999976158142,
"eval_rewards/chosen": -0.12189868092536926,
"eval_rewards/margins": 0.08641137927770615,
"eval_rewards/rejected": -0.2083100527524948,
"eval_runtime": 416.7259,
"eval_samples_per_second": 4.799,
"eval_steps_per_second": 1.2,
"step": 2100
},
{
"epoch": 1.1,
"learning_rate": 2.479440070418967e-06,
"logits/chosen": 0.5901846885681152,
"logits/rejected": 0.6195170283317566,
"logps/chosen": -249.45816040039062,
"logps/rejected": -253.7944793701172,
"loss": 2205.6043,
"rewards/accuracies": 0.6625000238418579,
"rewards/chosen": -0.13660283386707306,
"rewards/margins": 0.06599629670381546,
"rewards/rejected": -0.20259912312030792,
"step": 2110
},
{
"epoch": 1.11,
"learning_rate": 2.456597395532338e-06,
"logits/chosen": 0.5504690408706665,
"logits/rejected": 0.6531665921211243,
"logps/chosen": -259.79010009765625,
"logps/rejected": -284.0751647949219,
"loss": 1959.0818,
"rewards/accuracies": 0.731249988079071,
"rewards/chosen": -0.11810295283794403,
"rewards/margins": 0.0964241549372673,
"rewards/rejected": -0.21452713012695312,
"step": 2120
},
{
"epoch": 1.11,
"learning_rate": 2.433758344750691e-06,
"logits/chosen": 0.5741318464279175,
"logits/rejected": 0.6458116173744202,
"logps/chosen": -295.03192138671875,
"logps/rejected": -276.64251708984375,
"loss": 1911.3146,
"rewards/accuracies": 0.7437499761581421,
"rewards/chosen": -0.11703141778707504,
"rewards/margins": 0.10501817613840103,
"rewards/rejected": -0.2220495641231537,
"step": 2130
},
{
"epoch": 1.12,
"learning_rate": 2.4109248251281953e-06,
"logits/chosen": 0.5908122062683105,
"logits/rejected": 0.6558480858802795,
"logps/chosen": -283.0213928222656,
"logps/rejected": -252.56600952148438,
"loss": 1911.8793,
"rewards/accuracies": 0.699999988079071,
"rewards/chosen": -0.11346153914928436,
"rewards/margins": 0.10086224228143692,
"rewards/rejected": -0.2143237590789795,
"step": 2140
},
{
"epoch": 1.13,
"learning_rate": 2.3880987432571675e-06,
"logits/chosen": 0.5616129040718079,
"logits/rejected": 0.593204915523529,
"logps/chosen": -268.16583251953125,
"logps/rejected": -262.17755126953125,
"loss": 1994.0697,
"rewards/accuracies": 0.706250011920929,
"rewards/chosen": -0.11974780261516571,
"rewards/margins": 0.09457085281610489,
"rewards/rejected": -0.2143186628818512,
"step": 2150
},
{
"epoch": 1.13,
"learning_rate": 2.365282005108875e-06,
"logits/chosen": 0.5762392282485962,
"logits/rejected": 0.615722119808197,
"logps/chosen": -250.62509155273438,
"logps/rejected": -253.91049194335938,
"loss": 2099.1633,
"rewards/accuracies": 0.6312500238418579,
"rewards/chosen": -0.1276218444108963,
"rewards/margins": 0.07122951745986938,
"rewards/rejected": -0.19885137677192688,
"step": 2160
},
{
"epoch": 1.14,
"learning_rate": 2.3424765158743867e-06,
"logits/chosen": 0.6059794425964355,
"logits/rejected": 0.6645799875259399,
"logps/chosen": -255.7693634033203,
"logps/rejected": -251.96951293945312,
"loss": 2010.217,
"rewards/accuracies": 0.699999988079071,
"rewards/chosen": -0.12624487280845642,
"rewards/margins": 0.09581606835126877,
"rewards/rejected": -0.2220609486103058,
"step": 2170
},
{
"epoch": 1.14,
"learning_rate": 2.319684179805491e-06,
"logits/chosen": 0.516992449760437,
"logits/rejected": 0.5456847548484802,
"logps/chosen": -265.10321044921875,
"logps/rejected": -246.3701629638672,
"loss": 1933.4984,
"rewards/accuracies": 0.7124999761581421,
"rewards/chosen": -0.11627723276615143,
"rewards/margins": 0.0940864160656929,
"rewards/rejected": -0.21036362648010254,
"step": 2180
},
{
"epoch": 1.15,
"learning_rate": 2.296906900055691e-06,
"logits/chosen": 0.596808135509491,
"logits/rejected": 0.6393652558326721,
"logps/chosen": -264.455810546875,
"logps/rejected": -256.40667724609375,
"loss": 2172.6984,
"rewards/accuracies": 0.637499988079071,
"rewards/chosen": -0.12352782487869263,
"rewards/margins": 0.07605434954166412,
"rewards/rejected": -0.19958215951919556,
"step": 2190
},
{
"epoch": 1.15,
"learning_rate": 2.2741465785212905e-06,
"logits/chosen": 0.5940336585044861,
"logits/rejected": 0.6305769085884094,
"logps/chosen": -256.6434326171875,
"logps/rejected": -245.391357421875,
"loss": 2038.6354,
"rewards/accuracies": 0.675000011920929,
"rewards/chosen": -0.11867289245128632,
"rewards/margins": 0.09147666394710541,
"rewards/rejected": -0.21014957129955292,
"step": 2200
},
{
"epoch": 1.15,
"eval_logits/chosen": 0.5517618656158447,
"eval_logits/rejected": 0.6082795858383179,
"eval_logps/chosen": -268.6722106933594,
"eval_logps/rejected": -254.27310180664062,
"eval_loss": 2043.57275390625,
"eval_rewards/accuracies": 0.6880000233650208,
"eval_rewards/chosen": -0.12054779380559921,
"eval_rewards/margins": 0.08669425547122955,
"eval_rewards/rejected": -0.20724207162857056,
"eval_runtime": 416.7766,
"eval_samples_per_second": 4.799,
"eval_steps_per_second": 1.2,
"step": 2200
},
{
"epoch": 1.16,
"learning_rate": 2.251405115682587e-06,
"logits/chosen": 0.5902246236801147,
"logits/rejected": 0.5983418822288513,
"logps/chosen": -263.2071228027344,
"logps/rejected": -272.0802307128906,
"loss": 2017.8775,
"rewards/accuracies": 0.6937500238418579,
"rewards/chosen": -0.11642640829086304,
"rewards/margins": 0.09181423485279083,
"rewards/rejected": -0.20824062824249268,
"step": 2210
},
{
"epoch": 1.16,
"learning_rate": 2.2286844104451848e-06,
"logits/chosen": 0.5431746244430542,
"logits/rejected": 0.6418278217315674,
"logps/chosen": -264.33465576171875,
"logps/rejected": -251.9208984375,
"loss": 2095.6342,
"rewards/accuracies": 0.668749988079071,
"rewards/chosen": -0.10533900558948517,
"rewards/margins": 0.07776842266321182,
"rewards/rejected": -0.1831074208021164,
"step": 2220
},
{
"epoch": 1.17,
"learning_rate": 2.205986359981431e-06,
"logits/chosen": 0.5207514762878418,
"logits/rejected": 0.6270573139190674,
"logps/chosen": -285.76849365234375,
"logps/rejected": -277.93426513671875,
"loss": 1912.0979,
"rewards/accuracies": 0.7124999761581421,
"rewards/chosen": -0.11343447118997574,
"rewards/margins": 0.1047770231962204,
"rewards/rejected": -0.21821150183677673,
"step": 2230
},
{
"epoch": 1.17,
"learning_rate": 2.183312859572008e-06,
"logits/chosen": 0.5806037783622742,
"logits/rejected": 0.6543610095977783,
"logps/chosen": -281.1925354003906,
"logps/rejected": -278.2233581542969,
"loss": 2272.857,
"rewards/accuracies": 0.6000000238418579,
"rewards/chosen": -0.11976996809244156,
"rewards/margins": 0.06387045979499817,
"rewards/rejected": -0.18364043533802032,
"step": 2240
},
{
"epoch": 1.18,
"learning_rate": 2.1606658024476744e-06,
"logits/chosen": 0.5554038286209106,
"logits/rejected": 0.5429580211639404,
"logps/chosen": -269.9796447753906,
"logps/rejected": -250.5005340576172,
"loss": 2188.1607,
"rewards/accuracies": 0.6812499761581421,
"rewards/chosen": -0.1146581768989563,
"rewards/margins": 0.07278671860694885,
"rewards/rejected": -0.18744489550590515,
"step": 2250
},
{
"epoch": 1.18,
"learning_rate": 2.1380470796311843e-06,
"logits/chosen": 0.610127866268158,
"logits/rejected": 0.6246207356452942,
"logps/chosen": -274.421142578125,
"logps/rejected": -259.88336181640625,
"loss": 1878.2621,
"rewards/accuracies": 0.737500011920929,
"rewards/chosen": -0.10982956737279892,
"rewards/margins": 0.10056765377521515,
"rewards/rejected": -0.21039721369743347,
"step": 2260
},
{
"epoch": 1.19,
"learning_rate": 2.1154585797793826e-06,
"logits/chosen": 0.6410681009292603,
"logits/rejected": 0.6446506977081299,
"logps/chosen": -262.7099304199219,
"logps/rejected": -243.33847045898438,
"loss": 1969.0021,
"rewards/accuracies": 0.6812499761581421,
"rewards/chosen": -0.10027629137039185,
"rewards/margins": 0.08689162135124207,
"rewards/rejected": -0.1871679127216339,
"step": 2270
},
{
"epoch": 1.19,
"learning_rate": 2.092902189025507e-06,
"logits/chosen": 0.6191312670707703,
"logits/rejected": 0.6812275648117065,
"logps/chosen": -258.3789978027344,
"logps/rejected": -247.3417510986328,
"loss": 1765.025,
"rewards/accuracies": 0.71875,
"rewards/chosen": -0.10864460468292236,
"rewards/margins": 0.11250102519989014,
"rewards/rejected": -0.2211456298828125,
"step": 2280
},
{
"epoch": 1.2,
"learning_rate": 2.070379790821693e-06,
"logits/chosen": 0.5654376745223999,
"logits/rejected": 0.6462022066116333,
"logps/chosen": -301.2412414550781,
"logps/rejected": -276.8460998535156,
"loss": 2045.4492,
"rewards/accuracies": 0.7250000238418579,
"rewards/chosen": -0.10371474176645279,
"rewards/margins": 0.09003494679927826,
"rewards/rejected": -0.19374969601631165,
"step": 2290
},
{
"epoch": 1.2,
"learning_rate": 2.0478932657817105e-06,
"logits/chosen": 0.5810787081718445,
"logits/rejected": 0.6360457539558411,
"logps/chosen": -254.06838989257812,
"logps/rejected": -243.23989868164062,
"loss": 2022.9617,
"rewards/accuracies": 0.699999988079071,
"rewards/chosen": -0.11232425272464752,
"rewards/margins": 0.07984773069620132,
"rewards/rejected": -0.19217197597026825,
"step": 2300
},
{
"epoch": 1.2,
"eval_logits/chosen": 0.5535383820533752,
"eval_logits/rejected": 0.6101322174072266,
"eval_logps/chosen": -268.3490905761719,
"eval_logps/rejected": -253.95965576171875,
"eval_loss": 2035.585693359375,
"eval_rewards/accuracies": 0.6894999742507935,
"eval_rewards/chosen": -0.11731643229722977,
"eval_rewards/margins": 0.08679118007421494,
"eval_rewards/rejected": -0.2041076123714447,
"eval_runtime": 416.4094,
"eval_samples_per_second": 4.803,
"eval_steps_per_second": 1.201,
"step": 2300
},
{
"epoch": 1.21,
"learning_rate": 2.0254444915239287e-06,
"logits/chosen": 0.5468884706497192,
"logits/rejected": 0.5753307938575745,
"logps/chosen": -271.94940185546875,
"logps/rejected": -244.8318328857422,
"loss": 1994.7408,
"rewards/accuracies": 0.7124999761581421,
"rewards/chosen": -0.1074294000864029,
"rewards/margins": 0.08415937423706055,
"rewards/rejected": -0.19158877432346344,
"step": 2310
},
{
"epoch": 1.21,
"learning_rate": 2.0030353425145376e-06,
"logits/chosen": 0.6623051762580872,
"logits/rejected": 0.6782322525978088,
"logps/chosen": -220.7380828857422,
"logps/rejected": -242.05850219726562,
"loss": 1864.7361,
"rewards/accuracies": 0.7124999761581421,
"rewards/chosen": -0.10245666652917862,
"rewards/margins": 0.10949740558862686,
"rewards/rejected": -0.2119540423154831,
"step": 2320
},
{
"epoch": 1.22,
"learning_rate": 1.9806676899110305e-06,
"logits/chosen": 0.6308891773223877,
"logits/rejected": 0.6477428674697876,
"logps/chosen": -262.88897705078125,
"logps/rejected": -255.49362182617188,
"loss": 1843.8875,
"rewards/accuracies": 0.7749999761581421,
"rewards/chosen": -0.10279623419046402,
"rewards/margins": 0.11021213233470917,
"rewards/rejected": -0.21300837397575378,
"step": 2330
},
{
"epoch": 1.22,
"learning_rate": 1.958343401405964e-06,
"logits/chosen": 0.5211482048034668,
"logits/rejected": 0.6105703115463257,
"logps/chosen": -272.09698486328125,
"logps/rejected": -240.05392456054688,
"loss": 1991.6119,
"rewards/accuracies": 0.6937500238418579,
"rewards/chosen": -0.12355583906173706,
"rewards/margins": 0.08727772533893585,
"rewards/rejected": -0.2108335793018341,
"step": 2340
},
{
"epoch": 1.23,
"learning_rate": 1.9360643410710027e-06,
"logits/chosen": 0.6230972409248352,
"logits/rejected": 0.6428076028823853,
"logps/chosen": -297.76300048828125,
"logps/rejected": -262.3421325683594,
"loss": 2047.1437,
"rewards/accuracies": 0.7124999761581421,
"rewards/chosen": -0.10469541698694229,
"rewards/margins": 0.09010159224271774,
"rewards/rejected": -0.19479700922966003,
"step": 2350
},
{
"epoch": 1.24,
"learning_rate": 1.9138323692012734e-06,
"logits/chosen": 0.5903237462043762,
"logits/rejected": 0.6455060243606567,
"logps/chosen": -288.16815185546875,
"logps/rejected": -289.3240966796875,
"loss": 1579.4779,
"rewards/accuracies": 0.831250011920929,
"rewards/chosen": -0.08982647955417633,
"rewards/margins": 0.14532434940338135,
"rewards/rejected": -0.23515084385871887,
"step": 2360
},
{
"epoch": 1.24,
"learning_rate": 1.8916493421600287e-06,
"logits/chosen": 0.5603612065315247,
"logits/rejected": 0.5792626142501831,
"logps/chosen": -243.1230010986328,
"logps/rejected": -257.9002685546875,
"loss": 2017.2086,
"rewards/accuracies": 0.706250011920929,
"rewards/chosen": -0.12239034473896027,
"rewards/margins": 0.0815977230668068,
"rewards/rejected": -0.20398807525634766,
"step": 2370
},
{
"epoch": 1.25,
"learning_rate": 1.8695171122236443e-06,
"logits/chosen": 0.49118170142173767,
"logits/rejected": 0.5506534576416016,
"logps/chosen": -268.86822509765625,
"logps/rejected": -274.64202880859375,
"loss": 1977.4205,
"rewards/accuracies": 0.6937500238418579,
"rewards/chosen": -0.11895406246185303,
"rewards/margins": 0.10293842852115631,
"rewards/rejected": -0.22189247608184814,
"step": 2380
},
{
"epoch": 1.25,
"learning_rate": 1.84743752742695e-06,
"logits/chosen": 0.6215322613716125,
"logits/rejected": 0.6151038408279419,
"logps/chosen": -265.5515441894531,
"logps/rejected": -278.13177490234375,
"loss": 1889.5199,
"rewards/accuracies": 0.7437499761581421,
"rewards/chosen": -0.10073964297771454,
"rewards/margins": 0.11166242510080338,
"rewards/rejected": -0.2124020755290985,
"step": 2390
},
{
"epoch": 1.26,
"learning_rate": 1.8254124314089225e-06,
"logits/chosen": 0.6138418912887573,
"logits/rejected": 0.6189366579055786,
"logps/chosen": -263.2386169433594,
"logps/rejected": -239.09963989257812,
"loss": 1871.641,
"rewards/accuracies": 0.71875,
"rewards/chosen": -0.12337759882211685,
"rewards/margins": 0.09867466986179352,
"rewards/rejected": -0.22205229103565216,
"step": 2400
},
{
"epoch": 1.26,
"eval_logits/chosen": 0.5482152104377747,
"eval_logits/rejected": 0.6045916676521301,
"eval_logps/chosen": -268.51605224609375,
"eval_logps/rejected": -254.28311157226562,
"eval_loss": 2036.3372802734375,
"eval_rewards/accuracies": 0.6894999742507935,
"eval_rewards/chosen": -0.11898616701364517,
"eval_rewards/margins": 0.0883559137582779,
"eval_rewards/rejected": -0.20734207332134247,
"eval_runtime": 416.6781,
"eval_samples_per_second": 4.8,
"eval_steps_per_second": 1.2,
"step": 2400
},
{
"epoch": 1.26,
"learning_rate": 1.8034436632587394e-06,
"logits/chosen": 0.5728852152824402,
"logits/rejected": 0.6265703439712524,
"logps/chosen": -237.0697784423828,
"logps/rejected": -242.29367065429688,
"loss": 1969.1203,
"rewards/accuracies": 0.7124999761581421,
"rewards/chosen": -0.10528033971786499,
"rewards/margins": 0.0880698561668396,
"rewards/rejected": -0.1933501809835434,
"step": 2410
},
{
"epoch": 1.27,
"learning_rate": 1.781533057362221e-06,
"logits/chosen": 0.5749053359031677,
"logits/rejected": 0.6042163372039795,
"logps/chosen": -278.8114013671875,
"logps/rejected": -279.18695068359375,
"loss": 1906.6891,
"rewards/accuracies": 0.7250000238418579,
"rewards/chosen": -0.1059660091996193,
"rewards/margins": 0.10437663644552231,
"rewards/rejected": -0.210342675447464,
"step": 2420
},
{
"epoch": 1.27,
"learning_rate": 1.7596824432486537e-06,
"logits/chosen": 0.5984959602355957,
"logits/rejected": 0.6386197209358215,
"logps/chosen": -292.53143310546875,
"logps/rejected": -256.42620849609375,
"loss": 2003.0641,
"rewards/accuracies": 0.65625,
"rewards/chosen": -0.10370471328496933,
"rewards/margins": 0.09288345277309418,
"rewards/rejected": -0.1965881586074829,
"step": 2430
},
{
"epoch": 1.28,
"learning_rate": 1.7378936454380277e-06,
"logits/chosen": 0.5537322163581848,
"logits/rejected": 0.5942158102989197,
"logps/chosen": -246.1141815185547,
"logps/rejected": -253.85617065429688,
"loss": 2137.652,
"rewards/accuracies": 0.625,
"rewards/chosen": -0.13521219789981842,
"rewards/margins": 0.0802813172340393,
"rewards/rejected": -0.21549351513385773,
"step": 2440
},
{
"epoch": 1.28,
"learning_rate": 1.7161684832886893e-06,
"logits/chosen": 0.5406220555305481,
"logits/rejected": 0.540827751159668,
"logps/chosen": -242.9103546142578,
"logps/rejected": -247.41921997070312,
"loss": 2005.6266,
"rewards/accuracies": 0.7250000238418579,
"rewards/chosen": -0.12078257650136948,
"rewards/margins": 0.09046939015388489,
"rewards/rejected": -0.21125197410583496,
"step": 2450
},
{
"epoch": 1.29,
"learning_rate": 1.6945087708454273e-06,
"logits/chosen": 0.5730911493301392,
"logits/rejected": 0.5966663956642151,
"logps/chosen": -276.0887145996094,
"logps/rejected": -264.39910888671875,
"loss": 2211.1135,
"rewards/accuracies": 0.6499999761581421,
"rewards/chosen": -0.12395117431879044,
"rewards/margins": 0.07060922682285309,
"rewards/rejected": -0.19456037878990173,
"step": 2460
},
{
"epoch": 1.29,
"learning_rate": 1.6729163166879964e-06,
"logits/chosen": 0.5936635136604309,
"logits/rejected": 0.6355383396148682,
"logps/chosen": -258.3261413574219,
"logps/rejected": -233.99075317382812,
"loss": 1757.0482,
"rewards/accuracies": 0.75,
"rewards/chosen": -0.1076178103685379,
"rewards/margins": 0.11206640303134918,
"rewards/rejected": -0.21968421339988708,
"step": 2470
},
{
"epoch": 1.3,
"learning_rate": 1.651392923780105e-06,
"logits/chosen": 0.6025252342224121,
"logits/rejected": 0.6687902808189392,
"logps/chosen": -254.2070770263672,
"logps/rejected": -237.63967895507812,
"loss": 2055.3113,
"rewards/accuracies": 0.675000011920929,
"rewards/chosen": -0.1099240779876709,
"rewards/margins": 0.09202177077531815,
"rewards/rejected": -0.20194585621356964,
"step": 2480
},
{
"epoch": 1.3,
"learning_rate": 1.629940389318867e-06,
"logits/chosen": 0.5291022062301636,
"logits/rejected": 0.616036593914032,
"logps/chosen": -294.7336730957031,
"logps/rejected": -240.91796875,
"loss": 1902.4217,
"rewards/accuracies": 0.71875,
"rewards/chosen": -0.10780209302902222,
"rewards/margins": 0.10337891429662704,
"rewards/rejected": -0.21118099987506866,
"step": 2490
},
{
"epoch": 1.31,
"learning_rate": 1.608560504584737e-06,
"logits/chosen": 0.5608310103416443,
"logits/rejected": 0.6271076798439026,
"logps/chosen": -256.45770263671875,
"logps/rejected": -253.8623809814453,
"loss": 1907.3463,
"rewards/accuracies": 0.71875,
"rewards/chosen": -0.10689397901296616,
"rewards/margins": 0.10536620765924454,
"rewards/rejected": -0.2122601717710495,
"step": 2500
},
{
"epoch": 1.31,
"eval_logits/chosen": 0.5460030436515808,
"eval_logits/rejected": 0.6022311449050903,
"eval_logps/chosen": -268.7764587402344,
"eval_logps/rejected": -254.62974548339844,
"eval_loss": 2034.7010498046875,
"eval_rewards/accuracies": 0.6880000233650208,
"eval_rewards/chosen": -0.12159038335084915,
"eval_rewards/margins": 0.08921793848276138,
"eval_rewards/rejected": -0.21080833673477173,
"eval_runtime": 416.6626,
"eval_samples_per_second": 4.8,
"eval_steps_per_second": 1.2,
"step": 2500
},
{
"epoch": 1.31,
"learning_rate": 1.587255054791937e-06,
"logits/chosen": 0.5321905016899109,
"logits/rejected": 0.589474081993103,
"logps/chosen": -281.2105407714844,
"logps/rejected": -264.56298828125,
"loss": 2016.9854,
"rewards/accuracies": 0.6812499761581421,
"rewards/chosen": -0.10593251138925552,
"rewards/margins": 0.08659417182207108,
"rewards/rejected": -0.1925266534090042,
"step": 2510
},
{
"epoch": 1.32,
"learning_rate": 1.5660258189393945e-06,
"logits/chosen": 0.5880864262580872,
"logits/rejected": 0.6149991750717163,
"logps/chosen": -251.75973510742188,
"logps/rejected": -262.3134765625,
"loss": 2130.8975,
"rewards/accuracies": 0.675000011920929,
"rewards/chosen": -0.12219718843698502,
"rewards/margins": 0.08161304891109467,
"rewards/rejected": -0.2038102149963379,
"step": 2520
},
{
"epoch": 1.32,
"learning_rate": 1.5448745696621915e-06,
"logits/chosen": 0.5654980540275574,
"logits/rejected": 0.6478559970855713,
"logps/chosen": -272.79864501953125,
"logps/rejected": -258.56402587890625,
"loss": 2114.0654,
"rewards/accuracies": 0.65625,
"rewards/chosen": -0.11762849986553192,
"rewards/margins": 0.08065593242645264,
"rewards/rejected": -0.19828443229198456,
"step": 2530
},
{
"epoch": 1.33,
"learning_rate": 1.5238030730835578e-06,
"logits/chosen": 0.5662246942520142,
"logits/rejected": 0.6270356178283691,
"logps/chosen": -272.17449951171875,
"logps/rejected": -237.2474365234375,
"loss": 2106.9717,
"rewards/accuracies": 0.675000011920929,
"rewards/chosen": -0.11701546609401703,
"rewards/margins": 0.0775529146194458,
"rewards/rejected": -0.19456836581230164,
"step": 2540
},
{
"epoch": 1.33,
"learning_rate": 1.5028130886673936e-06,
"logits/chosen": 0.5928006172180176,
"logits/rejected": 0.641442060470581,
"logps/chosen": -263.0660705566406,
"logps/rejected": -252.6539306640625,
"loss": 2000.1971,
"rewards/accuracies": 0.6812499761581421,
"rewards/chosen": -0.111760713160038,
"rewards/margins": 0.0842764601111412,
"rewards/rejected": -0.1960371732711792,
"step": 2550
},
{
"epoch": 1.34,
"learning_rate": 1.4819063690713565e-06,
"logits/chosen": 0.5778559446334839,
"logits/rejected": 0.6045337915420532,
"logps/chosen": -284.07061767578125,
"logps/rejected": -270.4360656738281,
"loss": 1938.5168,
"rewards/accuracies": 0.675000011920929,
"rewards/chosen": -0.12265495210886002,
"rewards/margins": 0.09922391176223755,
"rewards/rejected": -0.22187885642051697,
"step": 2560
},
{
"epoch": 1.35,
"learning_rate": 1.4610846600005164e-06,
"logits/chosen": 0.6385133862495422,
"logits/rejected": 0.6164069175720215,
"logps/chosen": -291.48590087890625,
"logps/rejected": -241.2799072265625,
"loss": 2040.5221,
"rewards/accuracies": 0.6937500238418579,
"rewards/chosen": -0.11285148561000824,
"rewards/margins": 0.08863021433353424,
"rewards/rejected": -0.20148172974586487,
"step": 2570
},
{
"epoch": 1.35,
"learning_rate": 1.4403497000615885e-06,
"logits/chosen": 0.6022018194198608,
"logits/rejected": 0.6375949382781982,
"logps/chosen": -250.8367156982422,
"logps/rejected": -242.37881469726562,
"loss": 2072.085,
"rewards/accuracies": 0.6937500238418579,
"rewards/chosen": -0.11340691894292831,
"rewards/margins": 0.07300657033920288,
"rewards/rejected": -0.1864134818315506,
"step": 2580
},
{
"epoch": 1.36,
"learning_rate": 1.4197032206177618e-06,
"logits/chosen": 0.6561594605445862,
"logits/rejected": 0.7297431230545044,
"logps/chosen": -249.74887084960938,
"logps/rejected": -237.26779174804688,
"loss": 2067.224,
"rewards/accuracies": 0.668749988079071,
"rewards/chosen": -0.12197699397802353,
"rewards/margins": 0.08562152087688446,
"rewards/rejected": -0.2075985223054886,
"step": 2590
},
{
"epoch": 1.36,
"learning_rate": 1.3991469456441273e-06,
"logits/chosen": 0.6001744270324707,
"logits/rejected": 0.6553865075111389,
"logps/chosen": -252.2941436767578,
"logps/rejected": -243.48025512695312,
"loss": 1884.6086,
"rewards/accuracies": 0.737500011920929,
"rewards/chosen": -0.11671899259090424,
"rewards/margins": 0.10958864539861679,
"rewards/rejected": -0.22630766034126282,
"step": 2600
},
{
"epoch": 1.36,
"eval_logits/chosen": 0.545119047164917,
"eval_logits/rejected": 0.601308286190033,
"eval_logps/chosen": -268.77081298828125,
"eval_logps/rejected": -254.6013946533203,
"eval_loss": 2033.7977294921875,
"eval_rewards/accuracies": 0.6909999847412109,
"eval_rewards/chosen": -0.12153391540050507,
"eval_rewards/margins": 0.08899100124835968,
"eval_rewards/rejected": -0.21052493155002594,
"eval_runtime": 416.4661,
"eval_samples_per_second": 4.802,
"eval_steps_per_second": 1.201,
"step": 2600
},
{
"epoch": 1.37,
"learning_rate": 1.3786825915837299e-06,
"logits/chosen": 0.6044927835464478,
"logits/rejected": 0.608493447303772,
"logps/chosen": -268.0179138183594,
"logps/rejected": -251.26168823242188,
"loss": 1763.491,
"rewards/accuracies": 0.71875,
"rewards/chosen": -0.08965936303138733,
"rewards/margins": 0.12351739406585693,
"rewards/rejected": -0.21317675709724426,
"step": 2610
},
{
"epoch": 1.37,
"learning_rate": 1.3583118672042441e-06,
"logits/chosen": 0.5879210233688354,
"logits/rejected": 0.6254302263259888,
"logps/chosen": -268.9471740722656,
"logps/rejected": -246.9387664794922,
"loss": 1806.3043,
"rewards/accuracies": 0.731249988079071,
"rewards/chosen": -0.09920786321163177,
"rewards/margins": 0.1189170852303505,
"rewards/rejected": -0.21812494099140167,
"step": 2620
},
{
"epoch": 1.38,
"learning_rate": 1.3380364734552935e-06,
"logits/chosen": 0.6040158867835999,
"logits/rejected": 0.6454821825027466,
"logps/chosen": -239.55313110351562,
"logps/rejected": -252.41641235351562,
"loss": 1881.491,
"rewards/accuracies": 0.699999988079071,
"rewards/chosen": -0.11133377254009247,
"rewards/margins": 0.10438641160726547,
"rewards/rejected": -0.21572017669677734,
"step": 2630
},
{
"epoch": 1.38,
"learning_rate": 1.3178581033264218e-06,
"logits/chosen": 0.5422452688217163,
"logits/rejected": 0.5576962232589722,
"logps/chosen": -267.02020263671875,
"logps/rejected": -233.59628295898438,
"loss": 1922.1236,
"rewards/accuracies": 0.6937500238418579,
"rewards/chosen": -0.1054287701845169,
"rewards/margins": 0.10195982456207275,
"rewards/rejected": -0.20738859474658966,
"step": 2640
},
{
"epoch": 1.39,
"learning_rate": 1.2977784417057262e-06,
"logits/chosen": 0.5648713111877441,
"logits/rejected": 0.5970919132232666,
"logps/chosen": -266.724365234375,
"logps/rejected": -252.4540557861328,
"loss": 1788.8666,
"rewards/accuracies": 0.6937500238418579,
"rewards/chosen": -0.11531468480825424,
"rewards/margins": 0.11205389350652695,
"rewards/rejected": -0.22736859321594238,
"step": 2650
},
{
"epoch": 1.39,
"learning_rate": 1.2777991652391757e-06,
"logits/chosen": 0.5807424783706665,
"logits/rejected": 0.64664626121521,
"logps/chosen": -253.5209197998047,
"logps/rejected": -251.5238800048828,
"loss": 2035.5062,
"rewards/accuracies": 0.668749988079071,
"rewards/chosen": -0.12570129334926605,
"rewards/margins": 0.09053059667348862,
"rewards/rejected": -0.21623189747333527,
"step": 2660
},
{
"epoch": 1.4,
"learning_rate": 1.2579219421906049e-06,
"logits/chosen": 0.612740159034729,
"logits/rejected": 0.6295909285545349,
"logps/chosen": -275.30938720703125,
"logps/rejected": -246.85986328125,
"loss": 1918.4975,
"rewards/accuracies": 0.7437499761581421,
"rewards/chosen": -0.11256787925958633,
"rewards/margins": 0.1016065701842308,
"rewards/rejected": -0.21417441964149475,
"step": 2670
},
{
"epoch": 1.4,
"learning_rate": 1.2381484323024178e-06,
"logits/chosen": 0.5338586568832397,
"logits/rejected": 0.614523708820343,
"logps/chosen": -248.32406616210938,
"logps/rejected": -234.80862426757812,
"loss": 2030.9229,
"rewards/accuracies": 0.6875,
"rewards/chosen": -0.12608769536018372,
"rewards/margins": 0.09196772426366806,
"rewards/rejected": -0.21805541217327118,
"step": 2680
},
{
"epoch": 1.41,
"learning_rate": 1.2184802866569991e-06,
"logits/chosen": 0.5740771889686584,
"logits/rejected": 0.5626708269119263,
"logps/chosen": -256.43524169921875,
"logps/rejected": -254.8041534423828,
"loss": 1881.1102,
"rewards/accuracies": 0.706250011920929,
"rewards/chosen": -0.12152848392724991,
"rewards/margins": 0.1093037948012352,
"rewards/rejected": -0.2308322638273239,
"step": 2690
},
{
"epoch": 1.41,
"learning_rate": 1.1989191475388518e-06,
"logits/chosen": 0.5784533023834229,
"logits/rejected": 0.5800845623016357,
"logps/chosen": -261.47900390625,
"logps/rejected": -261.27618408203125,
"loss": 2034.9129,
"rewards/accuracies": 0.699999988079071,
"rewards/chosen": -0.12217319011688232,
"rewards/margins": 0.08460931479930878,
"rewards/rejected": -0.2067825049161911,
"step": 2700
},
{
"epoch": 1.41,
"eval_logits/chosen": 0.5425635576248169,
"eval_logits/rejected": 0.5986801385879517,
"eval_logps/chosen": -268.96331787109375,
"eval_logps/rejected": -254.94712829589844,
"eval_loss": 2032.544677734375,
"eval_rewards/accuracies": 0.6899999976158142,
"eval_rewards/chosen": -0.12345867604017258,
"eval_rewards/margins": 0.0905236080288887,
"eval_rewards/rejected": -0.21398229897022247,
"eval_runtime": 416.8138,
"eval_samples_per_second": 4.798,
"eval_steps_per_second": 1.2,
"step": 2700
},
{
"epoch": 1.42,
"learning_rate": 1.1794666482974617e-06,
"logits/chosen": 0.5704789161682129,
"logits/rejected": 0.6782268285751343,
"logps/chosen": -282.65875244140625,
"logps/rejected": -257.2977600097656,
"loss": 1989.7289,
"rewards/accuracies": 0.7124999761581421,
"rewards/chosen": -0.11434787511825562,
"rewards/margins": 0.0926680713891983,
"rewards/rejected": -0.20701594650745392,
"step": 2710
},
{
"epoch": 1.42,
"learning_rate": 1.160124413210918e-06,
"logits/chosen": 0.5337072014808655,
"logits/rejected": 0.5367878675460815,
"logps/chosen": -264.4056701660156,
"logps/rejected": -245.18301391601562,
"loss": 1913.7168,
"rewards/accuracies": 0.6937500238418579,
"rewards/chosen": -0.10953982174396515,
"rewards/margins": 0.10163428634405136,
"rewards/rejected": -0.2111741304397583,
"step": 2720
},
{
"epoch": 1.43,
"learning_rate": 1.1408940573502838e-06,
"logits/chosen": 0.5485426783561707,
"logits/rejected": 0.6499109864234924,
"logps/chosen": -264.18505859375,
"logps/rejected": -238.2395477294922,
"loss": 1907.4213,
"rewards/accuracies": 0.6875,
"rewards/chosen": -0.12582936882972717,
"rewards/margins": 0.09968879073858261,
"rewards/rejected": -0.22551818192005157,
"step": 2730
},
{
"epoch": 1.43,
"learning_rate": 1.1217771864447396e-06,
"logits/chosen": 0.5939881205558777,
"logits/rejected": 0.6111994981765747,
"logps/chosen": -261.0160827636719,
"logps/rejected": -244.2744140625,
"loss": 2013.2748,
"rewards/accuracies": 0.6812499761581421,
"rewards/chosen": -0.12591084837913513,
"rewards/margins": 0.08752218633890152,
"rewards/rejected": -0.21343302726745605,
"step": 2740
},
{
"epoch": 1.44,
"learning_rate": 1.1027753967475046e-06,
"logits/chosen": 0.5890164375305176,
"logits/rejected": 0.6029259562492371,
"logps/chosen": -259.98382568359375,
"logps/rejected": -254.6918487548828,
"loss": 1947.2754,
"rewards/accuracies": 0.6937500238418579,
"rewards/chosen": -0.1243789941072464,
"rewards/margins": 0.09525910019874573,
"rewards/rejected": -0.21963807940483093,
"step": 2750
},
{
"epoch": 1.44,
"learning_rate": 1.08389027490255e-06,
"logits/chosen": 0.5902668833732605,
"logits/rejected": 0.6080381274223328,
"logps/chosen": -248.3684844970703,
"logps/rejected": -263.76776123046875,
"loss": 2085.1875,
"rewards/accuracies": 0.6812499761581421,
"rewards/chosen": -0.13539178669452667,
"rewards/margins": 0.07983563095331192,
"rewards/rejected": -0.215227410197258,
"step": 2760
},
{
"epoch": 1.45,
"learning_rate": 1.0651233978121145e-06,
"logits/chosen": 0.5521366596221924,
"logits/rejected": 0.5906900763511658,
"logps/chosen": -300.05230712890625,
"logps/rejected": -272.1240234375,
"loss": 1883.4229,
"rewards/accuracies": 0.7437499761581421,
"rewards/chosen": -0.11396439373493195,
"rewards/margins": 0.09765832126140594,
"rewards/rejected": -0.2116227149963379,
"step": 2770
},
{
"epoch": 1.46,
"learning_rate": 1.046476332505036e-06,
"logits/chosen": 0.6347781419754028,
"logits/rejected": 0.6657929420471191,
"logps/chosen": -250.8879852294922,
"logps/rejected": -223.33255004882812,
"loss": 1819.1854,
"rewards/accuracies": 0.75,
"rewards/chosen": -0.11131460964679718,
"rewards/margins": 0.11518070846796036,
"rewards/rejected": -0.22649531066417694,
"step": 2780
},
{
"epoch": 1.46,
"learning_rate": 1.0279506360059005e-06,
"logits/chosen": 0.5551185011863708,
"logits/rejected": 0.5792326331138611,
"logps/chosen": -262.4249572753906,
"logps/rejected": -267.63763427734375,
"loss": 2209.3523,
"rewards/accuracies": 0.612500011920929,
"rewards/chosen": -0.12959793210029602,
"rewards/margins": 0.06600113213062286,
"rewards/rejected": -0.19559906423091888,
"step": 2790
},
{
"epoch": 1.47,
"learning_rate": 1.0095478552050348e-06,
"logits/chosen": 0.6019959449768066,
"logits/rejected": 0.6074908971786499,
"logps/chosen": -273.5536193847656,
"logps/rejected": -267.5385437011719,
"loss": 2068.2822,
"rewards/accuracies": 0.668749988079071,
"rewards/chosen": -0.13250732421875,
"rewards/margins": 0.09504042565822601,
"rewards/rejected": -0.2275477647781372,
"step": 2800
},
{
"epoch": 1.47,
"eval_logits/chosen": 0.5382584929466248,
"eval_logits/rejected": 0.5942660570144653,
"eval_logps/chosen": -269.1269836425781,
"eval_logps/rejected": -255.16705322265625,
"eval_loss": 2030.8697509765625,
"eval_rewards/accuracies": 0.6899999976158142,
"eval_rewards/chosen": -0.1250954419374466,
"eval_rewards/margins": 0.09108588099479675,
"eval_rewards/rejected": -0.21618132293224335,
"eval_runtime": 416.6389,
"eval_samples_per_second": 4.8,
"eval_steps_per_second": 1.2,
"step": 2800
},
{
"epoch": 1.47,
"learning_rate": 9.912695267293383e-07,
"logits/chosen": 0.5214653015136719,
"logits/rejected": 0.5876752734184265,
"logps/chosen": -265.47882080078125,
"logps/rejected": -239.1663360595703,
"loss": 1950.4564,
"rewards/accuracies": 0.699999988079071,
"rewards/chosen": -0.10924456268548965,
"rewards/margins": 0.09817437827587128,
"rewards/rejected": -0.20741891860961914,
"step": 2810
},
{
"epoch": 1.48,
"learning_rate": 9.731171768139808e-07,
"logits/chosen": 0.6136573553085327,
"logits/rejected": 0.6188865900039673,
"logps/chosen": -284.9826965332031,
"logps/rejected": -261.3671875,
"loss": 2210.9309,
"rewards/accuracies": 0.6812499761581421,
"rewards/chosen": -0.12047699838876724,
"rewards/margins": 0.06873573362827301,
"rewards/rejected": -0.18921272456645966,
"step": 2820
},
{
"epoch": 1.48,
"learning_rate": 9.550923211749557e-07,
"logits/chosen": 0.5326896905899048,
"logits/rejected": 0.5845073461532593,
"logps/chosen": -260.52069091796875,
"logps/rejected": -268.2504577636719,
"loss": 2028.4584,
"rewards/accuracies": 0.7124999761581421,
"rewards/chosen": -0.11890892684459686,
"rewards/margins": 0.08525559306144714,
"rewards/rejected": -0.2041645348072052,
"step": 2830
},
{
"epoch": 1.49,
"learning_rate": 9.371964648825221e-07,
"logits/chosen": 0.6162235736846924,
"logits/rejected": 0.5728213787078857,
"logps/chosen": -272.336181640625,
"logps/rejected": -252.33346557617188,
"loss": 1941.8066,
"rewards/accuracies": 0.737500011920929,
"rewards/chosen": -0.10710117965936661,
"rewards/margins": 0.10685823112726212,
"rewards/rejected": -0.21395941078662872,
"step": 2840
},
{
"epoch": 1.49,
"learning_rate": 9.194311022355279e-07,
"logits/chosen": 0.5015624761581421,
"logits/rejected": 0.5448901057243347,
"logps/chosen": -276.95538330078125,
"logps/rejected": -250.8726348876953,
"loss": 1832.3256,
"rewards/accuracies": 0.706250011920929,
"rewards/chosen": -0.10919035971164703,
"rewards/margins": 0.11640901863574982,
"rewards/rejected": -0.22559937834739685,
"step": 2850
},
{
"epoch": 1.5,
"learning_rate": 9.017977166366445e-07,
"logits/chosen": 0.5708821415901184,
"logits/rejected": 0.558485209941864,
"logps/chosen": -258.96807861328125,
"logps/rejected": -263.4228210449219,
"loss": 1947.8791,
"rewards/accuracies": 0.75,
"rewards/chosen": -0.11477123200893402,
"rewards/margins": 0.09229175001382828,
"rewards/rejected": -0.2070629894733429,
"step": 2860
},
{
"epoch": 1.5,
"learning_rate": 8.842977804684938e-07,
"logits/chosen": 0.5845485925674438,
"logits/rejected": 0.6778086423873901,
"logps/chosen": -245.46102905273438,
"logps/rejected": -233.6326141357422,
"loss": 2066.3828,
"rewards/accuracies": 0.668749988079071,
"rewards/chosen": -0.12042129039764404,
"rewards/margins": 0.08121231943368912,
"rewards/rejected": -0.20163361728191376,
"step": 2870
},
{
"epoch": 1.51,
"learning_rate": 8.669327549707096e-07,
"logits/chosen": 0.5501264929771423,
"logits/rejected": 0.6287878751754761,
"logps/chosen": -281.37371826171875,
"logps/rejected": -252.8254852294922,
"loss": 1868.0959,
"rewards/accuracies": 0.7250000238418579,
"rewards/chosen": -0.1148761659860611,
"rewards/margins": 0.1036820039153099,
"rewards/rejected": -0.2185581624507904,
"step": 2880
},
{
"epoch": 1.51,
"learning_rate": 8.497040901179232e-07,
"logits/chosen": 0.5025564432144165,
"logits/rejected": 0.5421415567398071,
"logps/chosen": -276.861572265625,
"logps/rejected": -267.47723388671875,
"loss": 1753.2416,
"rewards/accuracies": 0.762499988079071,
"rewards/chosen": -0.10830195993185043,
"rewards/margins": 0.12366169691085815,
"rewards/rejected": -0.2319636344909668,
"step": 2890
},
{
"epoch": 1.52,
"learning_rate": 8.326132244986932e-07,
"logits/chosen": 0.6039875745773315,
"logits/rejected": 0.6563787460327148,
"logps/chosen": -282.04266357421875,
"logps/rejected": -257.56622314453125,
"loss": 1977.4029,
"rewards/accuracies": 0.71875,
"rewards/chosen": -0.12028930336236954,
"rewards/margins": 0.09056351333856583,
"rewards/rejected": -0.21085281670093536,
"step": 2900
},
{
"epoch": 1.52,
"eval_logits/chosen": 0.5381389260292053,
"eval_logits/rejected": 0.5940784811973572,
"eval_logps/chosen": -269.125244140625,
"eval_logps/rejected": -255.16897583007812,
"eval_loss": 2030.603271484375,
"eval_rewards/accuracies": 0.6894999742507935,
"eval_rewards/chosen": -0.12507818639278412,
"eval_rewards/margins": 0.09112255275249481,
"eval_rewards/rejected": -0.21620073914527893,
"eval_runtime": 416.5425,
"eval_samples_per_second": 4.801,
"eval_steps_per_second": 1.2,
"step": 2900
},
{
"epoch": 1.52,
"learning_rate": 8.156615851953798e-07,
"logits/chosen": 0.559486448764801,
"logits/rejected": 0.5794366598129272,
"logps/chosen": -256.5633239746094,
"logps/rejected": -259.57696533203125,
"loss": 1948.7941,
"rewards/accuracies": 0.706250011920929,
"rewards/chosen": -0.10746540874242783,
"rewards/margins": 0.10354423522949219,
"rewards/rejected": -0.2110096514225006,
"step": 2910
},
{
"epoch": 1.53,
"learning_rate": 7.988505876649863e-07,
"logits/chosen": 0.6247807741165161,
"logits/rejected": 0.6010321974754333,
"logps/chosen": -271.8721923828125,
"logps/rejected": -256.86480712890625,
"loss": 2099.2482,
"rewards/accuracies": 0.699999988079071,
"rewards/chosen": -0.12578730285167694,
"rewards/margins": 0.084382563829422,
"rewards/rejected": -0.21016988158226013,
"step": 2920
},
{
"epoch": 1.53,
"learning_rate": 7.821816356209677e-07,
"logits/chosen": 0.5775936841964722,
"logits/rejected": 0.6070097088813782,
"logps/chosen": -272.50653076171875,
"logps/rejected": -251.46243286132812,
"loss": 2020.2645,
"rewards/accuracies": 0.6875,
"rewards/chosen": -0.10367625951766968,
"rewards/margins": 0.08927679061889648,
"rewards/rejected": -0.19295303523540497,
"step": 2930
},
{
"epoch": 1.54,
"learning_rate": 7.656561209160248e-07,
"logits/chosen": 0.521769642829895,
"logits/rejected": 0.5275167226791382,
"logps/chosen": -289.0915222167969,
"logps/rejected": -263.7314453125,
"loss": 1947.6189,
"rewards/accuracies": 0.706250011920929,
"rewards/chosen": -0.12388887256383896,
"rewards/margins": 0.1033380776643753,
"rewards/rejected": -0.22722692787647247,
"step": 2940
},
{
"epoch": 1.54,
"learning_rate": 7.492754234258794e-07,
"logits/chosen": 0.5926128625869751,
"logits/rejected": 0.6193209886550903,
"logps/chosen": -241.3407440185547,
"logps/rejected": -225.17724609375,
"loss": 1876.9375,
"rewards/accuracies": 0.7749999761581421,
"rewards/chosen": -0.10547138750553131,
"rewards/margins": 0.10478665679693222,
"rewards/rejected": -0.21025805175304413,
"step": 2950
},
{
"epoch": 1.55,
"learning_rate": 7.330409109340563e-07,
"logits/chosen": 0.5721119046211243,
"logits/rejected": 0.5718821287155151,
"logps/chosen": -267.39471435546875,
"logps/rejected": -244.5928192138672,
"loss": 2126.3432,
"rewards/accuracies": 0.643750011920929,
"rewards/chosen": -0.1287916600704193,
"rewards/margins": 0.07899868488311768,
"rewards/rejected": -0.2077903300523758,
"step": 2960
},
{
"epoch": 1.55,
"learning_rate": 7.169539390176769e-07,
"logits/chosen": 0.5741583704948425,
"logits/rejected": 0.5660156011581421,
"logps/chosen": -219.59640502929688,
"logps/rejected": -233.7797088623047,
"loss": 1845.3854,
"rewards/accuracies": 0.768750011920929,
"rewards/chosen": -0.12960752844810486,
"rewards/margins": 0.10263533890247345,
"rewards/rejected": -0.23224285244941711,
"step": 2970
},
{
"epoch": 1.56,
"learning_rate": 7.010158509342682e-07,
"logits/chosen": 0.5922077298164368,
"logits/rejected": 0.6388793587684631,
"logps/chosen": -258.38946533203125,
"logps/rejected": -236.8799285888672,
"loss": 1677.0656,
"rewards/accuracies": 0.762499988079071,
"rewards/chosen": -0.11209128051996231,
"rewards/margins": 0.13626167178153992,
"rewards/rejected": -0.24835292994976044,
"step": 2980
},
{
"epoch": 1.57,
"learning_rate": 6.852279775095976e-07,
"logits/chosen": 0.6180992722511292,
"logits/rejected": 0.6189014911651611,
"logps/chosen": -272.6584167480469,
"logps/rejected": -247.75033569335938,
"loss": 1925.8682,
"rewards/accuracies": 0.7437499761581421,
"rewards/chosen": -0.11567474901676178,
"rewards/margins": 0.09763548523187637,
"rewards/rejected": -0.21331021189689636,
"step": 2990
},
{
"epoch": 1.57,
"learning_rate": 6.695916370265529e-07,
"logits/chosen": 0.6014515161514282,
"logits/rejected": 0.5875986814498901,
"logps/chosen": -265.0668029785156,
"logps/rejected": -241.8825225830078,
"loss": 2110.2887,
"rewards/accuracies": 0.668749988079071,
"rewards/chosen": -0.1249975711107254,
"rewards/margins": 0.07782919704914093,
"rewards/rejected": -0.20282676815986633,
"step": 3000
},
{
"epoch": 1.57,
"eval_logits/chosen": 0.5348395109176636,
"eval_logits/rejected": 0.5908406972885132,
"eval_logps/chosen": -269.2049865722656,
"eval_logps/rejected": -255.2820587158203,
"eval_loss": 2030.5706787109375,
"eval_rewards/accuracies": 0.690500020980835,
"eval_rewards/chosen": -0.12587547302246094,
"eval_rewards/margins": 0.09145611524581909,
"eval_rewards/rejected": -0.21733158826828003,
"eval_runtime": 416.6652,
"eval_samples_per_second": 4.8,
"eval_steps_per_second": 1.2,
"step": 3000
},
{
"epoch": 1.58,
"learning_rate": 6.541081351150638e-07,
"logits/chosen": 0.5409640669822693,
"logits/rejected": 0.5331202149391174,
"logps/chosen": -279.83941650390625,
"logps/rejected": -291.9646301269531,
"loss": 2035.416,
"rewards/accuracies": 0.6937500238418579,
"rewards/chosen": -0.10843896865844727,
"rewards/margins": 0.09788022935390472,
"rewards/rejected": -0.2063191831111908,
"step": 3010
},
{
"epoch": 1.58,
"learning_rate": 6.387787646430854e-07,
"logits/chosen": 0.5321037769317627,
"logits/rejected": 0.5582699775695801,
"logps/chosen": -267.9813232421875,
"logps/rejected": -264.12567138671875,
"loss": 2006.4391,
"rewards/accuracies": 0.65625,
"rewards/chosen": -0.10174393653869629,
"rewards/margins": 0.08532971143722534,
"rewards/rejected": -0.18707364797592163,
"step": 3020
},
{
"epoch": 1.59,
"learning_rate": 6.2360480560864e-07,
"logits/chosen": 0.5698617696762085,
"logits/rejected": 0.5839768648147583,
"logps/chosen": -251.5703125,
"logps/rejected": -235.4780731201172,
"loss": 1821.7498,
"rewards/accuracies": 0.7749999761581421,
"rewards/chosen": -0.10450971126556396,
"rewards/margins": 0.10978861898183823,
"rewards/rejected": -0.2142982929944992,
"step": 3030
},
{
"epoch": 1.59,
"learning_rate": 6.085875250329401e-07,
"logits/chosen": 0.5382856726646423,
"logits/rejected": 0.6018794178962708,
"logps/chosen": -304.603271484375,
"logps/rejected": -263.90570068359375,
"loss": 1820.3352,
"rewards/accuracies": 0.6875,
"rewards/chosen": -0.10189330577850342,
"rewards/margins": 0.12546256184577942,
"rewards/rejected": -0.22735583782196045,
"step": 3040
},
{
"epoch": 1.6,
"learning_rate": 5.937281768545919e-07,
"logits/chosen": 0.600039005279541,
"logits/rejected": 0.5895189046859741,
"logps/chosen": -288.302734375,
"logps/rejected": -266.3108215332031,
"loss": 2142.2947,
"rewards/accuracies": 0.6875,
"rewards/chosen": -0.1374974250793457,
"rewards/margins": 0.08518020063638687,
"rewards/rejected": -0.22267761826515198,
"step": 3050
},
{
"epoch": 1.6,
"learning_rate": 5.79028001824894e-07,
"logits/chosen": 0.577072024345398,
"logits/rejected": 0.5782276391983032,
"logps/chosen": -258.71734619140625,
"logps/rejected": -252.88906860351562,
"loss": 2083.8391,
"rewards/accuracies": 0.706250011920929,
"rewards/chosen": -0.13044390082359314,
"rewards/margins": 0.09016549587249756,
"rewards/rejected": -0.22060942649841309,
"step": 3060
},
{
"epoch": 1.61,
"learning_rate": 5.644882274042285e-07,
"logits/chosen": 0.5784581303596497,
"logits/rejected": 0.5805580019950867,
"logps/chosen": -286.2455139160156,
"logps/rejected": -253.4708251953125,
"loss": 1979.6775,
"rewards/accuracies": 0.699999988079071,
"rewards/chosen": -0.11061519384384155,
"rewards/margins": 0.10562906414270401,
"rewards/rejected": -0.21624425053596497,
"step": 3070
},
{
"epoch": 1.61,
"learning_rate": 5.501100676595761e-07,
"logits/chosen": 0.5630078911781311,
"logits/rejected": 0.5682691335678101,
"logps/chosen": -267.6597900390625,
"logps/rejected": -250.3827667236328,
"loss": 1964.8426,
"rewards/accuracies": 0.6875,
"rewards/chosen": -0.1262790709733963,
"rewards/margins": 0.09482350945472717,
"rewards/rejected": -0.22110256552696228,
"step": 3080
},
{
"epoch": 1.62,
"learning_rate": 5.358947231631375e-07,
"logits/chosen": 0.534908652305603,
"logits/rejected": 0.5746644139289856,
"logps/chosen": -283.8402404785156,
"logps/rejected": -272.68670654296875,
"loss": 1792.1418,
"rewards/accuracies": 0.7749999761581421,
"rewards/chosen": -0.0998634472489357,
"rewards/margins": 0.1285194605588913,
"rewards/rejected": -0.2283829152584076,
"step": 3090
},
{
"epoch": 1.62,
"learning_rate": 5.218433808920884e-07,
"logits/chosen": 0.5141295194625854,
"logits/rejected": 0.5280352234840393,
"logps/chosen": -262.8772888183594,
"logps/rejected": -246.84671020507812,
"loss": 2068.2863,
"rewards/accuracies": 0.643750011920929,
"rewards/chosen": -0.10809364169836044,
"rewards/margins": 0.08412571996450424,
"rewards/rejected": -0.19221936166286469,
"step": 3100
},
{
"epoch": 1.62,
"eval_logits/chosen": 0.5356869697570801,
"eval_logits/rejected": 0.5913118720054626,
"eval_logps/chosen": -269.03900146484375,
"eval_logps/rejected": -255.10865783691406,
"eval_loss": 2029.4173583984375,
"eval_rewards/accuracies": 0.6934999823570251,
"eval_rewards/chosen": -0.12421557307243347,
"eval_rewards/margins": 0.09138190746307373,
"eval_rewards/rejected": -0.2155974805355072,
"eval_runtime": 416.645,
"eval_samples_per_second": 4.8,
"eval_steps_per_second": 1.2,
"step": 3100
},
{
"epoch": 1.63,
"learning_rate": 5.07957214129464e-07,
"logits/chosen": 0.6343733072280884,
"logits/rejected": 0.6377061605453491,
"logps/chosen": -230.1392059326172,
"logps/rejected": -217.2322540283203,
"loss": 2110.5152,
"rewards/accuracies": 0.643750011920929,
"rewards/chosen": -0.14483976364135742,
"rewards/margins": 0.082811638712883,
"rewards/rejected": -0.22765140235424042,
"step": 3110
},
{
"epoch": 1.63,
"learning_rate": 4.942373823661928e-07,
"logits/chosen": 0.5317670702934265,
"logits/rejected": 0.5477628707885742,
"logps/chosen": -253.5054931640625,
"logps/rejected": -295.16021728515625,
"loss": 2379.184,
"rewards/accuracies": 0.6312500238418579,
"rewards/chosen": -0.13306215405464172,
"rewards/margins": 0.05652584508061409,
"rewards/rejected": -0.18958799540996552,
"step": 3120
},
{
"epoch": 1.64,
"learning_rate": 4.806850312042782e-07,
"logits/chosen": 0.6451593637466431,
"logits/rejected": 0.5899637937545776,
"logps/chosen": -289.49151611328125,
"logps/rejected": -257.98443603515625,
"loss": 1992.5604,
"rewards/accuracies": 0.668749988079071,
"rewards/chosen": -0.12486691772937775,
"rewards/margins": 0.09148009121417999,
"rewards/rejected": -0.21634697914123535,
"step": 3130
},
{
"epoch": 1.64,
"learning_rate": 4.6730129226114363e-07,
"logits/chosen": 0.5886529684066772,
"logits/rejected": 0.5409609079360962,
"logps/chosen": -258.1368103027344,
"logps/rejected": -249.2908935546875,
"loss": 1995.9428,
"rewards/accuracies": 0.699999988079071,
"rewards/chosen": -0.11228666454553604,
"rewards/margins": 0.08399216085672379,
"rewards/rejected": -0.19627881050109863,
"step": 3140
},
{
"epoch": 1.65,
"learning_rate": 4.540872830751386e-07,
"logits/chosen": 0.5374349355697632,
"logits/rejected": 0.5601732134819031,
"logps/chosen": -266.9260559082031,
"logps/rejected": -266.8581237792969,
"loss": 2206.0141,
"rewards/accuracies": 0.6499999761581421,
"rewards/chosen": -0.12373526394367218,
"rewards/margins": 0.060657333582639694,
"rewards/rejected": -0.18439260125160217,
"step": 3150
},
{
"epoch": 1.65,
"learning_rate": 4.4104410701222703e-07,
"logits/chosen": 0.5198964476585388,
"logits/rejected": 0.5673514008522034,
"logps/chosen": -250.6959228515625,
"logps/rejected": -233.34774780273438,
"loss": 1759.9059,
"rewards/accuracies": 0.7749999761581421,
"rewards/chosen": -0.11838851869106293,
"rewards/margins": 0.12481342256069183,
"rewards/rejected": -0.24320194125175476,
"step": 3160
},
{
"epoch": 1.66,
"learning_rate": 4.281728531738563e-07,
"logits/chosen": 0.597510814666748,
"logits/rejected": 0.6312834620475769,
"logps/chosen": -268.5950012207031,
"logps/rejected": -250.58059692382812,
"loss": 1960.2506,
"rewards/accuracies": 0.699999988079071,
"rewards/chosen": -0.11448697745800018,
"rewards/margins": 0.09015806764364243,
"rewards/rejected": -0.204645037651062,
"step": 3170
},
{
"epoch": 1.66,
"learning_rate": 4.154745963060197e-07,
"logits/chosen": 0.507027268409729,
"logits/rejected": 0.5732488632202148,
"logps/chosen": -280.98382568359375,
"logps/rejected": -286.677734375,
"loss": 1965.5633,
"rewards/accuracies": 0.7250000238418579,
"rewards/chosen": -0.12583956122398376,
"rewards/margins": 0.10458560287952423,
"rewards/rejected": -0.230425164103508,
"step": 3180
},
{
"epoch": 1.67,
"learning_rate": 4.029503967095097e-07,
"logits/chosen": 0.4729984402656555,
"logits/rejected": 0.5827825665473938,
"logps/chosen": -268.3514709472656,
"logps/rejected": -247.7762451171875,
"loss": 1878.176,
"rewards/accuracies": 0.731249988079071,
"rewards/chosen": -0.1029181256890297,
"rewards/margins": 0.09399056434631348,
"rewards/rejected": -0.19690869748592377,
"step": 3190
},
{
"epoch": 1.67,
"learning_rate": 3.9060130015138863e-07,
"logits/chosen": 0.5795052647590637,
"logits/rejected": 0.6329609155654907,
"logps/chosen": -262.9614562988281,
"logps/rejected": -240.7771453857422,
"loss": 1977.8852,
"rewards/accuracies": 0.65625,
"rewards/chosen": -0.1271854192018509,
"rewards/margins": 0.08980287611484528,
"rewards/rejected": -0.21698825061321259,
"step": 3200
},
{
"epoch": 1.67,
"eval_logits/chosen": 0.5363709926605225,
"eval_logits/rejected": 0.5920352935791016,
"eval_logps/chosen": -269.10711669921875,
"eval_logps/rejected": -255.2016143798828,
"eval_loss": 2026.12890625,
"eval_rewards/accuracies": 0.6959999799728394,
"eval_rewards/chosen": -0.1248970478773117,
"eval_rewards/margins": 0.09163003414869308,
"eval_rewards/rejected": -0.21652711927890778,
"eval_runtime": 416.4729,
"eval_samples_per_second": 4.802,
"eval_steps_per_second": 1.201,
"step": 3200
},
{
"epoch": 1.68,
"learning_rate": 3.784283377776651e-07,
"logits/chosen": 0.6236351728439331,
"logits/rejected": 0.630204975605011,
"logps/chosen": -267.3162536621094,
"logps/rejected": -241.71484375,
"loss": 2151.366,
"rewards/accuracies": 0.637499988079071,
"rewards/chosen": -0.13855421543121338,
"rewards/margins": 0.07831588387489319,
"rewards/rejected": -0.21687009930610657,
"step": 3210
},
{
"epoch": 1.69,
"learning_rate": 3.664325260271953e-07,
"logits/chosen": 0.586463451385498,
"logits/rejected": 0.6219819784164429,
"logps/chosen": -240.7449493408203,
"logps/rejected": -260.012451171875,
"loss": 2145.4057,
"rewards/accuracies": 0.6312500238418579,
"rewards/chosen": -0.12818796932697296,
"rewards/margins": 0.07563059777021408,
"rewards/rejected": -0.20381855964660645,
"step": 3220
},
{
"epoch": 1.69,
"learning_rate": 3.5461486654680746e-07,
"logits/chosen": 0.5574949979782104,
"logits/rejected": 0.6360602378845215,
"logps/chosen": -262.3418884277344,
"logps/rejected": -256.82366943359375,
"loss": 2092.1422,
"rewards/accuracies": 0.6812499761581421,
"rewards/chosen": -0.10938684642314911,
"rewards/margins": 0.08223484456539154,
"rewards/rejected": -0.19162169098854065,
"step": 3230
},
{
"epoch": 1.7,
"learning_rate": 3.429763461076677e-07,
"logits/chosen": 0.5418500304222107,
"logits/rejected": 0.5625206828117371,
"logps/chosen": -271.43792724609375,
"logps/rejected": -255.0489501953125,
"loss": 2035.8221,
"rewards/accuracies": 0.699999988079071,
"rewards/chosen": -0.13326099514961243,
"rewards/margins": 0.08890596777200699,
"rewards/rejected": -0.2221669703722,
"step": 3240
},
{
"epoch": 1.7,
"learning_rate": 3.315179365228824e-07,
"logits/chosen": 0.5612285733222961,
"logits/rejected": 0.5996168851852417,
"logps/chosen": -284.46612548828125,
"logps/rejected": -261.4803771972656,
"loss": 1981.793,
"rewards/accuracies": 0.71875,
"rewards/chosen": -0.12592165172100067,
"rewards/margins": 0.0976746454834938,
"rewards/rejected": -0.22359630465507507,
"step": 3250
},
{
"epoch": 1.71,
"learning_rate": 3.202405945663556e-07,
"logits/chosen": 0.5869094729423523,
"logits/rejected": 0.5811904668807983,
"logps/chosen": -273.99249267578125,
"logps/rejected": -274.64410400390625,
"loss": 2093.5598,
"rewards/accuracies": 0.699999988079071,
"rewards/chosen": -0.11422860622406006,
"rewards/margins": 0.09214137494564056,
"rewards/rejected": -0.20636996626853943,
"step": 3260
},
{
"epoch": 1.71,
"learning_rate": 3.09145261892895e-07,
"logits/chosen": 0.5232716798782349,
"logits/rejected": 0.6522939801216125,
"logps/chosen": -266.1856384277344,
"logps/rejected": -255.4128875732422,
"loss": 1884.3191,
"rewards/accuracies": 0.71875,
"rewards/chosen": -0.12048964202404022,
"rewards/margins": 0.1096540093421936,
"rewards/rejected": -0.23014366626739502,
"step": 3270
},
{
"epoch": 1.72,
"learning_rate": 2.982328649595856e-07,
"logits/chosen": 0.5434025526046753,
"logits/rejected": 0.5516559481620789,
"logps/chosen": -261.6842346191406,
"logps/rejected": -268.0997619628906,
"loss": 2020.6832,
"rewards/accuracies": 0.6499999761581421,
"rewards/chosen": -0.12252092361450195,
"rewards/margins": 0.08777900040149689,
"rewards/rejected": -0.21029992401599884,
"step": 3280
},
{
"epoch": 1.72,
"learning_rate": 2.8750431494843076e-07,
"logits/chosen": 0.5793955326080322,
"logits/rejected": 0.5846759676933289,
"logps/chosen": -256.3296813964844,
"logps/rejected": -258.81512451171875,
"loss": 2103.4049,
"rewards/accuracies": 0.7250000238418579,
"rewards/chosen": -0.13012662529945374,
"rewards/margins": 0.07379056513309479,
"rewards/rejected": -0.20391719043254852,
"step": 3290
},
{
"epoch": 1.73,
"learning_rate": 2.7696050769026954e-07,
"logits/chosen": 0.5790996551513672,
"logits/rejected": 0.5938167572021484,
"logps/chosen": -229.91567993164062,
"logps/rejected": -236.54916381835938,
"loss": 2123.3787,
"rewards/accuracies": 0.65625,
"rewards/chosen": -0.12688665091991425,
"rewards/margins": 0.07247930765151978,
"rewards/rejected": -0.19936595857143402,
"step": 3300
},
{
"epoch": 1.73,
"eval_logits/chosen": 0.5370410680770874,
"eval_logits/rejected": 0.5926198363304138,
"eval_logps/chosen": -269.0932922363281,
"eval_logps/rejected": -255.16659545898438,
"eval_loss": 2027.355224609375,
"eval_rewards/accuracies": 0.6930000185966492,
"eval_rewards/chosen": -0.12475859373807907,
"eval_rewards/margins": 0.09141821414232254,
"eval_rewards/rejected": -0.2161768227815628,
"eval_runtime": 416.6319,
"eval_samples_per_second": 4.8,
"eval_steps_per_second": 1.2,
"step": 3300
},
{
"epoch": 1.73,
"learning_rate": 2.666023235899734e-07,
"logits/chosen": 0.5439051389694214,
"logits/rejected": 0.638985276222229,
"logps/chosen": -249.70217895507812,
"logps/rejected": -246.07040405273438,
"loss": 1936.2746,
"rewards/accuracies": 0.7124999761581421,
"rewards/chosen": -0.13094648718833923,
"rewards/margins": 0.10491780191659927,
"rewards/rejected": -0.23586425185203552,
"step": 3310
},
{
"epoch": 1.74,
"learning_rate": 2.564306275529341e-07,
"logits/chosen": 0.5696260929107666,
"logits/rejected": 0.6271142959594727,
"logps/chosen": -288.08721923828125,
"logps/rejected": -263.6356201171875,
"loss": 1974.6082,
"rewards/accuracies": 0.6812499761581421,
"rewards/chosen": -0.11936695873737335,
"rewards/margins": 0.10857198387384415,
"rewards/rejected": -0.2279389351606369,
"step": 3320
},
{
"epoch": 1.74,
"learning_rate": 2.4644626891284243e-07,
"logits/chosen": 0.5715283155441284,
"logits/rejected": 0.6530539393424988,
"logps/chosen": -245.0167236328125,
"logps/rejected": -238.81460571289062,
"loss": 2066.4543,
"rewards/accuracies": 0.6875,
"rewards/chosen": -0.12561528384685516,
"rewards/margins": 0.08124671876430511,
"rewards/rejected": -0.20686200261116028,
"step": 3330
},
{
"epoch": 1.75,
"learning_rate": 2.3665008136077332e-07,
"logits/chosen": 0.5698996186256409,
"logits/rejected": 0.6029760837554932,
"logps/chosen": -264.3577575683594,
"logps/rejected": -271.068115234375,
"loss": 2090.4994,
"rewards/accuracies": 0.675000011920929,
"rewards/chosen": -0.1360047161579132,
"rewards/margins": 0.08008682727813721,
"rewards/rejected": -0.2160915583372116,
"step": 3340
},
{
"epoch": 1.75,
"learning_rate": 2.2704288287556718e-07,
"logits/chosen": 0.5687640309333801,
"logits/rejected": 0.5940347909927368,
"logps/chosen": -257.6128845214844,
"logps/rejected": -248.91015625,
"loss": 2136.7539,
"rewards/accuracies": 0.675000011920929,
"rewards/chosen": -0.12395147979259491,
"rewards/margins": 0.0794539600610733,
"rewards/rejected": -0.2034054547548294,
"step": 3350
},
{
"epoch": 1.76,
"learning_rate": 2.1762547565553293e-07,
"logits/chosen": 0.5388206839561462,
"logits/rejected": 0.5561047792434692,
"logps/chosen": -261.9334716796875,
"logps/rejected": -260.14556884765625,
"loss": 1989.5404,
"rewards/accuracies": 0.6875,
"rewards/chosen": -0.1216350793838501,
"rewards/margins": 0.09191958606243134,
"rewards/rejected": -0.21355466544628143,
"step": 3360
},
{
"epoch": 1.76,
"learning_rate": 2.083986460514631e-07,
"logits/chosen": 0.5701113343238831,
"logits/rejected": 0.6196510195732117,
"logps/chosen": -251.4109344482422,
"logps/rejected": -252.1616668701172,
"loss": 1820.4641,
"rewards/accuracies": 0.7437499761581421,
"rewards/chosen": -0.12642905116081238,
"rewards/margins": 0.10934920608997345,
"rewards/rejected": -0.23577824234962463,
"step": 3370
},
{
"epoch": 1.77,
"learning_rate": 1.993631645009747e-07,
"logits/chosen": 0.5514119863510132,
"logits/rejected": 0.5474542379379272,
"logps/chosen": -256.45831298828125,
"logps/rejected": -227.55514526367188,
"loss": 1845.2707,
"rewards/accuracies": 0.7124999761581421,
"rewards/chosen": -0.10441195964813232,
"rewards/margins": 0.1090712919831276,
"rewards/rejected": -0.21348324418067932,
"step": 3380
},
{
"epoch": 1.77,
"learning_rate": 1.9051978546417715e-07,
"logits/chosen": 0.5202070474624634,
"logits/rejected": 0.5690991282463074,
"logps/chosen": -260.46600341796875,
"logps/rejected": -261.4082946777344,
"loss": 1912.7502,
"rewards/accuracies": 0.731249988079071,
"rewards/chosen": -0.10940267145633698,
"rewards/margins": 0.10180971771478653,
"rewards/rejected": -0.2112123966217041,
"step": 3390
},
{
"epoch": 1.78,
"learning_rate": 1.818692473606748e-07,
"logits/chosen": 0.5479332208633423,
"logits/rejected": 0.5715588331222534,
"logps/chosen": -258.5849609375,
"logps/rejected": -264.29388427734375,
"loss": 1945.4934,
"rewards/accuracies": 0.7124999761581421,
"rewards/chosen": -0.12341777980327606,
"rewards/margins": 0.09188680350780487,
"rewards/rejected": -0.21530456840991974,
"step": 3400
},
{
"epoch": 1.78,
"eval_logits/chosen": 0.5352820754051208,
"eval_logits/rejected": 0.5908908247947693,
"eval_logps/chosen": -269.1009826660156,
"eval_logps/rejected": -255.1898651123047,
"eval_loss": 2025.7803955078125,
"eval_rewards/accuracies": 0.6934999823570251,
"eval_rewards/chosen": -0.12483509629964828,
"eval_rewards/margins": 0.09157437831163406,
"eval_rewards/rejected": -0.21640948951244354,
"eval_runtime": 416.572,
"eval_samples_per_second": 4.801,
"eval_steps_per_second": 1.2,
"step": 3400
},
{
"epoch": 1.78,
"learning_rate": 1.7341227250790989e-07,
"logits/chosen": 0.5836583375930786,
"logits/rejected": 0.632857084274292,
"logps/chosen": -245.8205108642578,
"logps/rejected": -252.48471069335938,
"loss": 1828.1664,
"rewards/accuracies": 0.768750011920929,
"rewards/chosen": -0.10635235160589218,
"rewards/margins": 0.11655166000127792,
"rewards/rejected": -0.2229039967060089,
"step": 3410
},
{
"epoch": 1.79,
"learning_rate": 1.6514956706084885e-07,
"logits/chosen": 0.6221760511398315,
"logits/rejected": 0.5567342042922974,
"logps/chosen": -266.02239990234375,
"logps/rejected": -246.35385131835938,
"loss": 1826.1057,
"rewards/accuracies": 0.7250000238418579,
"rewards/chosen": -0.09994658827781677,
"rewards/margins": 0.1130019798874855,
"rewards/rejected": -0.21294856071472168,
"step": 3420
},
{
"epoch": 1.8,
"learning_rate": 1.5708182095301867e-07,
"logits/chosen": 0.6005284190177917,
"logits/rejected": 0.6083909869194031,
"logps/chosen": -280.53741455078125,
"logps/rejected": -261.88201904296875,
"loss": 1851.2512,
"rewards/accuracies": 0.731249988079071,
"rewards/chosen": -0.11887475103139877,
"rewards/margins": 0.10133900493383408,
"rewards/rejected": -0.22021374106407166,
"step": 3430
},
{
"epoch": 1.8,
"learning_rate": 1.4920970783889737e-07,
"logits/chosen": 0.5680890083312988,
"logits/rejected": 0.5507141351699829,
"logps/chosen": -271.96990966796875,
"logps/rejected": -241.1654815673828,
"loss": 2041.0072,
"rewards/accuracies": 0.668749988079071,
"rewards/chosen": -0.1112411841750145,
"rewards/margins": 0.08822907507419586,
"rewards/rejected": -0.19947026669979095,
"step": 3440
},
{
"epoch": 1.81,
"learning_rate": 1.4153388503766492e-07,
"logits/chosen": 0.5438860654830933,
"logits/rejected": 0.5644111633300781,
"logps/chosen": -279.3092346191406,
"logps/rejected": -239.37167358398438,
"loss": 1966.0102,
"rewards/accuracies": 0.7124999761581421,
"rewards/chosen": -0.1369594931602478,
"rewards/margins": 0.09016064554452896,
"rewards/rejected": -0.22712013125419617,
"step": 3450
},
{
"epoch": 1.81,
"learning_rate": 1.340549934783164e-07,
"logits/chosen": 0.6110261082649231,
"logits/rejected": 0.6002285480499268,
"logps/chosen": -255.5424346923828,
"logps/rejected": -258.6153259277344,
"loss": 1778.4668,
"rewards/accuracies": 0.768750011920929,
"rewards/chosen": -0.10746339708566666,
"rewards/margins": 0.12720224261283875,
"rewards/rejected": -0.23466560244560242,
"step": 3460
},
{
"epoch": 1.82,
"learning_rate": 1.2677365764614452e-07,
"logits/chosen": 0.6116484999656677,
"logits/rejected": 0.6142521500587463,
"logps/chosen": -251.9376983642578,
"logps/rejected": -247.8961639404297,
"loss": 1943.7922,
"rewards/accuracies": 0.7437499761581421,
"rewards/chosen": -0.12416845560073853,
"rewards/margins": 0.09600269794464111,
"rewards/rejected": -0.22017112374305725,
"step": 3470
},
{
"epoch": 1.82,
"learning_rate": 1.196904855305961e-07,
"logits/chosen": 0.5488280057907104,
"logits/rejected": 0.6329927444458008,
"logps/chosen": -261.445068359375,
"logps/rejected": -255.37142944335938,
"loss": 2087.1572,
"rewards/accuracies": 0.699999988079071,
"rewards/chosen": -0.11578680574893951,
"rewards/margins": 0.08292602747678757,
"rewards/rejected": -0.19871282577514648,
"step": 3480
},
{
"epoch": 1.83,
"learning_rate": 1.1280606857450387e-07,
"logits/chosen": 0.5712449550628662,
"logits/rejected": 0.6291993856430054,
"logps/chosen": -243.8418731689453,
"logps/rejected": -233.1319122314453,
"loss": 1809.2799,
"rewards/accuracies": 0.762499988079071,
"rewards/chosen": -0.11564090102910995,
"rewards/margins": 0.11533119529485703,
"rewards/rejected": -0.23097209632396698,
"step": 3490
},
{
"epoch": 1.83,
"learning_rate": 1.0612098162470302e-07,
"logits/chosen": 0.5486131906509399,
"logits/rejected": 0.6034047603607178,
"logps/chosen": -253.94577026367188,
"logps/rejected": -243.55593872070312,
"loss": 1937.2627,
"rewards/accuracies": 0.6875,
"rewards/chosen": -0.10992947965860367,
"rewards/margins": 0.09977956861257553,
"rewards/rejected": -0.2097090482711792,
"step": 3500
},
{
"epoch": 1.83,
"eval_logits/chosen": 0.5346845984458923,
"eval_logits/rejected": 0.5903106927871704,
"eval_logps/chosen": -269.0877990722656,
"eval_logps/rejected": -255.17501831054688,
"eval_loss": 2027.823974609375,
"eval_rewards/accuracies": 0.6930000185966492,
"eval_rewards/chosen": -0.12470405548810959,
"eval_rewards/margins": 0.09155706316232681,
"eval_rewards/rejected": -0.2162611186504364,
"eval_runtime": 416.489,
"eval_samples_per_second": 4.802,
"eval_steps_per_second": 1.201,
"step": 3500
},
{
"epoch": 1.84,
"learning_rate": 9.96357828840297e-08,
"logits/chosen": 0.5791751742362976,
"logits/rejected": 0.6535072326660156,
"logps/chosen": -262.40301513671875,
"logps/rejected": -260.7125549316406,
"loss": 1964.6746,
"rewards/accuracies": 0.706250011920929,
"rewards/chosen": -0.1026170626282692,
"rewards/margins": 0.08860354125499725,
"rewards/rejected": -0.19122058153152466,
"step": 3510
},
{
"epoch": 1.84,
"learning_rate": 9.335101386471285e-08,
"logits/chosen": 0.5727615356445312,
"logits/rejected": 0.5819220542907715,
"logps/chosen": -284.75616455078125,
"logps/rejected": -250.07083129882812,
"loss": 2092.7152,
"rewards/accuracies": 0.6812499761581421,
"rewards/chosen": -0.12308394908905029,
"rewards/margins": 0.08286546170711517,
"rewards/rejected": -0.20594939589500427,
"step": 3520
},
{
"epoch": 1.85,
"learning_rate": 8.726719934315648e-08,
"logits/chosen": 0.5491209626197815,
"logits/rejected": 0.5870348811149597,
"logps/chosen": -249.99295043945312,
"logps/rejected": -249.68679809570312,
"loss": 1912.009,
"rewards/accuracies": 0.699999988079071,
"rewards/chosen": -0.10659299790859222,
"rewards/margins": 0.09919796884059906,
"rewards/rejected": -0.20579096674919128,
"step": 3530
},
{
"epoch": 1.85,
"learning_rate": 8.138484731612273e-08,
"logits/chosen": 0.6029896140098572,
"logits/rejected": 0.6406581997871399,
"logps/chosen": -256.2204284667969,
"logps/rejected": -229.89990234375,
"loss": 1913.6865,
"rewards/accuracies": 0.7250000238418579,
"rewards/chosen": -0.10959267616271973,
"rewards/margins": 0.1080545037984848,
"rewards/rejected": -0.21764719486236572,
"step": 3540
},
{
"epoch": 1.86,
"learning_rate": 7.57044489583128e-08,
"logits/chosen": 0.5283448100090027,
"logits/rejected": 0.5756082534790039,
"logps/chosen": -266.2628479003906,
"logps/rejected": -251.70962524414062,
"loss": 2214.9162,
"rewards/accuracies": 0.675000011920929,
"rewards/chosen": -0.12659896910190582,
"rewards/margins": 0.06956067681312561,
"rewards/rejected": -0.19615966081619263,
"step": 3550
},
{
"epoch": 1.86,
"learning_rate": 7.022647858135501e-08,
"logits/chosen": 0.5648905038833618,
"logits/rejected": 0.5851987600326538,
"logps/chosen": -255.75704956054688,
"logps/rejected": -235.11666870117188,
"loss": 1911.8838,
"rewards/accuracies": 0.699999988079071,
"rewards/chosen": -0.10966980457305908,
"rewards/margins": 0.09904567152261734,
"rewards/rejected": -0.20871546864509583,
"step": 3560
},
{
"epoch": 1.87,
"learning_rate": 6.495139359419922e-08,
"logits/chosen": 0.5362564921379089,
"logits/rejected": 0.629612922668457,
"logps/chosen": -303.055419921875,
"logps/rejected": -271.39288330078125,
"loss": 1875.824,
"rewards/accuracies": 0.71875,
"rewards/chosen": -0.1098160371184349,
"rewards/margins": 0.11708301305770874,
"rewards/rejected": -0.22689905762672424,
"step": 3570
},
{
"epoch": 1.87,
"learning_rate": 5.987963446492384e-08,
"logits/chosen": 0.5597736239433289,
"logits/rejected": 0.5716227889060974,
"logps/chosen": -262.5528564453125,
"logps/rejected": -251.4736785888672,
"loss": 1887.5355,
"rewards/accuracies": 0.71875,
"rewards/chosen": -0.10405333340167999,
"rewards/margins": 0.11675725132226944,
"rewards/rejected": -0.22081057727336884,
"step": 3580
},
{
"epoch": 1.88,
"learning_rate": 5.501162468395688e-08,
"logits/chosen": 0.5817372798919678,
"logits/rejected": 0.5784239768981934,
"logps/chosen": -251.4989013671875,
"logps/rejected": -250.26522827148438,
"loss": 1920.6814,
"rewards/accuracies": 0.699999988079071,
"rewards/chosen": -0.12673336267471313,
"rewards/margins": 0.11053230613470078,
"rewards/rejected": -0.2372656762599945,
"step": 3590
},
{
"epoch": 1.88,
"learning_rate": 5.034777072871394e-08,
"logits/chosen": 0.5656172037124634,
"logits/rejected": 0.6273232102394104,
"logps/chosen": -250.90109252929688,
"logps/rejected": -256.27178955078125,
"loss": 2007.2062,
"rewards/accuracies": 0.7124999761581421,
"rewards/chosen": -0.1188703402876854,
"rewards/margins": 0.09515853226184845,
"rewards/rejected": -0.21402888000011444,
"step": 3600
},
{
"epoch": 1.88,
"eval_logits/chosen": 0.5352125763893127,
"eval_logits/rejected": 0.5910032391548157,
"eval_logps/chosen": -269.0622863769531,
"eval_logps/rejected": -255.18426513671875,
"eval_loss": 2025.32275390625,
"eval_rewards/accuracies": 0.6894999742507935,
"eval_rewards/chosen": -0.12444862723350525,
"eval_rewards/margins": 0.0919048860669136,
"eval_rewards/rejected": -0.21635350584983826,
"eval_runtime": 416.4513,
"eval_samples_per_second": 4.802,
"eval_steps_per_second": 1.201,
"step": 3600
},
{
"epoch": 1.89,
"learning_rate": 4.5888462029658186e-08,
"logits/chosen": 0.5575802326202393,
"logits/rejected": 0.5975883603096008,
"logps/chosen": -251.73623657226562,
"logps/rejected": -250.5563201904297,
"loss": 1952.7262,
"rewards/accuracies": 0.71875,
"rewards/chosen": -0.12859514355659485,
"rewards/margins": 0.0945589691400528,
"rewards/rejected": -0.22315411269664764,
"step": 3610
},
{
"epoch": 1.89,
"learning_rate": 4.163407093778243e-08,
"logits/chosen": 0.5034095048904419,
"logits/rejected": 0.5554597973823547,
"logps/chosen": -264.1334533691406,
"logps/rejected": -260.69805908203125,
"loss": 2102.8385,
"rewards/accuracies": 0.612500011920929,
"rewards/chosen": -0.1385246217250824,
"rewards/margins": 0.08275660127401352,
"rewards/rejected": -0.22128121554851532,
"step": 3620
},
{
"epoch": 1.9,
"learning_rate": 3.7584952693519025e-08,
"logits/chosen": 0.5984662175178528,
"logits/rejected": 0.5975054502487183,
"logps/chosen": -270.71661376953125,
"logps/rejected": -260.6340026855469,
"loss": 1962.2367,
"rewards/accuracies": 0.699999988079071,
"rewards/chosen": -0.11859796196222305,
"rewards/margins": 0.0891033262014389,
"rewards/rejected": -0.20770128071308136,
"step": 3630
},
{
"epoch": 1.91,
"learning_rate": 3.37414453970758e-08,
"logits/chosen": 0.5739267468452454,
"logits/rejected": 0.577027440071106,
"logps/chosen": -249.9311065673828,
"logps/rejected": -226.3584442138672,
"loss": 2134.9982,
"rewards/accuracies": 0.65625,
"rewards/chosen": -0.13262119889259338,
"rewards/margins": 0.08364128321409225,
"rewards/rejected": -0.21626248955726624,
"step": 3640
},
{
"epoch": 1.91,
"learning_rate": 3.0103869980206145e-08,
"logits/chosen": 0.6304140090942383,
"logits/rejected": 0.63347989320755,
"logps/chosen": -239.28915405273438,
"logps/rejected": -258.83660888671875,
"loss": 2049.8613,
"rewards/accuracies": 0.6937500238418579,
"rewards/chosen": -0.13652729988098145,
"rewards/margins": 0.08251725137233734,
"rewards/rejected": -0.2190445363521576,
"step": 3650
},
{
"epoch": 1.92,
"learning_rate": 2.6672530179410183e-08,
"logits/chosen": 0.5943226218223572,
"logits/rejected": 0.6503596305847168,
"logps/chosen": -264.0231018066406,
"logps/rejected": -245.2039031982422,
"loss": 2003.8398,
"rewards/accuracies": 0.675000011920929,
"rewards/chosen": -0.12203504145145416,
"rewards/margins": 0.09293092787265778,
"rewards/rejected": -0.21496596932411194,
"step": 3660
},
{
"epoch": 1.92,
"learning_rate": 2.3447712510573928e-08,
"logits/chosen": 0.6132981777191162,
"logits/rejected": 0.6643080115318298,
"logps/chosen": -258.1444091796875,
"logps/rejected": -241.70986938476562,
"loss": 1827.9928,
"rewards/accuracies": 0.7250000238418579,
"rewards/chosen": -0.12529726326465607,
"rewards/margins": 0.11168257147073746,
"rewards/rejected": -0.23697984218597412,
"step": 3670
},
{
"epoch": 1.93,
"learning_rate": 2.04296862450451e-08,
"logits/chosen": 0.5302231907844543,
"logits/rejected": 0.5352843999862671,
"logps/chosen": -270.13079833984375,
"logps/rejected": -240.57931518554688,
"loss": 2135.2258,
"rewards/accuracies": 0.65625,
"rewards/chosen": -0.10986328125,
"rewards/margins": 0.07277282327413559,
"rewards/rejected": -0.182636097073555,
"step": 3680
},
{
"epoch": 1.93,
"learning_rate": 1.7618703387147495e-08,
"logits/chosen": 0.5543524622917175,
"logits/rejected": 0.5519607663154602,
"logps/chosen": -281.1496276855469,
"logps/rejected": -274.3504943847656,
"loss": 1951.3572,
"rewards/accuracies": 0.71875,
"rewards/chosen": -0.10969813168048859,
"rewards/margins": 0.09588075429201126,
"rewards/rejected": -0.20557889342308044,
"step": 3690
},
{
"epoch": 1.94,
"learning_rate": 1.501499865314171e-08,
"logits/chosen": 0.5777779817581177,
"logits/rejected": 0.6056709289550781,
"logps/chosen": -258.98333740234375,
"logps/rejected": -245.7554473876953,
"loss": 2076.715,
"rewards/accuracies": 0.706250011920929,
"rewards/chosen": -0.11231188476085663,
"rewards/margins": 0.09381435066461563,
"rewards/rejected": -0.20612624287605286,
"step": 3700
},
{
"epoch": 1.94,
"eval_logits/chosen": 0.5358251333236694,
"eval_logits/rejected": 0.5913307666778564,
"eval_logps/chosen": -269.0487365722656,
"eval_logps/rejected": -255.13833618164062,
"eval_loss": 2027.4857177734375,
"eval_rewards/accuracies": 0.6919999718666077,
"eval_rewards/chosen": -0.12431324273347855,
"eval_rewards/margins": 0.09158134460449219,
"eval_rewards/rejected": -0.21589456498622894,
"eval_runtime": 416.7132,
"eval_samples_per_second": 4.799,
"eval_steps_per_second": 1.2,
"step": 3700
},
{
"epoch": 1.94,
"learning_rate": 1.2618789451623314e-08,
"logits/chosen": 0.5645478963851929,
"logits/rejected": 0.6109569072723389,
"logps/chosen": -224.93197631835938,
"logps/rejected": -236.52059936523438,
"loss": 2014.5852,
"rewards/accuracies": 0.706250011920929,
"rewards/chosen": -0.12038487195968628,
"rewards/margins": 0.08836686611175537,
"rewards/rejected": -0.20875172317028046,
"step": 3710
},
{
"epoch": 1.95,
"learning_rate": 1.0430275865371265e-08,
"logits/chosen": 0.5508732795715332,
"logits/rejected": 0.6115376353263855,
"logps/chosen": -280.9700927734375,
"logps/rejected": -276.9327087402344,
"loss": 2099.5396,
"rewards/accuracies": 0.625,
"rewards/chosen": -0.12365047633647919,
"rewards/margins": 0.08925069868564606,
"rewards/rejected": -0.21290118992328644,
"step": 3720
},
{
"epoch": 1.95,
"learning_rate": 8.449640634639878e-09,
"logits/chosen": 0.5355272889137268,
"logits/rejected": 0.5811390280723572,
"logps/chosen": -234.78927612304688,
"logps/rejected": -228.4997100830078,
"loss": 2043.5014,
"rewards/accuracies": 0.6812499761581421,
"rewards/chosen": -0.1230451837182045,
"rewards/margins": 0.08070604503154755,
"rewards/rejected": -0.20375123620033264,
"step": 3730
},
{
"epoch": 1.96,
"learning_rate": 6.677049141901315e-09,
"logits/chosen": 0.5882354974746704,
"logits/rejected": 0.572884738445282,
"logps/chosen": -238.9161834716797,
"logps/rejected": -247.5298309326172,
"loss": 2155.4453,
"rewards/accuracies": 0.65625,
"rewards/chosen": -0.13735826313495636,
"rewards/margins": 0.06890521943569183,
"rewards/rejected": -0.2062634974718094,
"step": 3740
},
{
"epoch": 1.96,
"learning_rate": 5.112649398034686e-09,
"logits/chosen": 0.6161108016967773,
"logits/rejected": 0.6904915571212769,
"logps/chosen": -284.59112548828125,
"logps/rejected": -254.5317840576172,
"loss": 2025.8148,
"rewards/accuracies": 0.699999988079071,
"rewards/chosen": -0.125542551279068,
"rewards/margins": 0.1034855991601944,
"rewards/rejected": -0.2290281355381012,
"step": 3750
},
{
"epoch": 1.97,
"learning_rate": 3.756572029968708e-09,
"logits/chosen": 0.5819270610809326,
"logits/rejected": 0.552914023399353,
"logps/chosen": -255.23886108398438,
"logps/rejected": -249.76144409179688,
"loss": 1779.8414,
"rewards/accuracies": 0.75,
"rewards/chosen": -0.11097989976406097,
"rewards/margins": 0.1095919981598854,
"rewards/rejected": -0.22057190537452698,
"step": 3760
},
{
"epoch": 1.97,
"learning_rate": 2.6089302697732133e-09,
"logits/chosen": 0.5825963020324707,
"logits/rejected": 0.5435997843742371,
"logps/chosen": -250.9562225341797,
"logps/rejected": -227.83010864257812,
"loss": 1853.0563,
"rewards/accuracies": 0.7562500238418579,
"rewards/chosen": -0.11256451904773712,
"rewards/margins": 0.10355798900127411,
"rewards/rejected": -0.21612253785133362,
"step": 3770
},
{
"epoch": 1.98,
"learning_rate": 1.6698199452053199e-09,
"logits/chosen": 0.6074197292327881,
"logits/rejected": 0.6451854705810547,
"logps/chosen": -269.6044616699219,
"logps/rejected": -231.6178436279297,
"loss": 1905.3814,
"rewards/accuracies": 0.71875,
"rewards/chosen": -0.10596567392349243,
"rewards/margins": 0.099820576608181,
"rewards/rejected": -0.20578625798225403,
"step": 3780
},
{
"epoch": 1.98,
"learning_rate": 9.393194717061127e-10,
"logits/chosen": 0.5966477394104004,
"logits/rejected": 0.57940673828125,
"logps/chosen": -261.906982421875,
"logps/rejected": -243.7214813232422,
"loss": 2099.1896,
"rewards/accuracies": 0.6875,
"rewards/chosen": -0.12468767166137695,
"rewards/margins": 0.0848483294248581,
"rewards/rejected": -0.20953598618507385,
"step": 3790
},
{
"epoch": 1.99,
"learning_rate": 4.1748984585560094e-10,
"logits/chosen": 0.5209355354309082,
"logits/rejected": 0.6011817455291748,
"logps/chosen": -257.47882080078125,
"logps/rejected": -253.18017578125,
"loss": 2055.2201,
"rewards/accuracies": 0.668749988079071,
"rewards/chosen": -0.12113461643457413,
"rewards/margins": 0.09108567237854004,
"rewards/rejected": -0.21222028136253357,
"step": 3800
},
{
"epoch": 1.99,
"eval_logits/chosen": 0.5346859693527222,
"eval_logits/rejected": 0.5902337431907654,
"eval_logps/chosen": -269.0542907714844,
"eval_logps/rejected": -255.1454620361328,
"eval_loss": 2027.8082275390625,
"eval_rewards/accuracies": 0.6919999718666077,
"eval_rewards/chosen": -0.12436838448047638,
"eval_rewards/margins": 0.09159712493419647,
"eval_rewards/rejected": -0.21596547961235046,
"eval_runtime": 416.5485,
"eval_samples_per_second": 4.801,
"eval_steps_per_second": 1.2,
"step": 3800
},
{
"epoch": 1.99,
"learning_rate": 1.0437464027707179e-10,
"logits/chosen": 0.5776988863945007,
"logits/rejected": 0.6123504638671875,
"logps/chosen": -265.8362121582031,
"logps/rejected": -237.8904571533203,
"loss": 2055.609,
"rewards/accuracies": 0.7124999761581421,
"rewards/chosen": -0.1267283707857132,
"rewards/margins": 0.08532574027776718,
"rewards/rejected": -0.21205410361289978,
"step": 3810
},
{
"epoch": 2.0,
"learning_rate": 0.0,
"logits/chosen": 0.5118510127067566,
"logits/rejected": 0.5845987200737,
"logps/chosen": -274.1031188964844,
"logps/rejected": -256.99688720703125,
"loss": 2139.6068,
"rewards/accuracies": 0.637499988079071,
"rewards/chosen": -0.13095693290233612,
"rewards/margins": 0.07750894129276276,
"rewards/rejected": -0.20846585929393768,
"step": 3820
},
{
"epoch": 2.0,
"step": 3820,
"total_flos": 0.0,
"train_loss": 2099.8451463309884,
"train_runtime": 42790.3459,
"train_samples_per_second": 1.429,
"train_steps_per_second": 0.089
}
],
"logging_steps": 10,
"max_steps": 3820,
"num_input_tokens_seen": 0,
"num_train_epochs": 2,
"save_steps": 100,
"total_flos": 0.0,
"train_batch_size": 4,
"trial_name": null,
"trial_params": null
}