slm-2-dpo-full / trainer_state.json
imelnyk's picture
Model save
152b38d verified
{
"best_metric": null,
"best_model_checkpoint": null,
"epoch": 0.999345977763244,
"eval_steps": 100,
"global_step": 764,
"is_hyper_param_search": false,
"is_local_process_zero": true,
"is_world_process_zero": true,
"log_history": [
{
"epoch": 0.0,
"learning_rate": 6.493506493506494e-09,
"logits/chosen": -2.0615594387054443,
"logits/rejected": -1.9222214221954346,
"logps/chosen": -3380.6083984375,
"logps/rejected": -2521.2978515625,
"loss": 0.0001,
"rewards/accuracies": 0.0,
"rewards/chosen": 0.0,
"rewards/margins": 0.0,
"rewards/rejected": 0.0,
"step": 1
},
{
"epoch": 0.01,
"learning_rate": 6.493506493506492e-08,
"logits/chosen": -1.674426555633545,
"logits/rejected": -1.637134313583374,
"logps/chosen": -2549.3515625,
"logps/rejected": -2319.4013671875,
"loss": 10.0505,
"rewards/accuracies": 0.4333333373069763,
"rewards/chosen": 0.0008169158827513456,
"rewards/margins": 0.0011402772506698966,
"rewards/rejected": -0.00032336192089132965,
"step": 10
},
{
"epoch": 0.03,
"learning_rate": 1.2987012987012984e-07,
"logits/chosen": -1.6043205261230469,
"logits/rejected": -1.5535523891448975,
"logps/chosen": -2340.101318359375,
"logps/rejected": -2224.145263671875,
"loss": 7.4843,
"rewards/accuracies": 0.5200000405311584,
"rewards/chosen": 0.00018432810611557215,
"rewards/margins": 0.0009077669237740338,
"rewards/rejected": -0.0007234388613142073,
"step": 20
},
{
"epoch": 0.04,
"learning_rate": 1.948051948051948e-07,
"logits/chosen": -1.6847426891326904,
"logits/rejected": -1.6577625274658203,
"logps/chosen": -2983.23681640625,
"logps/rejected": -2513.237060546875,
"loss": 9.1379,
"rewards/accuracies": 0.48000001907348633,
"rewards/chosen": 0.010261936113238335,
"rewards/margins": 0.004135974682867527,
"rewards/rejected": 0.006125961430370808,
"step": 30
},
{
"epoch": 0.05,
"learning_rate": 2.597402597402597e-07,
"logits/chosen": -1.859400987625122,
"logits/rejected": -1.8100417852401733,
"logps/chosen": -2879.57470703125,
"logps/rejected": -2273.878173828125,
"loss": 12.271,
"rewards/accuracies": 0.5,
"rewards/chosen": 0.012033696286380291,
"rewards/margins": 0.005555520299822092,
"rewards/rejected": 0.006478174589574337,
"step": 40
},
{
"epoch": 0.07,
"learning_rate": 3.2467532467532465e-07,
"logits/chosen": -1.828608751296997,
"logits/rejected": -1.805625319480896,
"logps/chosen": -2893.784423828125,
"logps/rejected": -2551.77294921875,
"loss": 8.7411,
"rewards/accuracies": 0.5600000023841858,
"rewards/chosen": 0.02166888490319252,
"rewards/margins": 0.007775471545755863,
"rewards/rejected": 0.013893413357436657,
"step": 50
},
{
"epoch": 0.08,
"learning_rate": 3.896103896103896e-07,
"logits/chosen": -1.7459495067596436,
"logits/rejected": -1.6628999710083008,
"logps/chosen": -3231.689697265625,
"logps/rejected": -2554.42919921875,
"loss": 9.758,
"rewards/accuracies": 0.559999942779541,
"rewards/chosen": 0.027519574388861656,
"rewards/margins": 0.008895651437342167,
"rewards/rejected": 0.018623923882842064,
"step": 60
},
{
"epoch": 0.09,
"learning_rate": 4.545454545454545e-07,
"logits/chosen": -1.8072433471679688,
"logits/rejected": -1.7838470935821533,
"logps/chosen": -2829.386474609375,
"logps/rejected": -2542.68701171875,
"loss": 11.0017,
"rewards/accuracies": 0.5,
"rewards/chosen": 0.024034958332777023,
"rewards/margins": 0.006175906863063574,
"rewards/rejected": 0.017859051004052162,
"step": 70
},
{
"epoch": 0.1,
"learning_rate": 4.99976474872689e-07,
"logits/chosen": -1.7730411291122437,
"logits/rejected": -1.7399647235870361,
"logps/chosen": -2769.705322265625,
"logps/rejected": -2476.75634765625,
"loss": 15.623,
"rewards/accuracies": 0.5400000214576721,
"rewards/chosen": 0.008623984642326832,
"rewards/margins": 0.008157819509506226,
"rewards/rejected": 0.0004661638231482357,
"step": 80
},
{
"epoch": 0.12,
"learning_rate": 4.995583735427465e-07,
"logits/chosen": -1.790204644203186,
"logits/rejected": -1.7226215600967407,
"logps/chosen": -2688.0732421875,
"logps/rejected": -2436.649658203125,
"loss": 11.9811,
"rewards/accuracies": 0.6100000143051147,
"rewards/chosen": 0.017978714779019356,
"rewards/margins": 0.017238261178135872,
"rewards/rejected": 0.0007404519128613174,
"step": 90
},
{
"epoch": 0.13,
"learning_rate": 4.986184978516146e-07,
"logits/chosen": -1.7211675643920898,
"logits/rejected": -1.6991230249404907,
"logps/chosen": -2611.177001953125,
"logps/rejected": -2212.4033203125,
"loss": 16.8403,
"rewards/accuracies": 0.5200001001358032,
"rewards/chosen": 0.024822045117616653,
"rewards/margins": 0.00336282467469573,
"rewards/rejected": 0.021459218114614487,
"step": 100
},
{
"epoch": 0.13,
"eval_logits/chosen": -1.6729556322097778,
"eval_logits/rejected": -1.6068017482757568,
"eval_logps/chosen": -2806.55517578125,
"eval_logps/rejected": -2491.901123046875,
"eval_loss": 19.51178741455078,
"eval_rewards/accuracies": 0.52734375,
"eval_rewards/chosen": 0.025559017434716225,
"eval_rewards/margins": 0.008243386633694172,
"eval_rewards/rejected": 0.017315629869699478,
"eval_runtime": 115.2508,
"eval_samples_per_second": 17.353,
"eval_steps_per_second": 0.278,
"step": 100
},
{
"epoch": 0.14,
"learning_rate": 4.971588128827782e-07,
"logits/chosen": -1.7473026514053345,
"logits/rejected": -1.6806236505508423,
"logps/chosen": -3125.757080078125,
"logps/rejected": -2645.337158203125,
"loss": 26.9149,
"rewards/accuracies": 0.6000000238418579,
"rewards/chosen": 0.016931097954511642,
"rewards/margins": 0.002864243695512414,
"rewards/rejected": 0.014066850766539574,
"step": 110
},
{
"epoch": 0.16,
"learning_rate": 4.951823705321981e-07,
"logits/chosen": -1.7069530487060547,
"logits/rejected": -1.6579583883285522,
"logps/chosen": -2828.78662109375,
"logps/rejected": -2442.76416015625,
"loss": 33.872,
"rewards/accuracies": 0.5600000023841858,
"rewards/chosen": 0.013961514458060265,
"rewards/margins": 0.00896529946476221,
"rewards/rejected": 0.004996216390281916,
"step": 120
},
{
"epoch": 0.17,
"learning_rate": 4.926933031274343e-07,
"logits/chosen": -1.7224699258804321,
"logits/rejected": -1.6934731006622314,
"logps/chosen": -2923.9306640625,
"logps/rejected": -2566.210693359375,
"loss": 39.0757,
"rewards/accuracies": 0.550000011920929,
"rewards/chosen": 0.038237668573856354,
"rewards/margins": 0.006029448006302118,
"rewards/rejected": 0.03220822289586067,
"step": 130
},
{
"epoch": 0.18,
"learning_rate": 4.896968147878145e-07,
"logits/chosen": -1.7280409336090088,
"logits/rejected": -1.7070726156234741,
"logps/chosen": -2737.75927734375,
"logps/rejected": -2486.45751953125,
"loss": 18.5231,
"rewards/accuracies": 0.6299999952316284,
"rewards/chosen": 0.031596291810274124,
"rewards/margins": 0.04216960817575455,
"rewards/rejected": -0.010573318228125572,
"step": 140
},
{
"epoch": 0.2,
"learning_rate": 4.861991705437081e-07,
"logits/chosen": -1.7859830856323242,
"logits/rejected": -1.7191492319107056,
"logps/chosen": -2743.43310546875,
"logps/rejected": -2297.162109375,
"loss": 20.7835,
"rewards/accuracies": 0.5800000429153442,
"rewards/chosen": 0.0336376316845417,
"rewards/margins": 0.011832155287265778,
"rewards/rejected": 0.021805476397275925,
"step": 150
},
{
"epoch": 0.21,
"learning_rate": 4.822076832376586e-07,
"logits/chosen": -1.8132251501083374,
"logits/rejected": -1.7665789127349854,
"logps/chosen": -2841.165771484375,
"logps/rejected": -2748.486572265625,
"loss": 57.9401,
"rewards/accuracies": 0.5099999904632568,
"rewards/chosen": 0.006118610501289368,
"rewards/margins": 0.0022155127953737974,
"rewards/rejected": 0.0039030970074236393,
"step": 160
},
{
"epoch": 0.22,
"learning_rate": 4.777306982347594e-07,
"logits/chosen": -1.6557657718658447,
"logits/rejected": -1.5996118783950806,
"logps/chosen": -3055.95361328125,
"logps/rejected": -2603.83642578125,
"loss": 23.1296,
"rewards/accuracies": 0.6200000047683716,
"rewards/chosen": 0.028251701965928078,
"rewards/margins": 0.020935241132974625,
"rewards/rejected": 0.007316464092582464,
"step": 170
},
{
"epoch": 0.24,
"learning_rate": 4.7277757597424075e-07,
"logits/chosen": -1.8335905075073242,
"logits/rejected": -1.7595329284667969,
"logps/chosen": -2963.73779296875,
"logps/rejected": -2540.163818359375,
"loss": 40.5046,
"rewards/accuracies": 0.5400000810623169,
"rewards/chosen": 0.018591446802020073,
"rewards/margins": -0.0024230503477156162,
"rewards/rejected": 0.02101449854671955,
"step": 180
},
{
"epoch": 0.25,
"learning_rate": 4.6735867239874904e-07,
"logits/chosen": -1.8637840747833252,
"logits/rejected": -1.7640159130096436,
"logps/chosen": -3237.434814453125,
"logps/rejected": -2429.197998046875,
"loss": 36.3042,
"rewards/accuracies": 0.6200000047683716,
"rewards/chosen": 0.04794805496931076,
"rewards/margins": 0.019117821007966995,
"rewards/rejected": 0.028830235823988914,
"step": 190
},
{
"epoch": 0.26,
"learning_rate": 4.6148531730223733e-07,
"logits/chosen": -1.6909841299057007,
"logits/rejected": -1.6915366649627686,
"logps/chosen": -2649.89404296875,
"logps/rejected": -2436.87353515625,
"loss": 28.1241,
"rewards/accuracies": 0.5300000309944153,
"rewards/chosen": 0.007661645300686359,
"rewards/margins": 0.0055509163066744804,
"rewards/rejected": 0.0021107294596731663,
"step": 200
},
{
"epoch": 0.26,
"eval_logits/chosen": -1.681164264678955,
"eval_logits/rejected": -1.618328332901001,
"eval_logps/chosen": -2808.258056640625,
"eval_logps/rejected": -2494.01953125,
"eval_loss": 32.517486572265625,
"eval_rewards/accuracies": 0.5234375,
"eval_rewards/chosen": 0.008527392521500587,
"eval_rewards/margins": 0.012391308322548866,
"eval_rewards/rejected": -0.0038639232516288757,
"eval_runtime": 113.682,
"eval_samples_per_second": 17.593,
"eval_steps_per_second": 0.281,
"step": 200
},
{
"epoch": 0.27,
"learning_rate": 4.5516979064173524e-07,
"logits/chosen": -1.749903917312622,
"logits/rejected": -1.7615283727645874,
"logps/chosen": -2285.7451171875,
"logps/rejected": -2269.229736328125,
"loss": 25.9535,
"rewards/accuracies": 0.6100000143051147,
"rewards/chosen": 0.011981850489974022,
"rewards/margins": 0.014764687046408653,
"rewards/rejected": -0.0027828349266201258,
"step": 210
},
{
"epoch": 0.29,
"learning_rate": 4.484252968625277e-07,
"logits/chosen": -1.716509222984314,
"logits/rejected": -1.6396989822387695,
"logps/chosen": -2435.95556640625,
"logps/rejected": -1922.770751953125,
"loss": 28.3739,
"rewards/accuracies": 0.6200000047683716,
"rewards/chosen": 0.004359879065304995,
"rewards/margins": 0.007711753249168396,
"rewards/rejected": -0.0033518739510327578,
"step": 220
},
{
"epoch": 0.3,
"learning_rate": 4.4126593729042193e-07,
"logits/chosen": -1.799469232559204,
"logits/rejected": -1.757004737854004,
"logps/chosen": -3254.6396484375,
"logps/rejected": -2515.59619140625,
"loss": 39.4707,
"rewards/accuracies": 0.5900000333786011,
"rewards/chosen": 0.03561704605817795,
"rewards/margins": 0.019830647855997086,
"rewards/rejected": 0.015786398202180862,
"step": 230
},
{
"epoch": 0.31,
"learning_rate": 4.3370668064882397e-07,
"logits/chosen": -1.7325947284698486,
"logits/rejected": -1.7474550008773804,
"logps/chosen": -2579.47412109375,
"logps/rejected": -2328.500732421875,
"loss": 44.2727,
"rewards/accuracies": 0.5100000500679016,
"rewards/chosen": 0.04269097000360489,
"rewards/margins": 0.02060030959546566,
"rewards/rejected": 0.02209065482020378,
"step": 240
},
{
"epoch": 0.33,
"learning_rate": 4.2576333176226944e-07,
"logits/chosen": -1.7366650104522705,
"logits/rejected": -1.706789255142212,
"logps/chosen": -2479.5576171875,
"logps/rejected": -2277.726318359375,
"loss": 29.5758,
"rewards/accuracies": 0.5300000309944153,
"rewards/chosen": 0.1058274507522583,
"rewards/margins": 0.013660475611686707,
"rewards/rejected": 0.0921669602394104,
"step": 250
},
{
"epoch": 0.34,
"learning_rate": 4.17452498511841e-07,
"logits/chosen": -1.7807962894439697,
"logits/rejected": -1.7134149074554443,
"logps/chosen": -2989.12841796875,
"logps/rejected": -2354.25830078125,
"loss": 38.7316,
"rewards/accuracies": 0.5200000405311584,
"rewards/chosen": 0.023859605193138123,
"rewards/margins": 0.005521018523722887,
"rewards/rejected": 0.018338587135076523,
"step": 260
},
{
"epoch": 0.35,
"learning_rate": 4.087915571115629e-07,
"logits/chosen": -1.8165556192398071,
"logits/rejected": -1.7687098979949951,
"logps/chosen": -2833.55859375,
"logps/rejected": -2183.32470703125,
"loss": 330.4642,
"rewards/accuracies": 0.5699999928474426,
"rewards/chosen": 0.031318746507167816,
"rewards/margins": 0.029996121302247047,
"rewards/rejected": 0.0013226259034126997,
"step": 270
},
{
"epoch": 0.37,
"learning_rate": 3.997986157783715e-07,
"logits/chosen": -1.6980018615722656,
"logits/rejected": -1.589050531387329,
"logps/chosen": -3510.792236328125,
"logps/rejected": -2689.208984375,
"loss": 58.1646,
"rewards/accuracies": 0.5200000405311584,
"rewards/chosen": 0.014776378870010376,
"rewards/margins": 0.011294273659586906,
"rewards/rejected": 0.0034821047447621822,
"step": 280
},
{
"epoch": 0.38,
"learning_rate": 3.9049247687162155e-07,
"logits/chosen": -1.7791646718978882,
"logits/rejected": -1.7399044036865234,
"logps/chosen": -2478.590576171875,
"logps/rejected": -2269.01416015625,
"loss": 31.6725,
"rewards/accuracies": 0.5699999928474426,
"rewards/chosen": 0.04884537309408188,
"rewards/margins": 0.0339895561337471,
"rewards/rejected": 0.014855814166367054,
"step": 290
},
{
"epoch": 0.39,
"learning_rate": 3.8089259758128543e-07,
"logits/chosen": -1.670789361000061,
"logits/rejected": -1.6030629873275757,
"logps/chosen": -2726.465576171875,
"logps/rejected": -2119.26123046875,
"loss": 84.7591,
"rewards/accuracies": 0.5699999928474426,
"rewards/chosen": 0.021672677248716354,
"rewards/margins": -0.010785548016428947,
"rewards/rejected": 0.03245822712779045,
"step": 300
},
{
"epoch": 0.39,
"eval_logits/chosen": -1.660080909729004,
"eval_logits/rejected": -1.596778154373169,
"eval_logps/chosen": -2806.140625,
"eval_logps/rejected": -2492.270263671875,
"eval_loss": 47.80431365966797,
"eval_rewards/accuracies": 0.5390625,
"eval_rewards/chosen": 0.029702020809054375,
"eval_rewards/margins": 0.01607733778655529,
"eval_rewards/rejected": 0.013624681159853935,
"eval_runtime": 116.3019,
"eval_samples_per_second": 17.197,
"eval_steps_per_second": 0.275,
"step": 300
},
{
"epoch": 0.41,
"learning_rate": 3.710190492470386e-07,
"logits/chosen": -1.6620228290557861,
"logits/rejected": -1.7311099767684937,
"logps/chosen": -2315.977294921875,
"logps/rejected": -2199.08251953125,
"loss": 43.6013,
"rewards/accuracies": 0.5400000214576721,
"rewards/chosen": 0.032384876161813736,
"rewards/margins": 0.008921505883336067,
"rewards/rejected": 0.02346337027847767,
"step": 310
},
{
"epoch": 0.42,
"learning_rate": 3.6089247539328616e-07,
"logits/chosen": -1.7675050497055054,
"logits/rejected": -1.7156997919082642,
"logps/chosen": -2859.810791015625,
"logps/rejected": -2569.75537109375,
"loss": 38.8904,
"rewards/accuracies": 0.559999942779541,
"rewards/chosen": 0.020630866289138794,
"rewards/margins": 0.0018306337296962738,
"rewards/rejected": 0.01880022883415222,
"step": 320
},
{
"epoch": 0.43,
"learning_rate": 3.5053404856787166e-07,
"logits/chosen": -1.6446609497070312,
"logits/rejected": -1.5918724536895752,
"logps/chosen": -3104.72802734375,
"logps/rejected": -2430.239013671875,
"loss": 84.9753,
"rewards/accuracies": 0.47999995946884155,
"rewards/chosen": 0.053660690784454346,
"rewards/margins": -0.005831834394484758,
"rewards/rejected": 0.05949252098798752,
"step": 330
},
{
"epoch": 0.44,
"learning_rate": 3.399654260747078e-07,
"logits/chosen": -1.699196219444275,
"logits/rejected": -1.7045748233795166,
"logps/chosen": -2584.699462890625,
"logps/rejected": -2263.678466796875,
"loss": 38.1532,
"rewards/accuracies": 0.5300000309944153,
"rewards/chosen": 0.02709970250725746,
"rewards/margins": 0.01412280835211277,
"rewards/rejected": 0.012976895086467266,
"step": 340
},
{
"epoch": 0.46,
"learning_rate": 3.2920870469288373e-07,
"logits/chosen": -1.7267532348632812,
"logits/rejected": -1.6659395694732666,
"logps/chosen": -2935.341796875,
"logps/rejected": -2503.583984375,
"loss": 47.1836,
"rewards/accuracies": 0.5199999809265137,
"rewards/chosen": 0.031215447932481766,
"rewards/margins": 0.022346725687384605,
"rewards/rejected": 0.008868719451129436,
"step": 350
},
{
"epoch": 0.47,
"learning_rate": 3.182863744769218e-07,
"logits/chosen": -1.7288787364959717,
"logits/rejected": -1.6928844451904297,
"logps/chosen": -2811.489501953125,
"logps/rejected": -2596.68310546875,
"loss": 36.8176,
"rewards/accuracies": 0.5099999904632568,
"rewards/chosen": 0.1375296413898468,
"rewards/margins": 0.0820910781621933,
"rewards/rejected": 0.05543852597475052,
"step": 360
},
{
"epoch": 0.48,
"learning_rate": 3.072212717347776e-07,
"logits/chosen": -1.7680120468139648,
"logits/rejected": -1.6781940460205078,
"logps/chosen": -3101.98583984375,
"logps/rejected": -2426.4716796875,
"loss": 36.7837,
"rewards/accuracies": 0.5199999809265137,
"rewards/chosen": 0.022122934460639954,
"rewards/margins": 0.011781491339206696,
"rewards/rejected": 0.010341441258788109,
"step": 370
},
{
"epoch": 0.5,
"learning_rate": 2.9603653128189665e-07,
"logits/chosen": -1.6812299489974976,
"logits/rejected": -1.7215496301651,
"logps/chosen": -2823.8291015625,
"logps/rejected": -2762.53076171875,
"loss": 42.732,
"rewards/accuracies": 0.5,
"rewards/chosen": 0.028385426849126816,
"rewards/margins": -0.006515379063785076,
"rewards/rejected": 0.03490080684423447,
"step": 380
},
{
"epoch": 0.51,
"learning_rate": 2.8475553807115387e-07,
"logits/chosen": -1.8070951700210571,
"logits/rejected": -1.7426990270614624,
"logps/chosen": -2697.833251953125,
"logps/rejected": -2263.9990234375,
"loss": 55.8683,
"rewards/accuracies": 0.5099999904632568,
"rewards/chosen": 0.012206131592392921,
"rewards/margins": 0.011461116373538971,
"rewards/rejected": 0.0007450145785696805,
"step": 390
},
{
"epoch": 0.52,
"learning_rate": 2.7340187829980883e-07,
"logits/chosen": -1.8249183893203735,
"logits/rejected": -1.7130759954452515,
"logps/chosen": -2940.11181640625,
"logps/rejected": -2463.068359375,
"loss": 40.7835,
"rewards/accuracies": 0.6000000238418579,
"rewards/chosen": 0.0059137181378901005,
"rewards/margins": 0.01795141212642193,
"rewards/rejected": -0.012037692591547966,
"step": 400
},
{
"epoch": 0.52,
"eval_logits/chosen": -1.6917269229888916,
"eval_logits/rejected": -1.628839373588562,
"eval_logps/chosen": -2807.42626953125,
"eval_logps/rejected": -2493.92041015625,
"eval_loss": 30.672218322753906,
"eval_rewards/accuracies": 0.5546875,
"eval_rewards/chosen": 0.016848012804985046,
"eval_rewards/margins": 0.019721925258636475,
"eval_rewards/rejected": -0.0028739143162965775,
"eval_runtime": 110.0303,
"eval_samples_per_second": 18.177,
"eval_steps_per_second": 0.291,
"step": 400
},
{
"epoch": 0.54,
"learning_rate": 2.6199929009569996e-07,
"logits/chosen": -1.7034717798233032,
"logits/rejected": -1.707564353942871,
"logps/chosen": -2599.38330078125,
"logps/rejected": -2273.864990234375,
"loss": 43.9981,
"rewards/accuracies": 0.5600000023841858,
"rewards/chosen": 0.02160579524934292,
"rewards/margins": 0.0038177832029759884,
"rewards/rejected": 0.017788011580705643,
"step": 410
},
{
"epoch": 0.55,
"learning_rate": 2.5057161388578505e-07,
"logits/chosen": -1.7964134216308594,
"logits/rejected": -1.730661392211914,
"logps/chosen": -3038.08740234375,
"logps/rejected": -2405.333740234375,
"loss": 31.4477,
"rewards/accuracies": 0.5600000619888306,
"rewards/chosen": 0.05013390630483627,
"rewards/margins": 0.029845798388123512,
"rewards/rejected": 0.02028810977935791,
"step": 420
},
{
"epoch": 0.56,
"learning_rate": 2.391427425507943e-07,
"logits/chosen": -1.6959331035614014,
"logits/rejected": -1.6784296035766602,
"logps/chosen": -2696.2236328125,
"logps/rejected": -2173.48583984375,
"loss": 32.2174,
"rewards/accuracies": 0.5600000619888306,
"rewards/chosen": 0.01727980561554432,
"rewards/margins": 0.013602805323898792,
"rewards/rejected": 0.0036770000588148832,
"step": 430
},
{
"epoch": 0.58,
"learning_rate": 2.2773657147021465e-07,
"logits/chosen": -1.8469693660736084,
"logits/rejected": -1.7459551095962524,
"logps/chosen": -3117.762451171875,
"logps/rejected": -2390.564208984375,
"loss": 37.6526,
"rewards/accuracies": 0.5,
"rewards/chosen": 0.01079155970364809,
"rewards/margins": 0.011436818167567253,
"rewards/rejected": -0.00064525764901191,
"step": 440
},
{
"epoch": 0.59,
"learning_rate": 2.1637694856204885e-07,
"logits/chosen": -1.751587152481079,
"logits/rejected": -1.6395552158355713,
"logps/chosen": -2887.770751953125,
"logps/rejected": -2129.771728515625,
"loss": 53.6906,
"rewards/accuracies": 0.550000011920929,
"rewards/chosen": 0.004514098167419434,
"rewards/margins": 0.00042482782737351954,
"rewards/rejected": 0.004089272115379572,
"step": 450
},
{
"epoch": 0.6,
"learning_rate": 2.0508762442180743e-07,
"logits/chosen": -1.8443762063980103,
"logits/rejected": -1.792295217514038,
"logps/chosen": -2964.06494140625,
"logps/rejected": -2577.50244140625,
"loss": 62.4137,
"rewards/accuracies": 0.5600000023841858,
"rewards/chosen": 0.04290894791483879,
"rewards/margins": 0.01085699163377285,
"rewards/rejected": 0.03205195814371109,
"step": 460
},
{
"epoch": 0.61,
"learning_rate": 1.93892202664981e-07,
"logits/chosen": -1.6403262615203857,
"logits/rejected": -1.712969183921814,
"logps/chosen": -2689.706787109375,
"logps/rejected": -2513.2998046875,
"loss": 31.7885,
"rewards/accuracies": 0.5100000500679016,
"rewards/chosen": 0.01223880797624588,
"rewards/margins": 0.011182873509824276,
"rewards/rejected": 0.0010559323709458113,
"step": 470
},
{
"epoch": 0.63,
"learning_rate": 1.8281409057681686e-07,
"logits/chosen": -1.651449203491211,
"logits/rejected": -1.5920675992965698,
"logps/chosen": -3211.50341796875,
"logps/rejected": -2753.0322265625,
"loss": 103.2519,
"rewards/accuracies": 0.550000011920929,
"rewards/chosen": 0.0340498685836792,
"rewards/margins": 0.005449384916573763,
"rewards/rejected": 0.028600484132766724,
"step": 480
},
{
"epoch": 0.64,
"learning_rate": 1.7187645017258195e-07,
"logits/chosen": -1.823428750038147,
"logits/rejected": -1.7740917205810547,
"logps/chosen": -2745.991455078125,
"logps/rejected": -2407.27978515625,
"loss": 48.2582,
"rewards/accuracies": 0.5300000309944153,
"rewards/chosen": 0.03057839907705784,
"rewards/margins": 0.003420495195314288,
"rewards/rejected": 0.027157902717590332,
"step": 490
},
{
"epoch": 0.65,
"learning_rate": 1.6110214977063343e-07,
"logits/chosen": -1.7967636585235596,
"logits/rejected": -1.7410199642181396,
"logps/chosen": -2905.251708984375,
"logps/rejected": -2435.401611328125,
"loss": 36.2204,
"rewards/accuracies": 0.5,
"rewards/chosen": 0.013903191313147545,
"rewards/margins": 0.00013892585411667824,
"rewards/rejected": 0.013764267787337303,
"step": 500
},
{
"epoch": 0.65,
"eval_logits/chosen": -1.6842743158340454,
"eval_logits/rejected": -1.6236169338226318,
"eval_logps/chosen": -2806.076171875,
"eval_logps/rejected": -2491.544677734375,
"eval_loss": 31.220157623291016,
"eval_rewards/accuracies": 0.53515625,
"eval_rewards/chosen": 0.030346479266881943,
"eval_rewards/margins": 0.009465347044169903,
"eval_rewards/rejected": 0.020881133154034615,
"eval_runtime": 112.3374,
"eval_samples_per_second": 17.804,
"eval_steps_per_second": 0.285,
"step": 500
},
{
"epoch": 0.67,
"learning_rate": 1.5051371617954777e-07,
"logits/chosen": -1.6810442209243774,
"logits/rejected": -1.6596931219100952,
"logps/chosen": -2559.396728515625,
"logps/rejected": -2228.120361328125,
"loss": 44.2046,
"rewards/accuracies": 0.5399999618530273,
"rewards/chosen": 0.016220757737755775,
"rewards/margins": 0.007862111553549767,
"rewards/rejected": 0.00835864432156086,
"step": 510
},
{
"epoch": 0.68,
"learning_rate": 1.4013328759927622e-07,
"logits/chosen": -1.6315361261367798,
"logits/rejected": -1.6191142797470093,
"logps/chosen": -2893.280029296875,
"logps/rejected": -2805.344970703125,
"loss": 31.2508,
"rewards/accuracies": 0.6299999952316284,
"rewards/chosen": 0.02805119752883911,
"rewards/margins": 0.01223050244152546,
"rewards/rejected": 0.0158206969499588,
"step": 520
},
{
"epoch": 0.69,
"learning_rate": 1.2998256733479896e-07,
"logits/chosen": -1.810739278793335,
"logits/rejected": -1.8173195123672485,
"logps/chosen": -2332.383056640625,
"logps/rejected": -1922.5443115234375,
"loss": 226.0483,
"rewards/accuracies": 0.5600000023841858,
"rewards/chosen": 0.020228227600455284,
"rewards/margins": 0.009395391680300236,
"rewards/rejected": 0.010832836851477623,
"step": 530
},
{
"epoch": 0.71,
"learning_rate": 1.200827784190537e-07,
"logits/chosen": -1.6795597076416016,
"logits/rejected": -1.6883628368377686,
"logps/chosen": -3027.91796875,
"logps/rejected": -2619.313232421875,
"loss": 29.3654,
"rewards/accuracies": 0.5600000619888306,
"rewards/chosen": 0.01968817412853241,
"rewards/margins": 0.008831174112856388,
"rewards/rejected": 0.010857000946998596,
"step": 540
},
{
"epoch": 0.72,
"learning_rate": 1.1045461924001323e-07,
"logits/chosen": -1.791738748550415,
"logits/rejected": -1.8031442165374756,
"logps/chosen": -2852.6904296875,
"logps/rejected": -2462.853271484375,
"loss": 45.3966,
"rewards/accuracies": 0.46000003814697266,
"rewards/chosen": 0.010838394984602928,
"rewards/margins": 0.003685446921736002,
"rewards/rejected": 0.007152946200221777,
"step": 550
},
{
"epoch": 0.73,
"learning_rate": 1.0111822026468514e-07,
"logits/chosen": -1.7872514724731445,
"logits/rejected": -1.658860445022583,
"logps/chosen": -2903.530029296875,
"logps/rejected": -2319.64599609375,
"loss": 67.4473,
"rewards/accuracies": 0.5700000524520874,
"rewards/chosen": 0.009608490392565727,
"rewards/margins": 0.004570655524730682,
"rewards/rejected": 0.0050378344021737576,
"step": 560
},
{
"epoch": 0.75,
"learning_rate": 9.209310195051581e-08,
"logits/chosen": -1.8252109289169312,
"logits/rejected": -1.6855742931365967,
"logps/chosen": -2538.860107421875,
"logps/rejected": -1955.6048583984375,
"loss": 63.0174,
"rewards/accuracies": 0.6299999952316284,
"rewards/chosen": 0.04829864576458931,
"rewards/margins": 0.022980675101280212,
"rewards/rejected": 0.025317972525954247,
"step": 570
},
{
"epoch": 0.76,
"learning_rate": 8.339813393219713e-08,
"logits/chosen": -1.739793062210083,
"logits/rejected": -1.641005516052246,
"logps/chosen": -2791.561767578125,
"logps/rejected": -2475.72998046875,
"loss": 59.8369,
"rewards/accuracies": 0.5699999928474426,
"rewards/chosen": 0.05231914669275284,
"rewards/margins": 0.021858692169189453,
"rewards/rejected": 0.030460450798273087,
"step": 580
},
{
"epoch": 0.77,
"learning_rate": 7.505149556920698e-08,
"logits/chosen": -1.8431494235992432,
"logits/rejected": -1.7774893045425415,
"logps/chosen": -2542.13427734375,
"logps/rejected": -2193.825927734375,
"loss": 29.3999,
"rewards/accuracies": 0.5800000429153442,
"rewards/chosen": 0.04752471297979355,
"rewards/margins": 0.017444033175706863,
"rewards/rejected": 0.030080681666731834,
"step": 590
},
{
"epoch": 0.78,
"learning_rate": 6.707063793657064e-08,
"logits/chosen": -1.7773969173431396,
"logits/rejected": -1.6891686916351318,
"logps/chosen": -2942.21240234375,
"logps/rejected": -2429.352294921875,
"loss": 99.7738,
"rewards/accuracies": 0.6200000047683716,
"rewards/chosen": 0.03306427597999573,
"rewards/margins": 0.01405587512999773,
"rewards/rejected": 0.019008399918675423,
"step": 600
},
{
"epoch": 0.78,
"eval_logits/chosen": -1.6827195882797241,
"eval_logits/rejected": -1.6222153902053833,
"eval_logps/chosen": -2804.348388671875,
"eval_logps/rejected": -2489.908935546875,
"eval_loss": 33.74028778076172,
"eval_rewards/accuracies": 0.5390625,
"eval_rewards/chosen": 0.04762275516986847,
"eval_rewards/margins": 0.010385587811470032,
"eval_rewards/rejected": 0.037237171083688736,
"eval_runtime": 106.3716,
"eval_samples_per_second": 18.802,
"eval_steps_per_second": 0.301,
"step": 600
},
{
"epoch": 0.8,
"learning_rate": 5.947224733831363e-08,
"logits/chosen": -1.759399175643921,
"logits/rejected": -1.7431520223617554,
"logps/chosen": -2756.701416015625,
"logps/rejected": -2470.905029296875,
"loss": 51.5387,
"rewards/accuracies": 0.5199999809265137,
"rewards/chosen": 0.011415710672736168,
"rewards/margins": 0.009652274660766125,
"rewards/rejected": 0.0017634350806474686,
"step": 610
},
{
"epoch": 0.81,
"learning_rate": 5.227221041988955e-08,
"logits/chosen": -1.7857062816619873,
"logits/rejected": -1.725630760192871,
"logps/chosen": -2520.410400390625,
"logps/rejected": -2319.78564453125,
"loss": 28.3912,
"rewards/accuracies": 0.5900000333786011,
"rewards/chosen": 0.01811736635863781,
"rewards/margins": 0.01297797542065382,
"rewards/rejected": 0.005139390472322702,
"step": 620
},
{
"epoch": 0.82,
"learning_rate": 4.548558095252758e-08,
"logits/chosen": -1.6374757289886475,
"logits/rejected": -1.673044204711914,
"logps/chosen": -2845.2119140625,
"logps/rejected": -2698.294677734375,
"loss": 42.0619,
"rewards/accuracies": 0.48000001907348633,
"rewards/chosen": 0.030720695853233337,
"rewards/margins": 0.020541973412036896,
"rewards/rejected": 0.010178723372519016,
"step": 630
},
{
"epoch": 0.84,
"learning_rate": 3.9126548358945635e-08,
"logits/chosen": -1.7063062191009521,
"logits/rejected": -1.6988853216171265,
"logps/chosen": -3136.520263671875,
"logps/rejected": -2731.25927734375,
"loss": 46.0608,
"rewards/accuracies": 0.5600000619888306,
"rewards/chosen": 0.029350021854043007,
"rewards/margins": 0.010425332933664322,
"rewards/rejected": 0.018924688920378685,
"step": 640
},
{
"epoch": 0.85,
"learning_rate": 3.3208408046234896e-08,
"logits/chosen": -1.8164535760879517,
"logits/rejected": -1.7656440734863281,
"logps/chosen": -2538.8046875,
"logps/rejected": -2061.81591796875,
"loss": 40.0838,
"rewards/accuracies": 0.550000011920929,
"rewards/chosen": 0.01678399369120598,
"rewards/margins": 0.009666666388511658,
"rewards/rejected": 0.007117328234016895,
"step": 650
},
{
"epoch": 0.86,
"learning_rate": 2.774353360794493e-08,
"logits/chosen": -1.7155154943466187,
"logits/rejected": -1.7442939281463623,
"logps/chosen": -2761.740966796875,
"logps/rejected": -2534.80419921875,
"loss": 36.8374,
"rewards/accuracies": 0.64000004529953,
"rewards/chosen": 0.03588343411684036,
"rewards/margins": 0.037469957023859024,
"rewards/rejected": -0.0015865217428654432,
"step": 660
},
{
"epoch": 0.88,
"learning_rate": 2.2743350953487422e-08,
"logits/chosen": -1.6992709636688232,
"logits/rejected": -1.7416222095489502,
"logps/chosen": -2850.97705078125,
"logps/rejected": -2569.76611328125,
"loss": 86.4259,
"rewards/accuracies": 0.5300000905990601,
"rewards/chosen": 0.018378589302301407,
"rewards/margins": 0.004103804472833872,
"rewards/rejected": 0.014274786226451397,
"step": 670
},
{
"epoch": 0.89,
"learning_rate": 1.8218314418949387e-08,
"logits/chosen": -1.718764305114746,
"logits/rejected": -1.6741740703582764,
"logps/chosen": -2353.093017578125,
"logps/rejected": -2190.77001953125,
"loss": 46.1473,
"rewards/accuracies": 0.5699999928474426,
"rewards/chosen": 0.0031641994137316942,
"rewards/margins": 0.0004996396601200104,
"rewards/rejected": 0.002664559753611684,
"step": 680
},
{
"epoch": 0.9,
"learning_rate": 1.4177884909263277e-08,
"logits/chosen": -1.6867101192474365,
"logits/rejected": -1.652515172958374,
"logps/chosen": -2937.97412109375,
"logps/rejected": -2552.07177734375,
"loss": 37.0029,
"rewards/accuracies": 0.5199999809265137,
"rewards/chosen": 0.007288885302841663,
"rewards/margins": -0.0001541988895041868,
"rewards/rejected": 0.007443083915859461,
"step": 690
},
{
"epoch": 0.92,
"learning_rate": 1.063051011743335e-08,
"logits/chosen": -1.7554800510406494,
"logits/rejected": -1.7419729232788086,
"logps/chosen": -2755.109619140625,
"logps/rejected": -2368.246337890625,
"loss": 41.8506,
"rewards/accuracies": 0.46000003814697266,
"rewards/chosen": 0.006798497401177883,
"rewards/margins": 0.010020612739026546,
"rewards/rejected": -0.0032221146393567324,
"step": 700
},
{
"epoch": 0.92,
"eval_logits/chosen": -1.682308554649353,
"eval_logits/rejected": -1.6210675239562988,
"eval_logps/chosen": -2806.1005859375,
"eval_logps/rejected": -2491.68505859375,
"eval_loss": 32.91334915161133,
"eval_rewards/accuracies": 0.5546875,
"eval_rewards/chosen": 0.030103469267487526,
"eval_rewards/margins": 0.01062812004238367,
"eval_rewards/rejected": 0.01947534643113613,
"eval_runtime": 110.9725,
"eval_samples_per_second": 18.022,
"eval_steps_per_second": 0.288,
"step": 700
},
{
"epoch": 0.93,
"learning_rate": 7.58360686217671e-09,
"logits/chosen": -1.7843902111053467,
"logits/rejected": -1.6857761144638062,
"logps/chosen": -2821.310546875,
"logps/rejected": -2445.586669921875,
"loss": 44.0616,
"rewards/accuracies": 0.6000000238418579,
"rewards/chosen": 0.025277357548475266,
"rewards/margins": 0.014521745964884758,
"rewards/rejected": 0.010755611583590508,
"step": 710
},
{
"epoch": 0.94,
"learning_rate": 5.043545580906694e-09,
"logits/chosen": -1.7102206945419312,
"logits/rejected": -1.6009165048599243,
"logps/chosen": -2682.336669921875,
"logps/rejected": -2234.36279296875,
"loss": 44.9558,
"rewards/accuracies": 0.6299999952316284,
"rewards/chosen": 0.018401915207505226,
"rewards/margins": 0.030992329120635986,
"rewards/rejected": -0.012590417638421059,
"step": 720
},
{
"epoch": 0.95,
"learning_rate": 3.015637010480576e-09,
"logits/chosen": -1.7701480388641357,
"logits/rejected": -1.7436597347259521,
"logps/chosen": -3042.88330078125,
"logps/rejected": -2501.55615234375,
"loss": 35.3819,
"rewards/accuracies": 0.5199999809265137,
"rewards/chosen": 0.012798592448234558,
"rewards/margins": -0.0020437492057681084,
"rewards/rejected": 0.014842341654002666,
"step": 730
},
{
"epoch": 0.97,
"learning_rate": 1.5041210835596285e-09,
"logits/chosen": -1.703537940979004,
"logits/rejected": -1.6897165775299072,
"logps/chosen": -2817.1484375,
"logps/rejected": -2390.23095703125,
"loss": 55.7217,
"rewards/accuracies": 0.5800000429153442,
"rewards/chosen": 0.022753870114684105,
"rewards/margins": 0.017634030431509018,
"rewards/rejected": 0.005119838751852512,
"step": 740
},
{
"epoch": 0.98,
"learning_rate": 5.121580637968137e-10,
"logits/chosen": -1.7322509288787842,
"logits/rejected": -1.6345192193984985,
"logps/chosen": -2836.63623046875,
"logps/rejected": -2363.93505859375,
"loss": 67.2692,
"rewards/accuracies": 0.6300000548362732,
"rewards/chosen": 0.02889620140194893,
"rewards/margins": 0.014323192648589611,
"rewards/rejected": 0.014573007822036743,
"step": 750
},
{
"epoch": 0.99,
"learning_rate": 4.1821938386477075e-11,
"logits/chosen": -1.7962977886199951,
"logits/rejected": -1.7127879858016968,
"logps/chosen": -2801.806640625,
"logps/rejected": -2341.844970703125,
"loss": 46.0674,
"rewards/accuracies": 0.5700000524520874,
"rewards/chosen": 0.009056088514626026,
"rewards/margins": 0.013271180912852287,
"rewards/rejected": -0.004215092398226261,
"step": 760
},
{
"epoch": 1.0,
"step": 764,
"total_flos": 0.0,
"train_loss": 46.613421885577296,
"train_runtime": 4597.6924,
"train_samples_per_second": 13.297,
"train_steps_per_second": 0.166
}
],
"logging_steps": 10,
"max_steps": 764,
"num_input_tokens_seen": 0,
"num_train_epochs": 1,
"save_steps": 100,
"total_flos": 0.0,
"train_batch_size": 5,
"trial_name": null,
"trial_params": null
}