zephyr-7b-dpo-qlora / trainer_state.json
shenxq's picture
Model save
29be7a4 verified
raw
history blame
67.9 kB
{
"best_metric": null,
"best_model_checkpoint": null,
"epoch": 0.9993998799759952,
"eval_steps": 100,
"global_step": 1249,
"is_hyper_param_search": false,
"is_local_process_zero": true,
"is_world_process_zero": true,
"log_history": [
{
"epoch": 0.0,
"learning_rate": 4e-08,
"logits/chosen": -2.5782995223999023,
"logits/rejected": -2.5818521976470947,
"logps/chosen": -405.7000427246094,
"logps/rejected": -395.45977783203125,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": 0.0,
"rewards/margins": 0.0,
"rewards/rejected": 0.0,
"step": 1
},
{
"epoch": 0.01,
"learning_rate": 4.0000000000000003e-07,
"logits/chosen": -2.6189024448394775,
"logits/rejected": -2.6425905227661133,
"logps/chosen": -430.23095703125,
"logps/rejected": -465.6946105957031,
"loss": 0.6937,
"rewards/accuracies": 0.3402777910232544,
"rewards/chosen": -0.0004277642001397908,
"rewards/margins": -0.0011669597588479519,
"rewards/rejected": 0.0007391955587081611,
"step": 10
},
{
"epoch": 0.02,
"learning_rate": 8.000000000000001e-07,
"logits/chosen": -2.565629482269287,
"logits/rejected": -2.587047576904297,
"logps/chosen": -392.698974609375,
"logps/rejected": -420.3116149902344,
"loss": 0.6932,
"rewards/accuracies": 0.5,
"rewards/chosen": 0.0005297298775985837,
"rewards/margins": -9.570456313667819e-05,
"rewards/rejected": 0.0006254343315958977,
"step": 20
},
{
"epoch": 0.02,
"learning_rate": 1.2000000000000002e-06,
"logits/chosen": -2.5378575325012207,
"logits/rejected": -2.594695568084717,
"logps/chosen": -418.8670349121094,
"logps/rejected": -452.1278381347656,
"loss": 0.6935,
"rewards/accuracies": 0.44999998807907104,
"rewards/chosen": -0.0007869214750826359,
"rewards/margins": -0.0006720771198160946,
"rewards/rejected": -0.00011484431888675317,
"step": 30
},
{
"epoch": 0.03,
"learning_rate": 1.6000000000000001e-06,
"logits/chosen": -2.53520131111145,
"logits/rejected": -2.5557379722595215,
"logps/chosen": -387.7684326171875,
"logps/rejected": -400.6465759277344,
"loss": 0.6932,
"rewards/accuracies": 0.4937500059604645,
"rewards/chosen": -0.0008191015804186463,
"rewards/margins": -7.259581616381183e-05,
"rewards/rejected": -0.0007465056842193007,
"step": 40
},
{
"epoch": 0.04,
"learning_rate": 2.0000000000000003e-06,
"logits/chosen": -2.5300962924957275,
"logits/rejected": -2.5252957344055176,
"logps/chosen": -379.98785400390625,
"logps/rejected": -387.24755859375,
"loss": 0.6933,
"rewards/accuracies": 0.4625000059604645,
"rewards/chosen": -0.001431723590940237,
"rewards/margins": -0.00021147821098566055,
"rewards/rejected": -0.0012202452635392547,
"step": 50
},
{
"epoch": 0.05,
"learning_rate": 2.4000000000000003e-06,
"logits/chosen": -2.5475428104400635,
"logits/rejected": -2.571965456008911,
"logps/chosen": -404.24560546875,
"logps/rejected": -431.3631896972656,
"loss": 0.6926,
"rewards/accuracies": 0.53125,
"rewards/chosen": -0.0013539999490603805,
"rewards/margins": 0.0010300865396857262,
"rewards/rejected": -0.0023840866051614285,
"step": 60
},
{
"epoch": 0.06,
"learning_rate": 2.8000000000000003e-06,
"logits/chosen": -2.5969393253326416,
"logits/rejected": -2.601594924926758,
"logps/chosen": -404.23236083984375,
"logps/rejected": -410.1368713378906,
"loss": 0.6927,
"rewards/accuracies": 0.550000011920929,
"rewards/chosen": -0.003309990745037794,
"rewards/margins": 0.0009536609286442399,
"rewards/rejected": -0.004263651557266712,
"step": 70
},
{
"epoch": 0.06,
"learning_rate": 3.2000000000000003e-06,
"logits/chosen": -2.6136419773101807,
"logits/rejected": -2.63590669631958,
"logps/chosen": -396.66680908203125,
"logps/rejected": -420.2757263183594,
"loss": 0.693,
"rewards/accuracies": 0.4749999940395355,
"rewards/chosen": -0.0044545335695147514,
"rewards/margins": 0.0003312411135993898,
"rewards/rejected": -0.004785774741321802,
"step": 80
},
{
"epoch": 0.07,
"learning_rate": 3.6000000000000003e-06,
"logits/chosen": -2.6318578720092773,
"logits/rejected": -2.627044677734375,
"logps/chosen": -447.1123962402344,
"logps/rejected": -461.43994140625,
"loss": 0.6926,
"rewards/accuracies": 0.543749988079071,
"rewards/chosen": -0.005481979344040155,
"rewards/margins": 0.0011420813389122486,
"rewards/rejected": -0.006624061614274979,
"step": 90
},
{
"epoch": 0.08,
"learning_rate": 4.000000000000001e-06,
"logits/chosen": -2.6037490367889404,
"logits/rejected": -2.600301504135132,
"logps/chosen": -396.48162841796875,
"logps/rejected": -414.06121826171875,
"loss": 0.6933,
"rewards/accuracies": 0.4749999940395355,
"rewards/chosen": -0.007383377756923437,
"rewards/margins": -0.0003390449855942279,
"rewards/rejected": -0.007044332567602396,
"step": 100
},
{
"epoch": 0.08,
"eval_logits/chosen": -2.4768383502960205,
"eval_logits/rejected": -2.483818292617798,
"eval_logps/chosen": -335.0983581542969,
"eval_logps/rejected": -344.84783935546875,
"eval_loss": 0.6929709315299988,
"eval_rewards/accuracies": 0.5176666378974915,
"eval_rewards/chosen": -0.007662294432520866,
"eval_rewards/margins": 0.0003858131531160325,
"eval_rewards/rejected": -0.008048107847571373,
"eval_runtime": 1616.6225,
"eval_samples_per_second": 1.852,
"eval_steps_per_second": 0.232,
"step": 100
},
{
"epoch": 0.09,
"learning_rate": 4.4e-06,
"logits/chosen": -2.6104788780212402,
"logits/rejected": -2.6167144775390625,
"logps/chosen": -433.07000732421875,
"logps/rejected": -446.54840087890625,
"loss": 0.6926,
"rewards/accuracies": 0.574999988079071,
"rewards/chosen": -0.009052934125065804,
"rewards/margins": 0.0011820209911093116,
"rewards/rejected": -0.01023495476692915,
"step": 110
},
{
"epoch": 0.1,
"learning_rate": 4.800000000000001e-06,
"logits/chosen": -2.603712797164917,
"logits/rejected": -2.6226086616516113,
"logps/chosen": -402.2064208984375,
"logps/rejected": -420.73345947265625,
"loss": 0.6929,
"rewards/accuracies": 0.5,
"rewards/chosen": -0.014477076940238476,
"rewards/margins": 0.0004566563293337822,
"rewards/rejected": -0.014933735132217407,
"step": 120
},
{
"epoch": 0.1,
"learning_rate": 4.999755876225375e-06,
"logits/chosen": -2.5681347846984863,
"logits/rejected": -2.575467348098755,
"logps/chosen": -426.24334716796875,
"logps/rejected": -450.0921936035156,
"loss": 0.6923,
"rewards/accuracies": 0.5687500238418579,
"rewards/chosen": -0.014384260401129723,
"rewards/margins": 0.0017711615655571222,
"rewards/rejected": -0.0161554254591465,
"step": 130
},
{
"epoch": 0.11,
"learning_rate": 4.997803172081864e-06,
"logits/chosen": -2.549048900604248,
"logits/rejected": -2.562979221343994,
"logps/chosen": -415.5914001464844,
"logps/rejected": -434.4419860839844,
"loss": 0.6928,
"rewards/accuracies": 0.5625,
"rewards/chosen": -0.0187313761562109,
"rewards/margins": 0.0007000064942985773,
"rewards/rejected": -0.019431382417678833,
"step": 140
},
{
"epoch": 0.12,
"learning_rate": 4.9938992891651825e-06,
"logits/chosen": -2.571101665496826,
"logits/rejected": -2.579190254211426,
"logps/chosen": -396.6254577636719,
"logps/rejected": -428.6380310058594,
"loss": 0.6919,
"rewards/accuracies": 0.5687500238418579,
"rewards/chosen": -0.014721980318427086,
"rewards/margins": 0.0025038637686520815,
"rewards/rejected": -0.0172258447855711,
"step": 150
},
{
"epoch": 0.13,
"learning_rate": 4.988047277024456e-06,
"logits/chosen": -2.6119332313537598,
"logits/rejected": -2.6318445205688477,
"logps/chosen": -410.3868713378906,
"logps/rejected": -429.6300354003906,
"loss": 0.6922,
"rewards/accuracies": 0.5625,
"rewards/chosen": -0.01405310444533825,
"rewards/margins": 0.001986552495509386,
"rewards/rejected": -0.016039658337831497,
"step": 160
},
{
"epoch": 0.14,
"learning_rate": 4.980251707005417e-06,
"logits/chosen": -2.6066763401031494,
"logits/rejected": -2.602252721786499,
"logps/chosen": -439.21417236328125,
"logps/rejected": -450.12139892578125,
"loss": 0.6931,
"rewards/accuracies": 0.4625000059604645,
"rewards/chosen": -0.016262266784906387,
"rewards/margins": 0.0002180577430408448,
"rewards/rejected": -0.016480324789881706,
"step": 170
},
{
"epoch": 0.14,
"learning_rate": 4.970518668679459e-06,
"logits/chosen": -2.5717625617980957,
"logits/rejected": -2.564636468887329,
"logps/chosen": -430.0758361816406,
"logps/rejected": -434.64593505859375,
"loss": 0.6914,
"rewards/accuracies": 0.625,
"rewards/chosen": -0.011936081573367119,
"rewards/margins": 0.0035087396390736103,
"rewards/rejected": -0.015444819815456867,
"step": 180
},
{
"epoch": 0.15,
"learning_rate": 4.958855765086722e-06,
"logits/chosen": -2.589442491531372,
"logits/rejected": -2.5929834842681885,
"logps/chosen": -402.8868408203125,
"logps/rejected": -411.501708984375,
"loss": 0.6922,
"rewards/accuracies": 0.5625,
"rewards/chosen": -0.01052446011453867,
"rewards/margins": 0.0019376948475837708,
"rewards/rejected": -0.01246215496212244,
"step": 190
},
{
"epoch": 0.16,
"learning_rate": 4.945272106796919e-06,
"logits/chosen": -2.5665698051452637,
"logits/rejected": -2.5826125144958496,
"logps/chosen": -399.679443359375,
"logps/rejected": -424.7906188964844,
"loss": 0.6926,
"rewards/accuracies": 0.581250011920929,
"rewards/chosen": -0.01273694820702076,
"rewards/margins": 0.0011030982714146376,
"rewards/rejected": -0.01384004671126604,
"step": 200
},
{
"epoch": 0.16,
"eval_logits/chosen": -2.476593017578125,
"eval_logits/rejected": -2.4835994243621826,
"eval_logps/chosen": -335.71136474609375,
"eval_logps/rejected": -345.59197998046875,
"eval_loss": 0.6923276782035828,
"eval_rewards/accuracies": 0.5426666736602783,
"eval_rewards/chosen": -0.013792484067380428,
"eval_rewards/margins": 0.0016968329437077045,
"eval_rewards/rejected": -0.015489316545426846,
"eval_runtime": 1616.1511,
"eval_samples_per_second": 1.853,
"eval_steps_per_second": 0.232,
"step": 200
},
{
"epoch": 0.17,
"learning_rate": 4.929778304792537e-06,
"logits/chosen": -2.58109974861145,
"logits/rejected": -2.592872381210327,
"logps/chosen": -438.9920959472656,
"logps/rejected": -439.1341247558594,
"loss": 0.6907,
"rewards/accuracies": 0.59375,
"rewards/chosen": -0.017467942088842392,
"rewards/margins": 0.005076024681329727,
"rewards/rejected": -0.02254396490752697,
"step": 210
},
{
"epoch": 0.18,
"learning_rate": 4.912386462179987e-06,
"logits/chosen": -2.6212189197540283,
"logits/rejected": -2.6276934146881104,
"logps/chosen": -417.43359375,
"logps/rejected": -453.41119384765625,
"loss": 0.6903,
"rewards/accuracies": 0.6000000238418579,
"rewards/chosen": -0.022168174386024475,
"rewards/margins": 0.00578011991456151,
"rewards/rejected": -0.027948295697569847,
"step": 220
},
{
"epoch": 0.18,
"learning_rate": 4.893110164735167e-06,
"logits/chosen": -2.637918710708618,
"logits/rejected": -2.6529829502105713,
"logps/chosen": -433.37969970703125,
"logps/rejected": -449.2330627441406,
"loss": 0.6913,
"rewards/accuracies": 0.5562499761581421,
"rewards/chosen": -0.02808026596903801,
"rewards/margins": 0.003863722085952759,
"rewards/rejected": -0.03194398805499077,
"step": 230
},
{
"epoch": 0.19,
"learning_rate": 4.871964470290823e-06,
"logits/chosen": -2.5649704933166504,
"logits/rejected": -2.590609073638916,
"logps/chosen": -433.03271484375,
"logps/rejected": -454.35528564453125,
"loss": 0.6906,
"rewards/accuracies": 0.5562499761581421,
"rewards/chosen": -0.02802122011780739,
"rewards/margins": 0.005218953359872103,
"rewards/rejected": -0.0332401767373085,
"step": 240
},
{
"epoch": 0.2,
"learning_rate": 4.848965896974006e-06,
"logits/chosen": -2.61820650100708,
"logits/rejected": -2.6432509422302246,
"logps/chosen": -416.4977111816406,
"logps/rejected": -439.9608459472656,
"loss": 0.693,
"rewards/accuracies": 0.53125,
"rewards/chosen": -0.03262058645486832,
"rewards/margins": 0.0004899540217593312,
"rewards/rejected": -0.033110540360212326,
"step": 250
},
{
"epoch": 0.21,
"learning_rate": 4.8241324103028055e-06,
"logits/chosen": -2.6041173934936523,
"logits/rejected": -2.574268341064453,
"logps/chosen": -422.29437255859375,
"logps/rejected": -432.9288635253906,
"loss": 0.6911,
"rewards/accuracies": 0.59375,
"rewards/chosen": -0.03160303831100464,
"rewards/margins": 0.00435088574886322,
"rewards/rejected": -0.03595392405986786,
"step": 260
},
{
"epoch": 0.22,
"learning_rate": 4.797483409152438e-06,
"logits/chosen": -2.6138675212860107,
"logits/rejected": -2.632070541381836,
"logps/chosen": -403.8775939941406,
"logps/rejected": -425.6439514160156,
"loss": 0.6899,
"rewards/accuracies": 0.5687500238418579,
"rewards/chosen": -0.024003220722079277,
"rewards/margins": 0.0069326674565672874,
"rewards/rejected": -0.03093588910996914,
"step": 270
},
{
"epoch": 0.22,
"learning_rate": 4.769039710601669e-06,
"logits/chosen": -2.617337226867676,
"logits/rejected": -2.6075143814086914,
"logps/chosen": -399.85308837890625,
"logps/rejected": -415.65460205078125,
"loss": 0.69,
"rewards/accuracies": 0.625,
"rewards/chosen": -0.022890212014317513,
"rewards/margins": 0.006663457490503788,
"rewards/rejected": -0.029553670436143875,
"step": 280
},
{
"epoch": 0.23,
"learning_rate": 4.738823533671383e-06,
"logits/chosen": -2.607677936553955,
"logits/rejected": -2.626984119415283,
"logps/chosen": -433.34503173828125,
"logps/rejected": -455.47113037109375,
"loss": 0.6922,
"rewards/accuracies": 0.512499988079071,
"rewards/chosen": -0.016143614426255226,
"rewards/margins": 0.0021604117937386036,
"rewards/rejected": -0.018304025754332542,
"step": 290
},
{
"epoch": 0.24,
"learning_rate": 4.706858481968017e-06,
"logits/chosen": -2.5796170234680176,
"logits/rejected": -2.5774731636047363,
"logps/chosen": -412.5723571777344,
"logps/rejected": -420.7684631347656,
"loss": 0.6906,
"rewards/accuracies": 0.53125,
"rewards/chosen": -0.015171040780842304,
"rewards/margins": 0.005369971971958876,
"rewards/rejected": -0.020541010424494743,
"step": 300
},
{
"epoch": 0.24,
"eval_logits/chosen": -2.4809017181396484,
"eval_logits/rejected": -2.487875461578369,
"eval_logps/chosen": -335.6323547363281,
"eval_logps/rejected": -345.656005859375,
"eval_loss": 0.691673219203949,
"eval_rewards/accuracies": 0.5523333549499512,
"eval_rewards/chosen": -0.01300230622291565,
"eval_rewards/margins": 0.0031275488436222076,
"eval_rewards/rejected": -0.016129855066537857,
"eval_runtime": 1615.9746,
"eval_samples_per_second": 1.853,
"eval_steps_per_second": 0.232,
"step": 300
},
{
"epoch": 0.25,
"learning_rate": 4.673169525245416e-06,
"logits/chosen": -2.5858876705169678,
"logits/rejected": -2.5731964111328125,
"logps/chosen": -414.9142150878906,
"logps/rejected": -443.6748962402344,
"loss": 0.6902,
"rewards/accuracies": 0.6000000238418579,
"rewards/chosen": -0.010048646479845047,
"rewards/margins": 0.006063834764063358,
"rewards/rejected": -0.01611248217523098,
"step": 310
},
{
"epoch": 0.26,
"learning_rate": 4.63778297989952e-06,
"logits/chosen": -2.5999462604522705,
"logits/rejected": -2.6060266494750977,
"logps/chosen": -412.861328125,
"logps/rejected": -434.5308532714844,
"loss": 0.689,
"rewards/accuracies": 0.606249988079071,
"rewards/chosen": -0.006168536841869354,
"rewards/margins": 0.00842990167438984,
"rewards/rejected": -0.014598439447581768,
"step": 320
},
{
"epoch": 0.26,
"learning_rate": 4.60072648841109e-06,
"logits/chosen": -2.58223295211792,
"logits/rejected": -2.5768980979919434,
"logps/chosen": -420.7120666503906,
"logps/rejected": -439.38421630859375,
"loss": 0.6882,
"rewards/accuracies": 0.5874999761581421,
"rewards/chosen": -0.00741790235042572,
"rewards/margins": 0.01021712925285101,
"rewards/rejected": -0.017635030671954155,
"step": 330
},
{
"epoch": 0.27,
"learning_rate": 4.562028997752574e-06,
"logits/chosen": -2.5425148010253906,
"logits/rejected": -2.5803589820861816,
"logps/chosen": -418.337890625,
"logps/rejected": -450.64459228515625,
"loss": 0.6885,
"rewards/accuracies": 0.6499999761581421,
"rewards/chosen": 0.0036701434291899204,
"rewards/margins": 0.009488197974860668,
"rewards/rejected": -0.0058180540800094604,
"step": 340
},
{
"epoch": 0.28,
"learning_rate": 4.521720736775947e-06,
"logits/chosen": -2.5993804931640625,
"logits/rejected": -2.5906174182891846,
"logps/chosen": -435.66845703125,
"logps/rejected": -436.15594482421875,
"loss": 0.6903,
"rewards/accuracies": 0.5874999761581421,
"rewards/chosen": -0.003759814891964197,
"rewards/margins": 0.006070856936275959,
"rewards/rejected": -0.009830671362578869,
"step": 350
},
{
"epoch": 0.29,
"learning_rate": 4.479833192599198e-06,
"logits/chosen": -2.6347668170928955,
"logits/rejected": -2.628312110900879,
"logps/chosen": -444.01788330078125,
"logps/rejected": -462.4187927246094,
"loss": 0.688,
"rewards/accuracies": 0.6187499761581421,
"rewards/chosen": 0.007655065506696701,
"rewards/margins": 0.010737383738160133,
"rewards/rejected": -0.003082319162786007,
"step": 360
},
{
"epoch": 0.3,
"learning_rate": 4.436399086009928e-06,
"logits/chosen": -2.65555477142334,
"logits/rejected": -2.6738853454589844,
"logps/chosen": -417.18560791015625,
"logps/rejected": -433.4161682128906,
"loss": 0.6885,
"rewards/accuracies": 0.6625000238418579,
"rewards/chosen": -0.007278016768395901,
"rewards/margins": 0.009610911831259727,
"rewards/rejected": -0.016888927668333054,
"step": 370
},
{
"epoch": 0.3,
"learning_rate": 4.391452345905239e-06,
"logits/chosen": -2.5911710262298584,
"logits/rejected": -2.6055374145507812,
"logps/chosen": -426.5325622558594,
"logps/rejected": -434.95758056640625,
"loss": 0.6903,
"rewards/accuracies": 0.59375,
"rewards/chosen": -0.020730096846818924,
"rewards/margins": 0.006341870874166489,
"rewards/rejected": -0.027071967720985413,
"step": 380
},
{
"epoch": 0.31,
"learning_rate": 4.3450280827879125e-06,
"logits/chosen": -2.6074564456939697,
"logits/rejected": -2.614403486251831,
"logps/chosen": -418.90106201171875,
"logps/rejected": -433.29046630859375,
"loss": 0.6884,
"rewards/accuracies": 0.612500011920929,
"rewards/chosen": -0.016112228855490685,
"rewards/margins": 0.01000655721873045,
"rewards/rejected": -0.02611878514289856,
"step": 390
},
{
"epoch": 0.32,
"learning_rate": 4.297162561339554e-06,
"logits/chosen": -2.5204920768737793,
"logits/rejected": -2.5613515377044678,
"logps/chosen": -425.9031677246094,
"logps/rejected": -446.0582580566406,
"loss": 0.6884,
"rewards/accuracies": 0.581250011920929,
"rewards/chosen": -0.00986536405980587,
"rewards/margins": 0.010156005620956421,
"rewards/rejected": -0.02002136781811714,
"step": 400
},
{
"epoch": 0.32,
"eval_logits/chosen": -2.490081787109375,
"eval_logits/rejected": -2.497150182723999,
"eval_logps/chosen": -335.07940673828125,
"eval_logps/rejected": -345.4989929199219,
"eval_loss": 0.6898096203804016,
"eval_rewards/accuracies": 0.5806666612625122,
"eval_rewards/chosen": -0.007472673896700144,
"eval_rewards/margins": 0.007086611352860928,
"eval_rewards/rejected": -0.014559285715222359,
"eval_runtime": 1615.277,
"eval_samples_per_second": 1.854,
"eval_steps_per_second": 0.232,
"step": 400
},
{
"epoch": 0.33,
"learning_rate": 4.247893172092157e-06,
"logits/chosen": -2.556941509246826,
"logits/rejected": -2.578604221343994,
"logps/chosen": -409.3045959472656,
"logps/rejected": -440.68914794921875,
"loss": 0.6865,
"rewards/accuracies": 0.606249988079071,
"rewards/chosen": -0.01420166902244091,
"rewards/margins": 0.013790594413876534,
"rewards/rejected": -0.027992259711027145,
"step": 410
},
{
"epoch": 0.34,
"learning_rate": 4.197258402220187e-06,
"logits/chosen": -2.5752522945404053,
"logits/rejected": -2.5826656818389893,
"logps/chosen": -419.6229553222656,
"logps/rejected": -430.58648681640625,
"loss": 0.6909,
"rewards/accuracies": 0.574999988079071,
"rewards/chosen": -0.03458824008703232,
"rewards/margins": 0.005263219587504864,
"rewards/rejected": -0.03985145688056946,
"step": 420
},
{
"epoch": 0.34,
"learning_rate": 4.145297805476023e-06,
"logits/chosen": -2.6343460083007812,
"logits/rejected": -2.621222972869873,
"logps/chosen": -417.43017578125,
"logps/rejected": -432.63818359375,
"loss": 0.6866,
"rewards/accuracies": 0.6499999761581421,
"rewards/chosen": -0.026808559894561768,
"rewards/margins": 0.013837064616382122,
"rewards/rejected": -0.040645625442266464,
"step": 430
},
{
"epoch": 0.35,
"learning_rate": 4.092051971292228e-06,
"logits/chosen": -2.593862771987915,
"logits/rejected": -2.611497163772583,
"logps/chosen": -407.7058410644531,
"logps/rejected": -427.201171875,
"loss": 0.6863,
"rewards/accuracies": 0.5687500238418579,
"rewards/chosen": -0.03197476640343666,
"rewards/margins": 0.015016555786132812,
"rewards/rejected": -0.04699132218956947,
"step": 440
},
{
"epoch": 0.36,
"learning_rate": 4.037562493074792e-06,
"logits/chosen": -2.5881259441375732,
"logits/rejected": -2.584801435470581,
"logps/chosen": -435.7823181152344,
"logps/rejected": -439.61297607421875,
"loss": 0.6835,
"rewards/accuracies": 0.637499988079071,
"rewards/chosen": -0.042981140315532684,
"rewards/margins": 0.020602630451321602,
"rewards/rejected": -0.06358377635478973,
"step": 450
},
{
"epoch": 0.37,
"learning_rate": 3.981871935712112e-06,
"logits/chosen": -2.6311769485473633,
"logits/rejected": -2.6546669006347656,
"logps/chosen": -413.837646484375,
"logps/rejected": -437.16986083984375,
"loss": 0.6839,
"rewards/accuracies": 0.637499988079071,
"rewards/chosen": -0.03683266043663025,
"rewards/margins": 0.019854258745908737,
"rewards/rejected": -0.056686919182538986,
"step": 460
},
{
"epoch": 0.38,
"learning_rate": 3.925023802325094e-06,
"logits/chosen": -2.6563210487365723,
"logits/rejected": -2.6776556968688965,
"logps/chosen": -437.06988525390625,
"logps/rejected": -455.8389587402344,
"loss": 0.6788,
"rewards/accuracies": 0.606249988079071,
"rewards/chosen": -0.052065037190914154,
"rewards/margins": 0.03126855939626694,
"rewards/rejected": -0.08333360403776169,
"step": 470
},
{
"epoch": 0.38,
"learning_rate": 3.867062500284342e-06,
"logits/chosen": -2.6086373329162598,
"logits/rejected": -2.623145818710327,
"logps/chosen": -424.25,
"logps/rejected": -455.4474182128906,
"loss": 0.6785,
"rewards/accuracies": 0.6312500238418579,
"rewards/chosen": -0.10639921575784683,
"rewards/margins": 0.033148206770420074,
"rewards/rejected": -0.1395474076271057,
"step": 480
},
{
"epoch": 0.39,
"learning_rate": 3.8080333065209885e-06,
"logits/chosen": -2.6073594093322754,
"logits/rejected": -2.6054632663726807,
"logps/chosen": -428.15185546875,
"logps/rejected": -422.6666564941406,
"loss": 0.6961,
"rewards/accuracies": 0.48750001192092896,
"rewards/chosen": -0.1707339584827423,
"rewards/margins": 0.0015567988157272339,
"rewards/rejected": -0.17229075729846954,
"step": 490
},
{
"epoch": 0.4,
"learning_rate": 3.7479823321582624e-06,
"logits/chosen": -2.6313910484313965,
"logits/rejected": -2.6622931957244873,
"logps/chosen": -435.267578125,
"logps/rejected": -466.4607849121094,
"loss": 0.6753,
"rewards/accuracies": 0.5874999761581421,
"rewards/chosen": -0.16189143061637878,
"rewards/margins": 0.04039152339100838,
"rewards/rejected": -0.20228295028209686,
"step": 500
},
{
"epoch": 0.4,
"eval_logits/chosen": -2.491628885269165,
"eval_logits/rejected": -2.4985597133636475,
"eval_logps/chosen": -348.1782531738281,
"eval_logps/rejected": -359.8316650390625,
"eval_loss": 0.6855587959289551,
"eval_rewards/accuracies": 0.5630000233650208,
"eval_rewards/chosen": -0.13846127688884735,
"eval_rewards/margins": 0.019424747675657272,
"eval_rewards/rejected": -0.15788602828979492,
"eval_runtime": 1618.1001,
"eval_samples_per_second": 1.85,
"eval_steps_per_second": 0.232,
"step": 500
},
{
"epoch": 0.41,
"learning_rate": 3.686956486491419e-06,
"logits/chosen": -2.585524082183838,
"logits/rejected": -2.593437671661377,
"logps/chosen": -448.7865295410156,
"logps/rejected": -462.56890869140625,
"loss": 0.6939,
"rewards/accuracies": 0.5249999761581421,
"rewards/chosen": -0.1920996755361557,
"rewards/margins": 0.004705216269940138,
"rewards/rejected": -0.19680488109588623,
"step": 510
},
{
"epoch": 0.42,
"learning_rate": 3.625003440344166e-06,
"logits/chosen": -2.603358268737793,
"logits/rejected": -2.620025157928467,
"logps/chosen": -422.2713317871094,
"logps/rejected": -432.52557373046875,
"loss": 0.678,
"rewards/accuracies": 0.625,
"rewards/chosen": -0.14511428773403168,
"rewards/margins": 0.03498765081167221,
"rewards/rejected": -0.18010194599628448,
"step": 520
},
{
"epoch": 0.42,
"learning_rate": 3.562171588830231e-06,
"logits/chosen": -2.666226625442505,
"logits/rejected": -2.6871166229248047,
"logps/chosen": -434.64678955078125,
"logps/rejected": -468.7852478027344,
"loss": 0.6804,
"rewards/accuracies": 0.6312500238418579,
"rewards/chosen": -0.13358792662620544,
"rewards/margins": 0.031595379114151,
"rewards/rejected": -0.16518330574035645,
"step": 530
},
{
"epoch": 0.43,
"learning_rate": 3.4985100135491245e-06,
"logits/chosen": -2.678806781768799,
"logits/rejected": -2.698000907897949,
"logps/chosen": -453.0816955566406,
"logps/rejected": -490.1700134277344,
"loss": 0.6683,
"rewards/accuracies": 0.6000000238418579,
"rewards/chosen": -0.19373668730258942,
"rewards/margins": 0.05805329233407974,
"rewards/rejected": -0.25178998708724976,
"step": 540
},
{
"epoch": 0.44,
"learning_rate": 3.4340684442456673e-06,
"logits/chosen": -2.5824878215789795,
"logits/rejected": -2.600515842437744,
"logps/chosen": -450.08758544921875,
"logps/rejected": -470.90838623046875,
"loss": 0.6789,
"rewards/accuracies": 0.612500011920929,
"rewards/chosen": -0.30373430252075195,
"rewards/margins": 0.03900415450334549,
"rewards/rejected": -0.34273844957351685,
"step": 550
},
{
"epoch": 0.45,
"learning_rate": 3.3688972199631974e-06,
"logits/chosen": -2.632826328277588,
"logits/rejected": -2.6091580390930176,
"logps/chosen": -455.7169494628906,
"logps/rejected": -464.70684814453125,
"loss": 0.6806,
"rewards/accuracies": 0.5874999761581421,
"rewards/chosen": -0.35484108328819275,
"rewards/margins": 0.0392206534743309,
"rewards/rejected": -0.39406174421310425,
"step": 560
},
{
"epoch": 0.46,
"learning_rate": 3.3030472497208354e-06,
"logits/chosen": -2.62416672706604,
"logits/rejected": -2.6495113372802734,
"logps/chosen": -439.15106201171875,
"logps/rejected": -497.6800842285156,
"loss": 0.6604,
"rewards/accuracies": 0.6312500238418579,
"rewards/chosen": -0.32003355026245117,
"rewards/margins": 0.07973183691501617,
"rewards/rejected": -0.39976540207862854,
"step": 570
},
{
"epoch": 0.46,
"learning_rate": 3.236569972745492e-06,
"logits/chosen": -2.6336703300476074,
"logits/rejected": -2.649231195449829,
"logps/chosen": -431.4183044433594,
"logps/rejected": -447.3372497558594,
"loss": 0.6801,
"rewards/accuracies": 0.574999988079071,
"rewards/chosen": -0.30185872316360474,
"rewards/margins": 0.040949635207653046,
"rewards/rejected": -0.342808336019516,
"step": 580
},
{
"epoch": 0.47,
"learning_rate": 3.1695173182897126e-06,
"logits/chosen": -2.5993151664733887,
"logits/rejected": -2.607579469680786,
"logps/chosen": -448.4823303222656,
"logps/rejected": -482.7305603027344,
"loss": 0.6645,
"rewards/accuracies": 0.5874999761581421,
"rewards/chosen": -0.3687121272087097,
"rewards/margins": 0.07179446518421173,
"rewards/rejected": -0.44050654768943787,
"step": 590
},
{
"epoch": 0.48,
"learning_rate": 3.10194166506673e-06,
"logits/chosen": -2.6293110847473145,
"logits/rejected": -2.649170398712158,
"logps/chosen": -456.01605224609375,
"logps/rejected": -483.74298095703125,
"loss": 0.6839,
"rewards/accuracies": 0.59375,
"rewards/chosen": -0.3835269808769226,
"rewards/margins": 0.037176936864852905,
"rewards/rejected": -0.4207039475440979,
"step": 600
},
{
"epoch": 0.48,
"eval_logits/chosen": -2.533261299133301,
"eval_logits/rejected": -2.5394067764282227,
"eval_logps/chosen": -366.2155456542969,
"eval_logps/rejected": -379.6049499511719,
"eval_loss": 0.6814756393432617,
"eval_rewards/accuracies": 0.5666666626930237,
"eval_rewards/chosen": -0.3188343644142151,
"eval_rewards/margins": 0.03678455203771591,
"eval_rewards/rejected": -0.3556188941001892,
"eval_runtime": 1615.2404,
"eval_samples_per_second": 1.854,
"eval_steps_per_second": 0.232,
"step": 600
},
{
"epoch": 0.49,
"learning_rate": 3.0338958003344115e-06,
"logits/chosen": -2.627558946609497,
"logits/rejected": -2.6516823768615723,
"logps/chosen": -437.21160888671875,
"logps/rejected": -464.7273864746094,
"loss": 0.6661,
"rewards/accuracies": 0.5687500238418579,
"rewards/chosen": -0.4052312970161438,
"rewards/margins": 0.0762629583477974,
"rewards/rejected": -0.4814942479133606,
"step": 610
},
{
"epoch": 0.5,
"learning_rate": 2.9654328786600823e-06,
"logits/chosen": -2.619433879852295,
"logits/rejected": -2.6456384658813477,
"logps/chosen": -452.12762451171875,
"logps/rejected": -496.452392578125,
"loss": 0.6522,
"rewards/accuracies": 0.59375,
"rewards/chosen": -0.47861284017562866,
"rewards/margins": 0.10839803516864777,
"rewards/rejected": -0.5870109796524048,
"step": 620
},
{
"epoch": 0.5,
"learning_rate": 2.896606380398402e-06,
"logits/chosen": -2.639660120010376,
"logits/rejected": -2.652381181716919,
"logps/chosen": -472.73089599609375,
"logps/rejected": -484.69561767578125,
"loss": 0.6922,
"rewards/accuracies": 0.5625,
"rewards/chosen": -0.5940805673599243,
"rewards/margins": 0.029440635815262794,
"rewards/rejected": -0.6235212087631226,
"step": 630
},
{
"epoch": 0.51,
"learning_rate": 2.827470069914772e-06,
"logits/chosen": -2.6815245151519775,
"logits/rejected": -2.687138080596924,
"logps/chosen": -475.3055114746094,
"logps/rejected": -506.28643798828125,
"loss": 0.6782,
"rewards/accuracies": 0.59375,
"rewards/chosen": -0.5460494160652161,
"rewards/margins": 0.060324352234601974,
"rewards/rejected": -0.6063737869262695,
"step": 640
},
{
"epoch": 0.52,
"learning_rate": 2.7580779535868675e-06,
"logits/chosen": -2.6303911209106445,
"logits/rejected": -2.646120071411133,
"logps/chosen": -471.96533203125,
"logps/rejected": -487.72271728515625,
"loss": 0.675,
"rewards/accuracies": 0.53125,
"rewards/chosen": -0.489617258310318,
"rewards/margins": 0.05899134278297424,
"rewards/rejected": -0.548608660697937,
"step": 650
},
{
"epoch": 0.53,
"learning_rate": 2.688484237617129e-06,
"logits/chosen": -2.647108316421509,
"logits/rejected": -2.6790130138397217,
"logps/chosen": -467.047607421875,
"logps/rejected": -493.06524658203125,
"loss": 0.6725,
"rewards/accuracies": 0.612500011920929,
"rewards/chosen": -0.4314625859260559,
"rewards/margins": 0.063529834151268,
"rewards/rejected": -0.4949924349784851,
"step": 660
},
{
"epoch": 0.54,
"learning_rate": 2.6187432856891585e-06,
"logits/chosen": -2.658151626586914,
"logits/rejected": -2.6678333282470703,
"logps/chosen": -471.19842529296875,
"logps/rejected": -504.4642639160156,
"loss": 0.6669,
"rewards/accuracies": 0.625,
"rewards/chosen": -0.44427603483200073,
"rewards/margins": 0.07264076918363571,
"rewards/rejected": -0.516916811466217,
"step": 670
},
{
"epoch": 0.54,
"learning_rate": 2.548909576501096e-06,
"logits/chosen": -2.7318921089172363,
"logits/rejected": -2.7216339111328125,
"logps/chosen": -474.5341796875,
"logps/rejected": -498.4226989746094,
"loss": 0.6642,
"rewards/accuracies": 0.637499988079071,
"rewards/chosen": -0.4827556014060974,
"rewards/margins": 0.08245094865560532,
"rewards/rejected": -0.5652065873146057,
"step": 680
},
{
"epoch": 0.55,
"learning_rate": 2.4790376612091503e-06,
"logits/chosen": -2.70528507232666,
"logits/rejected": -2.7278404235839844,
"logps/chosen": -492.2674255371094,
"logps/rejected": -518.07177734375,
"loss": 0.6686,
"rewards/accuracies": 0.5562499761581421,
"rewards/chosen": -0.579769492149353,
"rewards/margins": 0.0886869877576828,
"rewards/rejected": -0.6684564352035522,
"step": 690
},
{
"epoch": 0.56,
"learning_rate": 2.40918212081453e-06,
"logits/chosen": -2.643637180328369,
"logits/rejected": -2.6695291996002197,
"logps/chosen": -438.7723083496094,
"logps/rejected": -477.81451416015625,
"loss": 0.6535,
"rewards/accuracies": 0.581250011920929,
"rewards/chosen": -0.4713774621486664,
"rewards/margins": 0.10217118263244629,
"rewards/rejected": -0.5735486745834351,
"step": 700
},
{
"epoch": 0.56,
"eval_logits/chosen": -2.5424647331237793,
"eval_logits/rejected": -2.5483226776123047,
"eval_logps/chosen": -376.3719177246094,
"eval_logps/rejected": -391.4495544433594,
"eval_loss": 0.6770339012145996,
"eval_rewards/accuracies": 0.5763333439826965,
"eval_rewards/chosen": -0.42039749026298523,
"eval_rewards/margins": 0.05366775020956993,
"eval_rewards/rejected": -0.47406527400016785,
"eval_runtime": 1614.5505,
"eval_samples_per_second": 1.854,
"eval_steps_per_second": 0.232,
"step": 700
},
{
"epoch": 0.57,
"learning_rate": 2.3393975235270654e-06,
"logits/chosen": -2.6486973762512207,
"logits/rejected": -2.6384849548339844,
"logps/chosen": -470.92498779296875,
"logps/rejected": -504.40740966796875,
"loss": 0.6546,
"rewards/accuracies": 0.574999988079071,
"rewards/chosen": -0.49029403924942017,
"rewards/margins": 0.10647070407867432,
"rewards/rejected": -0.5967647433280945,
"step": 710
},
{
"epoch": 0.58,
"learning_rate": 2.2697383821388153e-06,
"logits/chosen": -2.6679904460906982,
"logits/rejected": -2.6391725540161133,
"logps/chosen": -463.16632080078125,
"logps/rejected": -473.4232482910156,
"loss": 0.6867,
"rewards/accuracies": 0.574999988079071,
"rewards/chosen": -0.4399467408657074,
"rewards/margins": 0.04117538779973984,
"rewards/rejected": -0.48112210631370544,
"step": 720
},
{
"epoch": 0.58,
"learning_rate": 2.2002591114409657e-06,
"logits/chosen": -2.677816867828369,
"logits/rejected": -2.6684961318969727,
"logps/chosen": -457.8728942871094,
"logps/rejected": -479.80609130859375,
"loss": 0.6756,
"rewards/accuracies": 0.543749988079071,
"rewards/chosen": -0.3563249111175537,
"rewards/margins": 0.05807635933160782,
"rewards/rejected": -0.41440123319625854,
"step": 730
},
{
"epoch": 0.59,
"learning_rate": 2.131013985717285e-06,
"logits/chosen": -2.678981065750122,
"logits/rejected": -2.669673204421997,
"logps/chosen": -464.6796875,
"logps/rejected": -506.2353515625,
"loss": 0.6495,
"rewards/accuracies": 0.625,
"rewards/chosen": -0.320025771856308,
"rewards/margins": 0.10963205248117447,
"rewards/rejected": -0.42965784668922424,
"step": 740
},
{
"epoch": 0.6,
"learning_rate": 2.062057096347338e-06,
"logits/chosen": -2.6860058307647705,
"logits/rejected": -2.694854736328125,
"logps/chosen": -461.962890625,
"logps/rejected": -472.4624938964844,
"loss": 0.6667,
"rewards/accuracies": 0.574999988079071,
"rewards/chosen": -0.38755279779434204,
"rewards/margins": 0.08047433197498322,
"rewards/rejected": -0.46802711486816406,
"step": 750
},
{
"epoch": 0.61,
"learning_rate": 1.9934423095525733e-06,
"logits/chosen": -2.6688058376312256,
"logits/rejected": -2.6453909873962402,
"logps/chosen": -467.1205139160156,
"logps/rejected": -475.4339904785156,
"loss": 0.6708,
"rewards/accuracies": 0.643750011920929,
"rewards/chosen": -0.385872483253479,
"rewards/margins": 0.07536690682172775,
"rewards/rejected": -0.46123939752578735,
"step": 760
},
{
"epoch": 0.62,
"learning_rate": 1.9252232243182986e-06,
"logits/chosen": -2.656658172607422,
"logits/rejected": -2.6702752113342285,
"logps/chosen": -434.9510192871094,
"logps/rejected": -466.2872009277344,
"loss": 0.6669,
"rewards/accuracies": 0.59375,
"rewards/chosen": -0.4157305657863617,
"rewards/margins": 0.0795067697763443,
"rewards/rejected": -0.4952372908592224,
"step": 770
},
{
"epoch": 0.62,
"learning_rate": 1.8574531305244043e-06,
"logits/chosen": -2.683166980743408,
"logits/rejected": -2.705665111541748,
"logps/chosen": -473.3404846191406,
"logps/rejected": -507.530029296875,
"loss": 0.6445,
"rewards/accuracies": 0.5874999761581421,
"rewards/chosen": -0.39788818359375,
"rewards/margins": 0.13318848609924316,
"rewards/rejected": -0.5310766696929932,
"step": 780
},
{
"epoch": 0.63,
"learning_rate": 1.7901849673175559e-06,
"logits/chosen": -2.7003445625305176,
"logits/rejected": -2.7315449714660645,
"logps/chosen": -467.36383056640625,
"logps/rejected": -495.95709228515625,
"loss": 0.6692,
"rewards/accuracies": 0.6312500238418579,
"rewards/chosen": -0.3450160622596741,
"rewards/margins": 0.07192648947238922,
"rewards/rejected": -0.4169425368309021,
"step": 790
},
{
"epoch": 0.64,
"learning_rate": 1.7234712817573555e-06,
"logits/chosen": -2.680144786834717,
"logits/rejected": -2.6287026405334473,
"logps/chosen": -486.04278564453125,
"logps/rejected": -492.35162353515625,
"loss": 0.6764,
"rewards/accuracies": 0.612500011920929,
"rewards/chosen": -0.37955719232559204,
"rewards/margins": 0.06455205380916595,
"rewards/rejected": -0.4441092014312744,
"step": 800
},
{
"epoch": 0.64,
"eval_logits/chosen": -2.5651443004608154,
"eval_logits/rejected": -2.571413516998291,
"eval_logps/chosen": -359.1413269042969,
"eval_logps/rejected": -374.912841796875,
"eval_loss": 0.672364354133606,
"eval_rewards/accuracies": 0.5989999771118164,
"eval_rewards/chosen": -0.24809196591377258,
"eval_rewards/margins": 0.06060539186000824,
"eval_rewards/rejected": -0.308697372674942,
"eval_runtime": 1620.9312,
"eval_samples_per_second": 1.847,
"eval_steps_per_second": 0.231,
"step": 800
},
{
"epoch": 0.65,
"learning_rate": 1.6573641877687936e-06,
"logits/chosen": -2.696709394454956,
"logits/rejected": -2.713754653930664,
"logps/chosen": -439.42864990234375,
"logps/rejected": -479.58612060546875,
"loss": 0.6545,
"rewards/accuracies": 0.625,
"rewards/chosen": -0.28763246536254883,
"rewards/margins": 0.10148018598556519,
"rewards/rejected": -0.389112651348114,
"step": 810
},
{
"epoch": 0.66,
"learning_rate": 1.591915325433034e-06,
"logits/chosen": -2.7134814262390137,
"logits/rejected": -2.6989309787750244,
"logps/chosen": -442.7323303222656,
"logps/rejected": -455.2479553222656,
"loss": 0.6775,
"rewards/accuracies": 0.581250011920929,
"rewards/chosen": -0.31017300486564636,
"rewards/margins": 0.056103698909282684,
"rewards/rejected": -0.36627671122550964,
"step": 820
},
{
"epoch": 0.66,
"learning_rate": 1.5271758206483664e-06,
"logits/chosen": -2.7078943252563477,
"logits/rejected": -2.699531078338623,
"logps/chosen": -458.97772216796875,
"logps/rejected": -478.4125061035156,
"loss": 0.6669,
"rewards/accuracies": 0.6499999761581421,
"rewards/chosen": -0.31177014112472534,
"rewards/margins": 0.0876721516251564,
"rewards/rejected": -0.39944231510162354,
"step": 830
},
{
"epoch": 0.67,
"learning_rate": 1.4631962451927966e-06,
"logits/chosen": -2.689883232116699,
"logits/rejected": -2.7185115814208984,
"logps/chosen": -453.9341735839844,
"logps/rejected": -486.0030822753906,
"loss": 0.6594,
"rewards/accuracies": 0.6312500238418579,
"rewards/chosen": -0.2792655825614929,
"rewards/margins": 0.0935577005147934,
"rewards/rejected": -0.3728232979774475,
"step": 840
},
{
"epoch": 0.68,
"learning_rate": 1.4000265772195032e-06,
"logits/chosen": -2.7169957160949707,
"logits/rejected": -2.7198026180267334,
"logps/chosen": -462.4480895996094,
"logps/rejected": -495.20330810546875,
"loss": 0.6628,
"rewards/accuracies": 0.6312500238418579,
"rewards/chosen": -0.3546587824821472,
"rewards/margins": 0.09027893841266632,
"rewards/rejected": -0.44493770599365234,
"step": 850
},
{
"epoch": 0.69,
"learning_rate": 1.3377161622160077e-06,
"logits/chosen": -2.7009379863739014,
"logits/rejected": -2.696585178375244,
"logps/chosen": -458.6473083496094,
"logps/rejected": -485.0048828125,
"loss": 0.6556,
"rewards/accuracies": 0.6312500238418579,
"rewards/chosen": -0.3736417591571808,
"rewards/margins": 0.10007522255182266,
"rewards/rejected": -0.4737170338630676,
"step": 860
},
{
"epoch": 0.7,
"learning_rate": 1.276313674457553e-06,
"logits/chosen": -2.700366497039795,
"logits/rejected": -2.7102417945861816,
"logps/chosen": -439.9691467285156,
"logps/rejected": -479.43231201171875,
"loss": 0.6373,
"rewards/accuracies": 0.643750011920929,
"rewards/chosen": -0.3865216374397278,
"rewards/margins": 0.15378279983997345,
"rewards/rejected": -0.5403044819831848,
"step": 870
},
{
"epoch": 0.7,
"learning_rate": 1.2158670789848095e-06,
"logits/chosen": -2.675424098968506,
"logits/rejected": -2.6478264331817627,
"logps/chosen": -472.1514587402344,
"logps/rejected": -488.82000732421875,
"loss": 0.6835,
"rewards/accuracies": 0.637499988079071,
"rewards/chosen": -0.5139307379722595,
"rewards/margins": 0.05513748526573181,
"rewards/rejected": -0.5690682530403137,
"step": 880
},
{
"epoch": 0.71,
"learning_rate": 1.1564235941356016e-06,
"logits/chosen": -2.647432804107666,
"logits/rejected": -2.6728672981262207,
"logps/chosen": -459.1905212402344,
"logps/rejected": -504.0580139160156,
"loss": 0.6369,
"rewards/accuracies": 0.6937500238418579,
"rewards/chosen": -0.5291575789451599,
"rewards/margins": 0.15966546535491943,
"rewards/rejected": -0.6888231039047241,
"step": 890
},
{
"epoch": 0.72,
"learning_rate": 1.0980296546599254e-06,
"logits/chosen": -2.6834964752197266,
"logits/rejected": -2.671112537384033,
"logps/chosen": -498.71478271484375,
"logps/rejected": -504.4794006347656,
"loss": 0.6753,
"rewards/accuracies": 0.550000011920929,
"rewards/chosen": -0.6096609830856323,
"rewards/margins": 0.08308340609073639,
"rewards/rejected": -0.6927444934844971,
"step": 900
},
{
"epoch": 0.72,
"eval_logits/chosen": -2.5749759674072266,
"eval_logits/rejected": -2.5806519985198975,
"eval_logps/chosen": -377.15924072265625,
"eval_logps/rejected": -394.6670837402344,
"eval_loss": 0.6703553795814514,
"eval_rewards/accuracies": 0.5983333587646484,
"eval_rewards/chosen": -0.4282708764076233,
"eval_rewards/margins": 0.0779690071940422,
"eval_rewards/rejected": -0.5062398910522461,
"eval_runtime": 1623.0619,
"eval_samples_per_second": 1.845,
"eval_steps_per_second": 0.231,
"step": 900
},
{
"epoch": 0.73,
"learning_rate": 1.040730875447083e-06,
"logits/chosen": -2.694941997528076,
"logits/rejected": -2.698484182357788,
"logps/chosen": -475.06201171875,
"logps/rejected": -494.41033935546875,
"loss": 0.656,
"rewards/accuracies": 0.6499999761581421,
"rewards/chosen": -0.5018836259841919,
"rewards/margins": 0.11171841621398926,
"rewards/rejected": -0.6136020421981812,
"step": 910
},
{
"epoch": 0.74,
"learning_rate": 9.845720158932414e-07,
"logits/chosen": -2.6612589359283447,
"logits/rejected": -2.6566224098205566,
"logps/chosen": -442.80841064453125,
"logps/rejected": -453.392578125,
"loss": 0.6669,
"rewards/accuracies": 0.5625,
"rewards/chosen": -0.4533756375312805,
"rewards/margins": 0.07945097982883453,
"rewards/rejected": -0.5328266024589539,
"step": 920
},
{
"epoch": 0.74,
"learning_rate": 9.295969449372796e-07,
"logits/chosen": -2.6861672401428223,
"logits/rejected": -2.712158441543579,
"logps/chosen": -449.2447204589844,
"logps/rejected": -474.03314208984375,
"loss": 0.6672,
"rewards/accuracies": 0.6000000238418579,
"rewards/chosen": -0.39123302698135376,
"rewards/margins": 0.08886702358722687,
"rewards/rejected": -0.4801000654697418,
"step": 930
},
{
"epoch": 0.75,
"learning_rate": 8.758486067922176e-07,
"logits/chosen": -2.6912190914154053,
"logits/rejected": -2.708655834197998,
"logps/chosen": -439.08233642578125,
"logps/rejected": -478.17950439453125,
"loss": 0.6498,
"rewards/accuracies": 0.6187499761581421,
"rewards/chosen": -0.35874584317207336,
"rewards/margins": 0.1233411431312561,
"rewards/rejected": -0.48208698630332947,
"step": 940
},
{
"epoch": 0.76,
"learning_rate": 8.233689873990006e-07,
"logits/chosen": -2.672585964202881,
"logits/rejected": -2.679046392440796,
"logps/chosen": -440.7559509277344,
"logps/rejected": -479.00164794921875,
"loss": 0.6422,
"rewards/accuracies": 0.637499988079071,
"rewards/chosen": -0.32873159646987915,
"rewards/margins": 0.13754822313785553,
"rewards/rejected": -0.4662798345088959,
"step": 950
},
{
"epoch": 0.77,
"learning_rate": 7.721990816288555e-07,
"logits/chosen": -2.7491555213928223,
"logits/rejected": -2.769662380218506,
"logps/chosen": -422.5555114746094,
"logps/rejected": -451.8642578125,
"loss": 0.6705,
"rewards/accuracies": 0.59375,
"rewards/chosen": -0.32741206884384155,
"rewards/margins": 0.07545635104179382,
"rewards/rejected": -0.4028684198856354,
"step": 960
},
{
"epoch": 0.78,
"learning_rate": 7.223788612598148e-07,
"logits/chosen": -2.7415757179260254,
"logits/rejected": -2.750936269760132,
"logps/chosen": -452.7373962402344,
"logps/rejected": -484.138671875,
"loss": 0.6439,
"rewards/accuracies": 0.6499999761581421,
"rewards/chosen": -0.27024340629577637,
"rewards/margins": 0.12849459052085876,
"rewards/rejected": -0.39873799681663513,
"step": 970
},
{
"epoch": 0.78,
"learning_rate": 6.73947243752448e-07,
"logits/chosen": -2.718167304992676,
"logits/rejected": -2.7125675678253174,
"logps/chosen": -470.5419921875,
"logps/rejected": -465.5101013183594,
"loss": 0.6762,
"rewards/accuracies": 0.5375000238418579,
"rewards/chosen": -0.36285415291786194,
"rewards/margins": 0.06569617241621017,
"rewards/rejected": -0.4285503029823303,
"step": 980
},
{
"epoch": 0.79,
"learning_rate": 6.269420618491759e-07,
"logits/chosen": -2.75072979927063,
"logits/rejected": -2.751321315765381,
"logps/chosen": -432.2750549316406,
"logps/rejected": -462.3431701660156,
"loss": 0.6574,
"rewards/accuracies": 0.6499999761581421,
"rewards/chosen": -0.2954108715057373,
"rewards/margins": 0.09935091435909271,
"rewards/rejected": -0.3947618007659912,
"step": 990
},
{
"epoch": 0.8,
"learning_rate": 5.814000340209267e-07,
"logits/chosen": -2.7202508449554443,
"logits/rejected": -2.721200466156006,
"logps/chosen": -453.22100830078125,
"logps/rejected": -491.32080078125,
"loss": 0.6459,
"rewards/accuracies": 0.6625000238418579,
"rewards/chosen": -0.30317172408103943,
"rewards/margins": 0.12803982198238373,
"rewards/rejected": -0.43121156096458435,
"step": 1000
},
{
"epoch": 0.8,
"eval_logits/chosen": -2.5858356952667236,
"eval_logits/rejected": -2.59238338470459,
"eval_logps/chosen": -358.3893737792969,
"eval_logps/rejected": -375.67333984375,
"eval_loss": 0.6679545640945435,
"eval_rewards/accuracies": 0.612666666507721,
"eval_rewards/chosen": -0.24057278037071228,
"eval_rewards/margins": 0.07572992891073227,
"eval_rewards/rejected": -0.31630268692970276,
"eval_runtime": 1618.0765,
"eval_samples_per_second": 1.85,
"eval_steps_per_second": 0.232,
"step": 1000
},
{
"epoch": 0.81,
"learning_rate": 5.373567357842111e-07,
"logits/chosen": -2.6775119304656982,
"logits/rejected": -2.707977771759033,
"logps/chosen": -443.8631896972656,
"logps/rejected": -472.0171813964844,
"loss": 0.6591,
"rewards/accuracies": 0.6312500238418579,
"rewards/chosen": -0.25422340631484985,
"rewards/margins": 0.09956096857786179,
"rewards/rejected": -0.35378438234329224,
"step": 1010
},
{
"epoch": 0.82,
"learning_rate": 4.948465719110226e-07,
"logits/chosen": -2.69923996925354,
"logits/rejected": -2.714949607849121,
"logps/chosen": -425.83245849609375,
"logps/rejected": -450.9254455566406,
"loss": 0.6449,
"rewards/accuracies": 0.6499999761581421,
"rewards/chosen": -0.2896645665168762,
"rewards/margins": 0.12861916422843933,
"rewards/rejected": -0.41828376054763794,
"step": 1020
},
{
"epoch": 0.82,
"learning_rate": 4.539027495532766e-07,
"logits/chosen": -2.6870169639587402,
"logits/rejected": -2.6863129138946533,
"logps/chosen": -437.5118713378906,
"logps/rejected": -460.01373291015625,
"loss": 0.6558,
"rewards/accuracies": 0.65625,
"rewards/chosen": -0.3346652388572693,
"rewards/margins": 0.11685864627361298,
"rewards/rejected": -0.45152387022972107,
"step": 1030
},
{
"epoch": 0.83,
"learning_rate": 4.14557252302783e-07,
"logits/chosen": -2.691601276397705,
"logits/rejected": -2.7064461708068848,
"logps/chosen": -443.50445556640625,
"logps/rejected": -478.5107421875,
"loss": 0.6522,
"rewards/accuracies": 0.6499999761581421,
"rewards/chosen": -0.3916120231151581,
"rewards/margins": 0.11733059585094452,
"rewards/rejected": -0.5089426636695862,
"step": 1040
},
{
"epoch": 0.84,
"learning_rate": 3.7684081520700884e-07,
"logits/chosen": -2.716027021408081,
"logits/rejected": -2.721525192260742,
"logps/chosen": -477.7105407714844,
"logps/rejected": -481.0179748535156,
"loss": 0.6819,
"rewards/accuracies": 0.625,
"rewards/chosen": -0.4463655352592468,
"rewards/margins": 0.05458784103393555,
"rewards/rejected": -0.5009533166885376,
"step": 1050
},
{
"epoch": 0.85,
"learning_rate": 3.407829007601507e-07,
"logits/chosen": -2.6834683418273926,
"logits/rejected": -2.6970880031585693,
"logps/chosen": -457.0318298339844,
"logps/rejected": -498.78485107421875,
"loss": 0.6357,
"rewards/accuracies": 0.612500011920929,
"rewards/chosen": -0.36882373690605164,
"rewards/margins": 0.1661624312400818,
"rewards/rejected": -0.5349861979484558,
"step": 1060
},
{
"epoch": 0.86,
"learning_rate": 3.064116758882724e-07,
"logits/chosen": -2.7176353931427,
"logits/rejected": -2.7239794731140137,
"logps/chosen": -464.91534423828125,
"logps/rejected": -508.00927734375,
"loss": 0.6459,
"rewards/accuracies": 0.6812499761581421,
"rewards/chosen": -0.38777628540992737,
"rewards/margins": 0.14257176220417023,
"rewards/rejected": -0.5303480625152588,
"step": 1070
},
{
"epoch": 0.86,
"learning_rate": 2.737539899464908e-07,
"logits/chosen": -2.6851329803466797,
"logits/rejected": -2.698014736175537,
"logps/chosen": -423.3038024902344,
"logps/rejected": -450.4856872558594,
"loss": 0.6599,
"rewards/accuracies": 0.6187499761581421,
"rewards/chosen": -0.3559909462928772,
"rewards/margins": 0.09332160651683807,
"rewards/rejected": -0.44931259751319885,
"step": 1080
},
{
"epoch": 0.87,
"learning_rate": 2.4283535374538645e-07,
"logits/chosen": -2.744837999343872,
"logits/rejected": -2.747127056121826,
"logps/chosen": -465.623046875,
"logps/rejected": -489.3487243652344,
"loss": 0.6522,
"rewards/accuracies": 0.65625,
"rewards/chosen": -0.3236229419708252,
"rewards/margins": 0.12912145256996155,
"rewards/rejected": -0.45274439454078674,
"step": 1090
},
{
"epoch": 0.88,
"learning_rate": 2.1367991962303298e-07,
"logits/chosen": -2.7007431983947754,
"logits/rejected": -2.71804141998291,
"logps/chosen": -428.20147705078125,
"logps/rejected": -458.3008728027344,
"loss": 0.6541,
"rewards/accuracies": 0.59375,
"rewards/chosen": -0.3504807949066162,
"rewards/margins": 0.11653472483158112,
"rewards/rejected": -0.46701550483703613,
"step": 1100
},
{
"epoch": 0.88,
"eval_logits/chosen": -2.587815046310425,
"eval_logits/rejected": -2.594228744506836,
"eval_logps/chosen": -362.3882141113281,
"eval_logps/rejected": -380.2968444824219,
"eval_loss": 0.6669934988021851,
"eval_rewards/accuracies": 0.6156666874885559,
"eval_rewards/chosen": -0.28056082129478455,
"eval_rewards/margins": 0.08197694271802902,
"eval_rewards/rejected": -0.36253777146339417,
"eval_runtime": 1611.4517,
"eval_samples_per_second": 1.858,
"eval_steps_per_second": 0.233,
"step": 1100
},
{
"epoch": 0.89,
"learning_rate": 1.8631046257820278e-07,
"logits/chosen": -2.728276014328003,
"logits/rejected": -2.7310478687286377,
"logps/chosen": -461.82879638671875,
"logps/rejected": -471.50775146484375,
"loss": 0.6887,
"rewards/accuracies": 0.6312500238418579,
"rewards/chosen": -0.3837016522884369,
"rewards/margins": 0.04261630028486252,
"rewards/rejected": -0.4263179898262024,
"step": 1110
},
{
"epoch": 0.9,
"learning_rate": 1.6074836247950143e-07,
"logits/chosen": -2.7541463375091553,
"logits/rejected": -2.7452800273895264,
"logps/chosen": -471.38446044921875,
"logps/rejected": -493.8583068847656,
"loss": 0.656,
"rewards/accuracies": 0.6187499761581421,
"rewards/chosen": -0.32993000745773315,
"rewards/margins": 0.11021927744150162,
"rewards/rejected": -0.44014930725097656,
"step": 1120
},
{
"epoch": 0.9,
"learning_rate": 1.370135873643097e-07,
"logits/chosen": -2.711472749710083,
"logits/rejected": -2.6935694217681885,
"logps/chosen": -458.6504821777344,
"logps/rejected": -458.78839111328125,
"loss": 0.6649,
"rewards/accuracies": 0.625,
"rewards/chosen": -0.33027976751327515,
"rewards/margins": 0.08629810065031052,
"rewards/rejected": -0.41657787561416626,
"step": 1130
},
{
"epoch": 0.91,
"learning_rate": 1.1512467784059372e-07,
"logits/chosen": -2.7174072265625,
"logits/rejected": -2.726274013519287,
"logps/chosen": -421.85833740234375,
"logps/rejected": -443.1368713378906,
"loss": 0.6782,
"rewards/accuracies": 0.5687500238418579,
"rewards/chosen": -0.3433849811553955,
"rewards/margins": 0.07356986403465271,
"rewards/rejected": -0.4169548451900482,
"step": 1140
},
{
"epoch": 0.92,
"learning_rate": 9.509873260376251e-08,
"logits/chosen": -2.6908252239227295,
"logits/rejected": -2.697906017303467,
"logps/chosen": -438.10089111328125,
"logps/rejected": -496.07269287109375,
"loss": 0.6257,
"rewards/accuracies": 0.6499999761581421,
"rewards/chosen": -0.2791033387184143,
"rewards/margins": 0.17611822485923767,
"rewards/rejected": -0.455221563577652,
"step": 1150
},
{
"epoch": 0.93,
"learning_rate": 7.695139507988559e-08,
"logits/chosen": -2.7109904289245605,
"logits/rejected": -2.704934597015381,
"logps/chosen": -467.7078552246094,
"logps/rejected": -485.38580322265625,
"loss": 0.6916,
"rewards/accuracies": 0.5625,
"rewards/chosen": -0.37871813774108887,
"rewards/margins": 0.04057822376489639,
"rewards/rejected": -0.41929635405540466,
"step": 1160
},
{
"epoch": 0.94,
"learning_rate": 6.069684120570684e-08,
"logits/chosen": -2.6606979370117188,
"logits/rejected": -2.6916613578796387,
"logps/chosen": -442.80462646484375,
"logps/rejected": -487.12701416015625,
"loss": 0.6493,
"rewards/accuracies": 0.6875,
"rewards/chosen": -0.3210916817188263,
"rewards/margins": 0.13079658150672913,
"rewards/rejected": -0.45188823342323303,
"step": 1170
},
{
"epoch": 0.94,
"learning_rate": 4.634776835499871e-08,
"logits/chosen": -2.669241428375244,
"logits/rejected": -2.6664905548095703,
"logps/chosen": -426.0979919433594,
"logps/rejected": -453.2581481933594,
"loss": 0.6681,
"rewards/accuracies": 0.5874999761581421,
"rewards/chosen": -0.31668686866760254,
"rewards/margins": 0.09003134816884995,
"rewards/rejected": -0.4067181944847107,
"step": 1180
},
{
"epoch": 0.95,
"learning_rate": 3.3915385419908964e-08,
"logits/chosen": -2.7456703186035156,
"logits/rejected": -2.72196626663208,
"logps/chosen": -457.28515625,
"logps/rejected": -476.6426696777344,
"loss": 0.6645,
"rewards/accuracies": 0.612500011920929,
"rewards/chosen": -0.33392366766929626,
"rewards/margins": 0.10097813606262207,
"rewards/rejected": -0.43490180373191833,
"step": 1190
},
{
"epoch": 0.96,
"learning_rate": 2.3409404055043938e-08,
"logits/chosen": -2.6708192825317383,
"logits/rejected": -2.6985325813293457,
"logps/chosen": -443.9369201660156,
"logps/rejected": -475.53955078125,
"loss": 0.6422,
"rewards/accuracies": 0.6625000238418579,
"rewards/chosen": -0.3015444874763489,
"rewards/margins": 0.1372503787279129,
"rewards/rejected": -0.43879491090774536,
"step": 1200
},
{
"epoch": 0.96,
"eval_logits/chosen": -2.5898184776306152,
"eval_logits/rejected": -2.596299409866333,
"eval_logps/chosen": -360.897216796875,
"eval_logps/rejected": -378.7738342285156,
"eval_loss": 0.6668795943260193,
"eval_rewards/accuracies": 0.6156666874885559,
"eval_rewards/chosen": -0.26565054059028625,
"eval_rewards/margins": 0.08165695518255234,
"eval_rewards/rejected": -0.3473075032234192,
"eval_runtime": 1611.7613,
"eval_samples_per_second": 1.858,
"eval_steps_per_second": 0.233,
"step": 1200
},
{
"epoch": 0.97,
"learning_rate": 1.4838031091134186e-08,
"logits/chosen": -2.6734347343444824,
"logits/rejected": -2.7052102088928223,
"logps/chosen": -414.99566650390625,
"logps/rejected": -469.2049255371094,
"loss": 0.6318,
"rewards/accuracies": 0.65625,
"rewards/chosen": -0.2892751097679138,
"rewards/margins": 0.15364858508110046,
"rewards/rejected": -0.4429236948490143,
"step": 1210
},
{
"epoch": 0.98,
"learning_rate": 8.207962124201774e-09,
"logits/chosen": -2.7094740867614746,
"logits/rejected": -2.7131965160369873,
"logps/chosen": -459.8565979003906,
"logps/rejected": -477.9366149902344,
"loss": 0.6578,
"rewards/accuracies": 0.59375,
"rewards/chosen": -0.3082069456577301,
"rewards/margins": 0.10618587583303452,
"rewards/rejected": -0.4143927991390228,
"step": 1220
},
{
"epoch": 0.98,
"learning_rate": 3.5243762852441023e-09,
"logits/chosen": -2.7074780464172363,
"logits/rejected": -2.7236151695251465,
"logps/chosen": -444.84161376953125,
"logps/rejected": -480.5126037597656,
"loss": 0.6562,
"rewards/accuracies": 0.574999988079071,
"rewards/chosen": -0.31734251976013184,
"rewards/margins": 0.11356744915246964,
"rewards/rejected": -0.4309099614620209,
"step": 1230
},
{
"epoch": 0.99,
"learning_rate": 7.909321945129278e-10,
"logits/chosen": -2.7246460914611816,
"logits/rejected": -2.733558177947998,
"logps/chosen": -476.4629821777344,
"logps/rejected": -513.1619873046875,
"loss": 0.6426,
"rewards/accuracies": 0.637499988079071,
"rewards/chosen": -0.3152047097682953,
"rewards/margins": 0.15284833312034607,
"rewards/rejected": -0.46805301308631897,
"step": 1240
},
{
"epoch": 1.0,
"step": 1249,
"total_flos": 0.0,
"train_loss": 0.6736802890646257,
"train_runtime": 42164.1735,
"train_samples_per_second": 0.474,
"train_steps_per_second": 0.03
}
],
"logging_steps": 10,
"max_steps": 1249,
"num_input_tokens_seen": 0,
"num_train_epochs": 1,
"save_steps": 100,
"total_flos": 0.0,
"train_batch_size": 4,
"trial_name": null,
"trial_params": null
}