|
{ |
|
"best_metric": null, |
|
"best_model_checkpoint": null, |
|
"epoch": 0.9981298423724285, |
|
"eval_steps": 400, |
|
"global_step": 467, |
|
"is_hyper_param_search": false, |
|
"is_local_process_zero": true, |
|
"is_world_process_zero": true, |
|
"log_history": [ |
|
{ |
|
"epoch": 0.0021373230029388193, |
|
"grad_norm": 42.099143078937416, |
|
"learning_rate": 1.0638297872340425e-08, |
|
"logits/chosen": 2.2472424507141113, |
|
"logits/rejected": 2.264739513397217, |
|
"logps/chosen": -0.8019136786460876, |
|
"logps/rejected": -0.7812027335166931, |
|
"loss": 1.7463, |
|
"rewards/accuracies": 0.4375, |
|
"rewards/chosen": -2.004784345626831, |
|
"rewards/margins": -0.05177761986851692, |
|
"rewards/rejected": -1.953006625175476, |
|
"step": 1 |
|
}, |
|
{ |
|
"epoch": 0.010686615014694095, |
|
"grad_norm": 43.49437439863032, |
|
"learning_rate": 5.3191489361702123e-08, |
|
"logits/chosen": 2.1162009239196777, |
|
"logits/rejected": 2.0480892658233643, |
|
"logps/chosen": -1.0361735820770264, |
|
"logps/rejected": -1.1913045644760132, |
|
"loss": 1.9184, |
|
"rewards/accuracies": 0.453125, |
|
"rewards/chosen": -2.5904340744018555, |
|
"rewards/margins": 0.38782745599746704, |
|
"rewards/rejected": -2.9782614707946777, |
|
"step": 5 |
|
}, |
|
{ |
|
"epoch": 0.02137323002938819, |
|
"grad_norm": 73.58193785863429, |
|
"learning_rate": 1.0638297872340425e-07, |
|
"logits/chosen": 2.0012879371643066, |
|
"logits/rejected": 1.9172722101211548, |
|
"logps/chosen": -1.5639755725860596, |
|
"logps/rejected": -1.5803353786468506, |
|
"loss": 1.9077, |
|
"rewards/accuracies": 0.5249999761581421, |
|
"rewards/chosen": -3.9099388122558594, |
|
"rewards/margins": 0.04089919477701187, |
|
"rewards/rejected": -3.950838088989258, |
|
"step": 10 |
|
}, |
|
{ |
|
"epoch": 0.03205984504408229, |
|
"grad_norm": 18.887588182169267, |
|
"learning_rate": 1.5957446808510638e-07, |
|
"logits/chosen": 1.9121309518814087, |
|
"logits/rejected": 1.8687114715576172, |
|
"logps/chosen": -1.1899975538253784, |
|
"logps/rejected": -1.2847260236740112, |
|
"loss": 1.832, |
|
"rewards/accuracies": 0.550000011920929, |
|
"rewards/chosen": -2.974993944168091, |
|
"rewards/margins": 0.2368215024471283, |
|
"rewards/rejected": -3.211815357208252, |
|
"step": 15 |
|
}, |
|
{ |
|
"epoch": 0.04274646005877638, |
|
"grad_norm": 8.819209234657059, |
|
"learning_rate": 2.127659574468085e-07, |
|
"logits/chosen": 1.8210818767547607, |
|
"logits/rejected": 1.8469772338867188, |
|
"logps/chosen": -1.21903657913208, |
|
"logps/rejected": -1.7520463466644287, |
|
"loss": 1.7601, |
|
"rewards/accuracies": 0.7124999761581421, |
|
"rewards/chosen": -3.0475919246673584, |
|
"rewards/margins": 1.3325239419937134, |
|
"rewards/rejected": -4.380115509033203, |
|
"step": 20 |
|
}, |
|
{ |
|
"epoch": 0.053433075073470476, |
|
"grad_norm": 33.18790384628351, |
|
"learning_rate": 2.659574468085106e-07, |
|
"logits/chosen": 2.0562968254089355, |
|
"logits/rejected": 2.0691914558410645, |
|
"logps/chosen": -1.0988093614578247, |
|
"logps/rejected": -1.1313331127166748, |
|
"loss": 1.815, |
|
"rewards/accuracies": 0.5249999761581421, |
|
"rewards/chosen": -2.747023344039917, |
|
"rewards/margins": 0.08130944520235062, |
|
"rewards/rejected": -2.8283333778381348, |
|
"step": 25 |
|
}, |
|
{ |
|
"epoch": 0.06411969008816458, |
|
"grad_norm": 35.16962019463958, |
|
"learning_rate": 3.1914893617021275e-07, |
|
"logits/chosen": 1.9061098098754883, |
|
"logits/rejected": 1.9218835830688477, |
|
"logps/chosen": -1.2370525598526, |
|
"logps/rejected": -1.4245789051055908, |
|
"loss": 1.6802, |
|
"rewards/accuracies": 0.4749999940395355, |
|
"rewards/chosen": -3.0926311016082764, |
|
"rewards/margins": 0.46881628036499023, |
|
"rewards/rejected": -3.5614471435546875, |
|
"step": 30 |
|
}, |
|
{ |
|
"epoch": 0.07480630510285867, |
|
"grad_norm": 44.11653715682163, |
|
"learning_rate": 3.7234042553191484e-07, |
|
"logits/chosen": 1.9321657419204712, |
|
"logits/rejected": 1.955640435218811, |
|
"logps/chosen": -1.344245433807373, |
|
"logps/rejected": -1.36150062084198, |
|
"loss": 1.8113, |
|
"rewards/accuracies": 0.42500001192092896, |
|
"rewards/chosen": -3.3606135845184326, |
|
"rewards/margins": 0.0431380569934845, |
|
"rewards/rejected": -3.4037513732910156, |
|
"step": 35 |
|
}, |
|
{ |
|
"epoch": 0.08549292011755276, |
|
"grad_norm": 36.76549905569564, |
|
"learning_rate": 4.25531914893617e-07, |
|
"logits/chosen": 1.8915932178497314, |
|
"logits/rejected": 1.9206931591033936, |
|
"logps/chosen": -1.2968499660491943, |
|
"logps/rejected": -1.3007665872573853, |
|
"loss": 1.8425, |
|
"rewards/accuracies": 0.5874999761581421, |
|
"rewards/chosen": -3.2421250343322754, |
|
"rewards/margins": 0.009791326709091663, |
|
"rewards/rejected": -3.2519161701202393, |
|
"step": 40 |
|
}, |
|
{ |
|
"epoch": 0.09617953513224686, |
|
"grad_norm": 17.026888436798433, |
|
"learning_rate": 4.787234042553192e-07, |
|
"logits/chosen": 1.8335082530975342, |
|
"logits/rejected": 1.803972601890564, |
|
"logps/chosen": -1.1560873985290527, |
|
"logps/rejected": -1.391876459121704, |
|
"loss": 1.748, |
|
"rewards/accuracies": 0.550000011920929, |
|
"rewards/chosen": -2.8902182579040527, |
|
"rewards/margins": 0.5894726514816284, |
|
"rewards/rejected": -3.4796910285949707, |
|
"step": 45 |
|
}, |
|
{ |
|
"epoch": 0.10686615014694095, |
|
"grad_norm": 12.410804374401861, |
|
"learning_rate": 4.999370587356267e-07, |
|
"logits/chosen": 1.864829659461975, |
|
"logits/rejected": 1.8258002996444702, |
|
"logps/chosen": -1.0499851703643799, |
|
"logps/rejected": -1.454998254776001, |
|
"loss": 1.5838, |
|
"rewards/accuracies": 0.5249999761581421, |
|
"rewards/chosen": -2.6249632835388184, |
|
"rewards/margins": 1.0125324726104736, |
|
"rewards/rejected": -3.637495517730713, |
|
"step": 50 |
|
}, |
|
{ |
|
"epoch": 0.11755276516163506, |
|
"grad_norm": 16.97890172462393, |
|
"learning_rate": 4.995525324419337e-07, |
|
"logits/chosen": 1.8395353555679321, |
|
"logits/rejected": 1.8361022472381592, |
|
"logps/chosen": -0.9045565724372864, |
|
"logps/rejected": -1.1536071300506592, |
|
"loss": 1.6796, |
|
"rewards/accuracies": 0.550000011920929, |
|
"rewards/chosen": -2.2613916397094727, |
|
"rewards/margins": 0.6226258873939514, |
|
"rewards/rejected": -2.8840174674987793, |
|
"step": 55 |
|
}, |
|
{ |
|
"epoch": 0.12823938017632916, |
|
"grad_norm": 18.13814698710649, |
|
"learning_rate": 4.988189843662815e-07, |
|
"logits/chosen": 1.9378535747528076, |
|
"logits/rejected": 1.9659252166748047, |
|
"logps/chosen": -1.1384503841400146, |
|
"logps/rejected": -1.0124545097351074, |
|
"loss": 1.7608, |
|
"rewards/accuracies": 0.550000011920929, |
|
"rewards/chosen": -2.846126079559326, |
|
"rewards/margins": -0.3149900436401367, |
|
"rewards/rejected": -2.5311360359191895, |
|
"step": 60 |
|
}, |
|
{ |
|
"epoch": 0.13892599519102325, |
|
"grad_norm": 31.166552691677705, |
|
"learning_rate": 4.977374404419837e-07, |
|
"logits/chosen": 2.1393017768859863, |
|
"logits/rejected": 2.103208303451538, |
|
"logps/chosen": -0.8744574785232544, |
|
"logps/rejected": -0.9523324966430664, |
|
"loss": 1.6552, |
|
"rewards/accuracies": 0.612500011920929, |
|
"rewards/chosen": -2.186143636703491, |
|
"rewards/margins": 0.19468756020069122, |
|
"rewards/rejected": -2.380831241607666, |
|
"step": 65 |
|
}, |
|
{ |
|
"epoch": 0.14961261020571734, |
|
"grad_norm": 52.728695245981314, |
|
"learning_rate": 4.963094133060148e-07, |
|
"logits/chosen": 2.0067527294158936, |
|
"logits/rejected": 1.9597012996673584, |
|
"logps/chosen": -0.9174124598503113, |
|
"logps/rejected": -1.1131672859191895, |
|
"loss": 1.7212, |
|
"rewards/accuracies": 0.550000011920929, |
|
"rewards/chosen": -2.2935314178466797, |
|
"rewards/margins": 0.4893869459629059, |
|
"rewards/rejected": -2.7829184532165527, |
|
"step": 70 |
|
}, |
|
{ |
|
"epoch": 0.16029922522041143, |
|
"grad_norm": 46.624386507512185, |
|
"learning_rate": 4.945369001834514e-07, |
|
"logits/chosen": 1.8767681121826172, |
|
"logits/rejected": 1.8939720392227173, |
|
"logps/chosen": -0.9977117776870728, |
|
"logps/rejected": -1.1487284898757935, |
|
"loss": 1.6147, |
|
"rewards/accuracies": 0.5874999761581421, |
|
"rewards/chosen": -2.494279384613037, |
|
"rewards/margins": 0.3775419294834137, |
|
"rewards/rejected": -2.8718209266662598, |
|
"step": 75 |
|
}, |
|
{ |
|
"epoch": 0.17098584023510552, |
|
"grad_norm": 13.45137638802328, |
|
"learning_rate": 4.924223800941717e-07, |
|
"logits/chosen": 1.912676453590393, |
|
"logits/rejected": 1.8263626098632812, |
|
"logps/chosen": -0.9859241247177124, |
|
"logps/rejected": -1.0560874938964844, |
|
"loss": 1.651, |
|
"rewards/accuracies": 0.5375000238418579, |
|
"rewards/chosen": -2.464810609817505, |
|
"rewards/margins": 0.17540840804576874, |
|
"rewards/rejected": -2.640218734741211, |
|
"step": 80 |
|
}, |
|
{ |
|
"epoch": 0.18167245524979964, |
|
"grad_norm": 23.876540575284714, |
|
"learning_rate": 4.899688103857222e-07, |
|
"logits/chosen": 1.9252907037734985, |
|
"logits/rejected": 1.9396631717681885, |
|
"logps/chosen": -0.9312021136283875, |
|
"logps/rejected": -1.1102412939071655, |
|
"loss": 1.618, |
|
"rewards/accuracies": 0.550000011920929, |
|
"rewards/chosen": -2.328005313873291, |
|
"rewards/margins": 0.44759804010391235, |
|
"rewards/rejected": -2.7756032943725586, |
|
"step": 85 |
|
}, |
|
{ |
|
"epoch": 0.19235907026449373, |
|
"grad_norm": 7.705041238913054, |
|
"learning_rate": 4.871796225971999e-07, |
|
"logits/chosen": 1.9238191843032837, |
|
"logits/rejected": 1.8918044567108154, |
|
"logps/chosen": -0.9711843729019165, |
|
"logps/rejected": -1.096145749092102, |
|
"loss": 1.6624, |
|
"rewards/accuracies": 0.512499988079071, |
|
"rewards/chosen": -2.4279608726501465, |
|
"rewards/margins": 0.3124033510684967, |
|
"rewards/rejected": -2.7403643131256104, |
|
"step": 90 |
|
}, |
|
{ |
|
"epoch": 0.20304568527918782, |
|
"grad_norm": 16.98405162222745, |
|
"learning_rate": 4.840587176599343e-07, |
|
"logits/chosen": 2.0161519050598145, |
|
"logits/rejected": 2.037721872329712, |
|
"logps/chosen": -0.9618253707885742, |
|
"logps/rejected": -1.1678836345672607, |
|
"loss": 1.6553, |
|
"rewards/accuracies": 0.637499988079071, |
|
"rewards/chosen": -2.4045631885528564, |
|
"rewards/margins": 0.5151457786560059, |
|
"rewards/rejected": -2.9197089672088623, |
|
"step": 95 |
|
}, |
|
{ |
|
"epoch": 0.2137323002938819, |
|
"grad_norm": 26.59166513674174, |
|
"learning_rate": 4.806104604416823e-07, |
|
"logits/chosen": 1.9551931619644165, |
|
"logits/rejected": 1.9663063287734985, |
|
"logps/chosen": -0.9686329960823059, |
|
"logps/rejected": -0.964672863483429, |
|
"loss": 1.6952, |
|
"rewards/accuracies": 0.48750001192092896, |
|
"rewards/chosen": -2.4215824604034424, |
|
"rewards/margins": -0.009900185279548168, |
|
"rewards/rejected": -2.41168212890625, |
|
"step": 100 |
|
}, |
|
{ |
|
"epoch": 0.224418915308576, |
|
"grad_norm": 22.597015867331866, |
|
"learning_rate": 4.768396736419662e-07, |
|
"logits/chosen": 2.0987541675567627, |
|
"logits/rejected": 2.1704182624816895, |
|
"logps/chosen": -0.8102301359176636, |
|
"logps/rejected": -1.0246448516845703, |
|
"loss": 1.6353, |
|
"rewards/accuracies": 0.574999988079071, |
|
"rewards/chosen": -2.0255753993988037, |
|
"rewards/margins": 0.5360367894172668, |
|
"rewards/rejected": -2.561612367630005, |
|
"step": 105 |
|
}, |
|
{ |
|
"epoch": 0.2351055303232701, |
|
"grad_norm": 7.710202530561847, |
|
"learning_rate": 4.7275163104709194e-07, |
|
"logits/chosen": 1.8211323022842407, |
|
"logits/rejected": 1.8467600345611572, |
|
"logps/chosen": -0.8923318982124329, |
|
"logps/rejected": -0.9886472821235657, |
|
"loss": 1.6826, |
|
"rewards/accuracies": 0.612500011920929, |
|
"rewards/chosen": -2.2308297157287598, |
|
"rewards/margins": 0.24078825116157532, |
|
"rewards/rejected": -2.4716179370880127, |
|
"step": 110 |
|
}, |
|
{ |
|
"epoch": 0.2457921453379642, |
|
"grad_norm": 9.211974257959199, |
|
"learning_rate": 4.683520501542824e-07, |
|
"logits/chosen": 1.822178840637207, |
|
"logits/rejected": 1.8460540771484375, |
|
"logps/chosen": -0.937783420085907, |
|
"logps/rejected": -0.9862428903579712, |
|
"loss": 1.6087, |
|
"rewards/accuracies": 0.48750001192092896, |
|
"rewards/chosen": -2.34445858001709, |
|
"rewards/margins": 0.12114912271499634, |
|
"rewards/rejected": -2.4656076431274414, |
|
"step": 115 |
|
}, |
|
{ |
|
"epoch": 0.2564787603526583, |
|
"grad_norm": 17.97332526477704, |
|
"learning_rate": 4.636470841752404e-07, |
|
"logits/chosen": 1.9925601482391357, |
|
"logits/rejected": 2.0132079124450684, |
|
"logps/chosen": -0.8938210606575012, |
|
"logps/rejected": -0.9105901718139648, |
|
"loss": 1.6647, |
|
"rewards/accuracies": 0.4749999940395355, |
|
"rewards/chosen": -2.2345528602600098, |
|
"rewards/margins": 0.04192250221967697, |
|
"rewards/rejected": -2.276475429534912, |
|
"step": 120 |
|
}, |
|
{ |
|
"epoch": 0.2671653753673524, |
|
"grad_norm": 21.241131663632817, |
|
"learning_rate": 4.5864331343032565e-07, |
|
"logits/chosen": 1.821298360824585, |
|
"logits/rejected": 1.8526639938354492, |
|
"logps/chosen": -0.9126847386360168, |
|
"logps/rejected": -1.0501748323440552, |
|
"loss": 1.7558, |
|
"rewards/accuracies": 0.625, |
|
"rewards/chosen": -2.281712055206299, |
|
"rewards/margins": 0.3437250852584839, |
|
"rewards/rejected": -2.6254372596740723, |
|
"step": 125 |
|
}, |
|
{ |
|
"epoch": 0.2778519903820465, |
|
"grad_norm": 18.947973877308787, |
|
"learning_rate": 4.533477361453819e-07, |
|
"logits/chosen": 2.178891658782959, |
|
"logits/rejected": 2.143307685852051, |
|
"logps/chosen": -0.9617658853530884, |
|
"logps/rejected": -1.223284125328064, |
|
"loss": 1.6892, |
|
"rewards/accuracies": 0.5249999761581421, |
|
"rewards/chosen": -2.404414653778076, |
|
"rewards/margins": 0.6537958383560181, |
|
"rewards/rejected": -3.0582103729248047, |
|
"step": 130 |
|
}, |
|
{ |
|
"epoch": 0.2885386053967406, |
|
"grad_norm": 13.76012227966358, |
|
"learning_rate": 4.4776775866408533e-07, |
|
"logits/chosen": 1.8531124591827393, |
|
"logits/rejected": 1.8338654041290283, |
|
"logps/chosen": -0.8667293787002563, |
|
"logps/rejected": -0.9339718818664551, |
|
"loss": 1.658, |
|
"rewards/accuracies": 0.5249999761581421, |
|
"rewards/chosen": -2.166823148727417, |
|
"rewards/margins": 0.16810639202594757, |
|
"rewards/rejected": -2.334929943084717, |
|
"step": 135 |
|
}, |
|
{ |
|
"epoch": 0.2992252204114347, |
|
"grad_norm": 12.832812020829884, |
|
"learning_rate": 4.4191118508950277e-07, |
|
"logits/chosen": 1.8038040399551392, |
|
"logits/rejected": 1.7551212310791016, |
|
"logps/chosen": -0.8944159746170044, |
|
"logps/rejected": -1.0033025741577148, |
|
"loss": 1.6771, |
|
"rewards/accuracies": 0.637499988079071, |
|
"rewards/chosen": -2.236039638519287, |
|
"rewards/margins": 0.2722164988517761, |
|
"rewards/rejected": -2.508256435394287, |
|
"step": 140 |
|
}, |
|
{ |
|
"epoch": 0.30991183542612877, |
|
"grad_norm": 9.09964498419542, |
|
"learning_rate": 4.357862063693485e-07, |
|
"logits/chosen": 2.027944564819336, |
|
"logits/rejected": 2.005831718444824, |
|
"logps/chosen": -0.8879717588424683, |
|
"logps/rejected": -0.9542590975761414, |
|
"loss": 1.7133, |
|
"rewards/accuracies": 0.550000011920929, |
|
"rewards/chosen": -2.2199294567108154, |
|
"rewards/margins": 0.16571831703186035, |
|
"rewards/rejected": -2.3856475353240967, |
|
"step": 145 |
|
}, |
|
{ |
|
"epoch": 0.32059845044082286, |
|
"grad_norm": 24.685014270210495, |
|
"learning_rate": 4.294013888402029e-07, |
|
"logits/chosen": 2.138366222381592, |
|
"logits/rejected": 2.2007527351379395, |
|
"logps/chosen": -0.8750768899917603, |
|
"logps/rejected": -1.0953787565231323, |
|
"loss": 1.6625, |
|
"rewards/accuracies": 0.574999988079071, |
|
"rewards/chosen": -2.187692165374756, |
|
"rewards/margins": 0.5507549047470093, |
|
"rewards/rejected": -2.7384471893310547, |
|
"step": 150 |
|
}, |
|
{ |
|
"epoch": 0.33128506545551695, |
|
"grad_norm": 24.61662782262514, |
|
"learning_rate": 4.227656622467162e-07, |
|
"logits/chosen": 1.911320686340332, |
|
"logits/rejected": 1.9191503524780273, |
|
"logps/chosen": -0.8599516153335571, |
|
"logps/rejected": -1.1553843021392822, |
|
"loss": 1.6495, |
|
"rewards/accuracies": 0.550000011920929, |
|
"rewards/chosen": -2.149878978729248, |
|
"rewards/margins": 0.7385817766189575, |
|
"rewards/rejected": -2.888460874557495, |
|
"step": 155 |
|
}, |
|
{ |
|
"epoch": 0.34197168047021104, |
|
"grad_norm": 9.846236022517108, |
|
"learning_rate": 4.158883072525528e-07, |
|
"logits/chosen": 2.061549425125122, |
|
"logits/rejected": 2.0822834968566895, |
|
"logps/chosen": -0.8990328907966614, |
|
"logps/rejected": -0.985998809337616, |
|
"loss": 1.6348, |
|
"rewards/accuracies": 0.4625000059604645, |
|
"rewards/chosen": -2.247582197189331, |
|
"rewards/margins": 0.21741452813148499, |
|
"rewards/rejected": -2.464996814727783, |
|
"step": 160 |
|
}, |
|
{ |
|
"epoch": 0.3526582954849052, |
|
"grad_norm": 19.106656482086205, |
|
"learning_rate": 4.087789424605447e-07, |
|
"logits/chosen": 2.265204906463623, |
|
"logits/rejected": 2.245617151260376, |
|
"logps/chosen": -0.818160355091095, |
|
"logps/rejected": -0.9623705744743347, |
|
"loss": 1.618, |
|
"rewards/accuracies": 0.5, |
|
"rewards/chosen": -2.045400857925415, |
|
"rewards/margins": 0.3605257570743561, |
|
"rewards/rejected": -2.405926465988159, |
|
"step": 165 |
|
}, |
|
{ |
|
"epoch": 0.36334491049959927, |
|
"grad_norm": 14.919110153191296, |
|
"learning_rate": 4.0144751096020497e-07, |
|
"logits/chosen": 1.9928451776504517, |
|
"logits/rejected": 1.9697974920272827, |
|
"logps/chosen": -1.013695478439331, |
|
"logps/rejected": -1.2301123142242432, |
|
"loss": 1.5848, |
|
"rewards/accuracies": 0.550000011920929, |
|
"rewards/chosen": -2.534238815307617, |
|
"rewards/margins": 0.5410416722297668, |
|
"rewards/rejected": -3.0752806663513184, |
|
"step": 170 |
|
}, |
|
{ |
|
"epoch": 0.37403152551429336, |
|
"grad_norm": 11.830869809507767, |
|
"learning_rate": 3.939042664214184e-07, |
|
"logits/chosen": 2.031292676925659, |
|
"logits/rejected": 2.066436767578125, |
|
"logps/chosen": -0.97381991147995, |
|
"logps/rejected": -1.2112761735916138, |
|
"loss": 1.5954, |
|
"rewards/accuracies": 0.6000000238418579, |
|
"rewards/chosen": -2.4345498085021973, |
|
"rewards/margins": 0.5936408638954163, |
|
"rewards/rejected": -3.0281906127929688, |
|
"step": 175 |
|
}, |
|
{ |
|
"epoch": 0.38471814052898745, |
|
"grad_norm": 14.628170167780459, |
|
"learning_rate": 3.8615975875375676e-07, |
|
"logits/chosen": 2.258387804031372, |
|
"logits/rejected": 2.3218982219696045, |
|
"logps/chosen": -1.0137016773223877, |
|
"logps/rejected": -1.1529831886291504, |
|
"loss": 1.6695, |
|
"rewards/accuracies": 0.6499999761581421, |
|
"rewards/chosen": -2.534254312515259, |
|
"rewards/margins": 0.3482036888599396, |
|
"rewards/rejected": -2.882457971572876, |
|
"step": 180 |
|
}, |
|
{ |
|
"epoch": 0.39540475554368154, |
|
"grad_norm": 16.49947715615614, |
|
"learning_rate": 3.7822481935147655e-07, |
|
"logits/chosen": 2.0560050010681152, |
|
"logits/rejected": 2.1167311668395996, |
|
"logps/chosen": -0.9939271211624146, |
|
"logps/rejected": -1.1564804315567017, |
|
"loss": 1.6754, |
|
"rewards/accuracies": 0.5249999761581421, |
|
"rewards/chosen": -2.4848177433013916, |
|
"rewards/margins": 0.40638333559036255, |
|
"rewards/rejected": -2.8912010192871094, |
|
"step": 185 |
|
}, |
|
{ |
|
"epoch": 0.40609137055837563, |
|
"grad_norm": 13.5081717360946, |
|
"learning_rate": 3.7011054594483443e-07, |
|
"logits/chosen": 2.1290013790130615, |
|
"logits/rejected": 2.223310947418213, |
|
"logps/chosen": -0.9123891592025757, |
|
"logps/rejected": -1.1287665367126465, |
|
"loss": 1.5831, |
|
"rewards/accuracies": 0.675000011920929, |
|
"rewards/chosen": -2.280972957611084, |
|
"rewards/margins": 0.5409437417984009, |
|
"rewards/rejected": -2.8219165802001953, |
|
"step": 190 |
|
}, |
|
{ |
|
"epoch": 0.4167779855730697, |
|
"grad_norm": 17.61863203205812, |
|
"learning_rate": 3.618282870789081e-07, |
|
"logits/chosen": 2.1478934288024902, |
|
"logits/rejected": 2.1439719200134277, |
|
"logps/chosen": -0.974346935749054, |
|
"logps/rejected": -1.1090627908706665, |
|
"loss": 1.64, |
|
"rewards/accuracies": 0.5249999761581421, |
|
"rewards/chosen": -2.4358673095703125, |
|
"rewards/margins": 0.33678966760635376, |
|
"rewards/rejected": -2.7726569175720215, |
|
"step": 195 |
|
}, |
|
{ |
|
"epoch": 0.4274646005877638, |
|
"grad_norm": 12.457201720784246, |
|
"learning_rate": 3.5338962624163016e-07, |
|
"logits/chosen": 2.2304863929748535, |
|
"logits/rejected": 2.2417056560516357, |
|
"logps/chosen": -0.9161917567253113, |
|
"logps/rejected": -1.0349229574203491, |
|
"loss": 1.672, |
|
"rewards/accuracies": 0.6499999761581421, |
|
"rewards/chosen": -2.2904791831970215, |
|
"rewards/margins": 0.29682818055152893, |
|
"rewards/rejected": -2.5873074531555176, |
|
"step": 200 |
|
}, |
|
{ |
|
"epoch": 0.4381512156024579, |
|
"grad_norm": 25.419906192801392, |
|
"learning_rate": 3.448063656632321e-07, |
|
"logits/chosen": 2.2168259620666504, |
|
"logits/rejected": 2.2494800090789795, |
|
"logps/chosen": -0.9495501518249512, |
|
"logps/rejected": -1.0588171482086182, |
|
"loss": 1.6462, |
|
"rewards/accuracies": 0.550000011920929, |
|
"rewards/chosen": -2.373875379562378, |
|
"rewards/margins": 0.2731671929359436, |
|
"rewards/rejected": -2.647042751312256, |
|
"step": 205 |
|
}, |
|
{ |
|
"epoch": 0.448837830617152, |
|
"grad_norm": 31.43495532901771, |
|
"learning_rate": 3.360905098097587e-07, |
|
"logits/chosen": 2.0355324745178223, |
|
"logits/rejected": 2.0058634281158447, |
|
"logps/chosen": -0.9030116200447083, |
|
"logps/rejected": -1.100672960281372, |
|
"loss": 1.7332, |
|
"rewards/accuracies": 0.5249999761581421, |
|
"rewards/chosen": -2.2575290203094482, |
|
"rewards/margins": 0.49415332078933716, |
|
"rewards/rejected": -2.751682758331299, |
|
"step": 210 |
|
}, |
|
{ |
|
"epoch": 0.45952444563184613, |
|
"grad_norm": 9.866165199153304, |
|
"learning_rate": 3.272542485937368e-07, |
|
"logits/chosen": 2.188046932220459, |
|
"logits/rejected": 2.209195375442505, |
|
"logps/chosen": -0.8452509045600891, |
|
"logps/rejected": -0.9856406450271606, |
|
"loss": 1.6173, |
|
"rewards/accuracies": 0.574999988079071, |
|
"rewards/chosen": -2.1131272315979004, |
|
"rewards/margins": 0.3509744107723236, |
|
"rewards/rejected": -2.464101791381836, |
|
"step": 215 |
|
}, |
|
{ |
|
"epoch": 0.4702110606465402, |
|
"grad_norm": 11.268271977292386, |
|
"learning_rate": 3.1830994032548e-07, |
|
"logits/chosen": 2.2558753490448, |
|
"logits/rejected": 2.271909475326538, |
|
"logps/chosen": -0.9621235728263855, |
|
"logps/rejected": -1.0470136404037476, |
|
"loss": 1.6623, |
|
"rewards/accuracies": 0.5625, |
|
"rewards/chosen": -2.405308961868286, |
|
"rewards/margins": 0.21222510933876038, |
|
"rewards/rejected": -2.6175341606140137, |
|
"step": 220 |
|
}, |
|
{ |
|
"epoch": 0.4808976756612343, |
|
"grad_norm": 8.220154130520008, |
|
"learning_rate": 3.0927009442887437e-07, |
|
"logits/chosen": 2.033316135406494, |
|
"logits/rejected": 2.0451745986938477, |
|
"logps/chosen": -0.9348015785217285, |
|
"logps/rejected": -0.9307538866996765, |
|
"loss": 1.7025, |
|
"rewards/accuracies": 0.512499988079071, |
|
"rewards/chosen": -2.3370039463043213, |
|
"rewards/margins": -0.010119226761162281, |
|
"rewards/rejected": -2.3268847465515137, |
|
"step": 225 |
|
}, |
|
{ |
|
"epoch": 0.4915842906759284, |
|
"grad_norm": 8.761484125412297, |
|
"learning_rate": 3.001473539458182e-07, |
|
"logits/chosen": 2.039243221282959, |
|
"logits/rejected": 2.0847630500793457, |
|
"logps/chosen": -0.9077224731445312, |
|
"logps/rejected": -0.9697052240371704, |
|
"loss": 1.6369, |
|
"rewards/accuracies": 0.5375000238418579, |
|
"rewards/chosen": -2.269306182861328, |
|
"rewards/margins": 0.15495693683624268, |
|
"rewards/rejected": -2.4242632389068604, |
|
"step": 230 |
|
}, |
|
{ |
|
"epoch": 0.5022709056906225, |
|
"grad_norm": 21.069821104434805, |
|
"learning_rate": 2.909544778537844e-07, |
|
"logits/chosen": 2.1515793800354004, |
|
"logits/rejected": 2.0550239086151123, |
|
"logps/chosen": -0.9554904699325562, |
|
"logps/rejected": -1.0575495958328247, |
|
"loss": 1.6471, |
|
"rewards/accuracies": 0.4625000059604645, |
|
"rewards/chosen": -2.3887264728546143, |
|
"rewards/margins": 0.2551475167274475, |
|
"rewards/rejected": -2.643873691558838, |
|
"step": 235 |
|
}, |
|
{ |
|
"epoch": 0.5129575207053166, |
|
"grad_norm": 18.45689011337618, |
|
"learning_rate": 2.817043232212371e-07, |
|
"logits/chosen": 2.3277316093444824, |
|
"logits/rejected": 2.359060287475586, |
|
"logps/chosen": -0.9175651669502258, |
|
"logps/rejected": -1.075629472732544, |
|
"loss": 1.5723, |
|
"rewards/accuracies": 0.5249999761581421, |
|
"rewards/chosen": -2.2939131259918213, |
|
"rewards/margins": 0.39516064524650574, |
|
"rewards/rejected": -2.6890735626220703, |
|
"step": 240 |
|
}, |
|
{ |
|
"epoch": 0.5236441357200107, |
|
"grad_norm": 5.945491870174955, |
|
"learning_rate": 2.7240982722585837e-07, |
|
"logits/chosen": 2.2272980213165283, |
|
"logits/rejected": 2.216254472732544, |
|
"logps/chosen": -0.887627124786377, |
|
"logps/rejected": -1.0360788106918335, |
|
"loss": 1.6259, |
|
"rewards/accuracies": 0.5249999761581421, |
|
"rewards/chosen": -2.2190678119659424, |
|
"rewards/margins": 0.3711293637752533, |
|
"rewards/rejected": -2.5901970863342285, |
|
"step": 245 |
|
}, |
|
{ |
|
"epoch": 0.5343307507347048, |
|
"grad_norm": 10.042684286997996, |
|
"learning_rate": 2.63083989060736e-07, |
|
"logits/chosen": 2.162696361541748, |
|
"logits/rejected": 2.178349494934082, |
|
"logps/chosen": -0.9883350133895874, |
|
"logps/rejected": -1.1994340419769287, |
|
"loss": 1.6589, |
|
"rewards/accuracies": 0.625, |
|
"rewards/chosen": -2.4708378314971924, |
|
"rewards/margins": 0.5277475118637085, |
|
"rewards/rejected": -2.9985852241516113, |
|
"step": 250 |
|
}, |
|
{ |
|
"epoch": 0.5450173657493989, |
|
"grad_norm": 10.520900876521566, |
|
"learning_rate": 2.537398517538159e-07, |
|
"logits/chosen": 2.2908027172088623, |
|
"logits/rejected": 2.235353708267212, |
|
"logps/chosen": -0.9431841969490051, |
|
"logps/rejected": -1.015700101852417, |
|
"loss": 1.6741, |
|
"rewards/accuracies": 0.5249999761581421, |
|
"rewards/chosen": -2.3579602241516113, |
|
"rewards/margins": 0.18129023909568787, |
|
"rewards/rejected": -2.539250612258911, |
|
"step": 255 |
|
}, |
|
{ |
|
"epoch": 0.555703980764093, |
|
"grad_norm": 9.313778052567603, |
|
"learning_rate": 2.4439048392604877e-07, |
|
"logits/chosen": 2.1532225608825684, |
|
"logits/rejected": 2.2637267112731934, |
|
"logps/chosen": -0.9131342768669128, |
|
"logps/rejected": -1.2010549306869507, |
|
"loss": 1.5779, |
|
"rewards/accuracies": 0.699999988079071, |
|
"rewards/chosen": -2.2828357219696045, |
|
"rewards/margins": 0.7198014259338379, |
|
"rewards/rejected": -3.0026373863220215, |
|
"step": 260 |
|
}, |
|
{ |
|
"epoch": 0.566390595778787, |
|
"grad_norm": 17.127300405069718, |
|
"learning_rate": 2.3504896151374144e-07, |
|
"logits/chosen": 2.2980525493621826, |
|
"logits/rejected": 2.2274222373962402, |
|
"logps/chosen": -0.9766885042190552, |
|
"logps/rejected": -1.0872061252593994, |
|
"loss": 1.6741, |
|
"rewards/accuracies": 0.550000011920929, |
|
"rewards/chosen": -2.4417214393615723, |
|
"rewards/margins": 0.2762938141822815, |
|
"rewards/rejected": -2.718015193939209, |
|
"step": 265 |
|
}, |
|
{ |
|
"epoch": 0.5770772107934812, |
|
"grad_norm": 14.096322545468484, |
|
"learning_rate": 2.2572834948067795e-07, |
|
"logits/chosen": 2.13866925239563, |
|
"logits/rejected": 2.09075665473938, |
|
"logps/chosen": -0.866176962852478, |
|
"logps/rejected": -1.0079139471054077, |
|
"loss": 1.6445, |
|
"rewards/accuracies": 0.5, |
|
"rewards/chosen": -2.165442705154419, |
|
"rewards/margins": 0.3543422520160675, |
|
"rewards/rejected": -2.519784688949585, |
|
"step": 270 |
|
}, |
|
{ |
|
"epoch": 0.5877638258081752, |
|
"grad_norm": 11.519350142557252, |
|
"learning_rate": 2.164416835455862e-07, |
|
"logits/chosen": 1.9599807262420654, |
|
"logits/rejected": 1.9166367053985596, |
|
"logps/chosen": -0.9016801714897156, |
|
"logps/rejected": -1.060001015663147, |
|
"loss": 1.6308, |
|
"rewards/accuracies": 0.5249999761581421, |
|
"rewards/chosen": -2.2542004585266113, |
|
"rewards/margins": 0.39580219984054565, |
|
"rewards/rejected": -2.6500024795532227, |
|
"step": 275 |
|
}, |
|
{ |
|
"epoch": 0.5984504408228694, |
|
"grad_norm": 9.94425188158257, |
|
"learning_rate": 2.072019519505062e-07, |
|
"logits/chosen": 2.006683349609375, |
|
"logits/rejected": 1.9877275228500366, |
|
"logps/chosen": -0.9040509462356567, |
|
"logps/rejected": -1.0043412446975708, |
|
"loss": 1.6201, |
|
"rewards/accuracies": 0.512499988079071, |
|
"rewards/chosen": -2.260127305984497, |
|
"rewards/margins": 0.2507258355617523, |
|
"rewards/rejected": -2.5108530521392822, |
|
"step": 280 |
|
}, |
|
{ |
|
"epoch": 0.6091370558375635, |
|
"grad_norm": 17.126362799060438, |
|
"learning_rate": 1.980220772955602e-07, |
|
"logits/chosen": 2.040419816970825, |
|
"logits/rejected": 1.9795433282852173, |
|
"logps/chosen": -1.0711162090301514, |
|
"logps/rejected": -1.1054927110671997, |
|
"loss": 1.6696, |
|
"rewards/accuracies": 0.48750001192092896, |
|
"rewards/chosen": -2.677790641784668, |
|
"rewards/margins": 0.08594076335430145, |
|
"rewards/rejected": -2.7637314796447754, |
|
"step": 285 |
|
}, |
|
{ |
|
"epoch": 0.6198236708522575, |
|
"grad_norm": 13.492215975630849, |
|
"learning_rate": 1.8891489846552644e-07, |
|
"logits/chosen": 2.3427670001983643, |
|
"logits/rejected": 2.3189902305603027, |
|
"logps/chosen": -0.9356976747512817, |
|
"logps/rejected": -1.0561234951019287, |
|
"loss": 1.6313, |
|
"rewards/accuracies": 0.5249999761581421, |
|
"rewards/chosen": -2.3392443656921387, |
|
"rewards/margins": 0.30106478929519653, |
|
"rewards/rejected": -2.6403088569641113, |
|
"step": 290 |
|
}, |
|
{ |
|
"epoch": 0.6305102858669517, |
|
"grad_norm": 6.893975837082491, |
|
"learning_rate": 1.7989315267349933e-07, |
|
"logits/chosen": 2.3545644283294678, |
|
"logits/rejected": 2.3726115226745605, |
|
"logps/chosen": -0.9937087893486023, |
|
"logps/rejected": -1.0849647521972656, |
|
"loss": 1.6816, |
|
"rewards/accuracies": 0.574999988079071, |
|
"rewards/chosen": -2.484271764755249, |
|
"rewards/margins": 0.22813992202281952, |
|
"rewards/rejected": -2.712411642074585, |
|
"step": 295 |
|
}, |
|
{ |
|
"epoch": 0.6411969008816457, |
|
"grad_norm": 6.6767839699246405, |
|
"learning_rate": 1.7096945764674398e-07, |
|
"logits/chosen": 2.0134410858154297, |
|
"logits/rejected": 1.9672603607177734, |
|
"logps/chosen": -1.038081169128418, |
|
"logps/rejected": -1.0271114110946655, |
|
"loss": 1.6598, |
|
"rewards/accuracies": 0.4375, |
|
"rewards/chosen": -2.595202922821045, |
|
"rewards/margins": -0.027424562722444534, |
|
"rewards/rejected": -2.5677783489227295, |
|
"step": 300 |
|
}, |
|
{ |
|
"epoch": 0.6518835158963399, |
|
"grad_norm": 8.402566753074327, |
|
"learning_rate": 1.621562939796643e-07, |
|
"logits/chosen": 2.099931240081787, |
|
"logits/rejected": 2.131023406982422, |
|
"logps/chosen": -0.9801462888717651, |
|
"logps/rejected": -1.1142746210098267, |
|
"loss": 1.6239, |
|
"rewards/accuracies": 0.4749999940395355, |
|
"rewards/chosen": -2.4503657817840576, |
|
"rewards/margins": 0.33532071113586426, |
|
"rewards/rejected": -2.785686492919922, |
|
"step": 305 |
|
}, |
|
{ |
|
"epoch": 0.6625701309110339, |
|
"grad_norm": 13.484008579562406, |
|
"learning_rate": 1.5346598767856345e-07, |
|
"logits/chosen": 2.3738818168640137, |
|
"logits/rejected": 2.2915918827056885, |
|
"logps/chosen": -0.9194884300231934, |
|
"logps/rejected": -1.1123372316360474, |
|
"loss": 1.6509, |
|
"rewards/accuracies": 0.612500011920929, |
|
"rewards/chosen": -2.2987208366394043, |
|
"rewards/margins": 0.4821222722530365, |
|
"rewards/rejected": -2.7808432579040527, |
|
"step": 310 |
|
}, |
|
{ |
|
"epoch": 0.673256745925728, |
|
"grad_norm": 11.276062854222866, |
|
"learning_rate": 1.4491069292260866e-07, |
|
"logits/chosen": 2.0688905715942383, |
|
"logits/rejected": 2.0706334114074707, |
|
"logps/chosen": -0.9416360855102539, |
|
"logps/rejected": -0.9889582395553589, |
|
"loss": 1.6263, |
|
"rewards/accuracies": 0.48750001192092896, |
|
"rewards/chosen": -2.354090452194214, |
|
"rewards/margins": 0.11830528825521469, |
|
"rewards/rejected": -2.472395658493042, |
|
"step": 315 |
|
}, |
|
{ |
|
"epoch": 0.6839433609404221, |
|
"grad_norm": 12.406779732367644, |
|
"learning_rate": 1.365023750651133e-07, |
|
"logits/chosen": 2.029076337814331, |
|
"logits/rejected": 2.07525634765625, |
|
"logps/chosen": -0.8669158220291138, |
|
"logps/rejected": -1.1443169116973877, |
|
"loss": 1.5861, |
|
"rewards/accuracies": 0.625, |
|
"rewards/chosen": -2.1672892570495605, |
|
"rewards/margins": 0.6935025453567505, |
|
"rewards/rejected": -2.8607916831970215, |
|
"step": 320 |
|
}, |
|
{ |
|
"epoch": 0.6946299759551162, |
|
"grad_norm": 9.410463016483124, |
|
"learning_rate": 1.2825279389890818e-07, |
|
"logits/chosen": 2.2680742740631104, |
|
"logits/rejected": 2.2626852989196777, |
|
"logps/chosen": -0.9127257466316223, |
|
"logps/rejected": -1.036811113357544, |
|
"loss": 1.5807, |
|
"rewards/accuracies": 0.550000011920929, |
|
"rewards/chosen": -2.2818140983581543, |
|
"rewards/margins": 0.31021344661712646, |
|
"rewards/rejected": -2.5920276641845703, |
|
"step": 325 |
|
}, |
|
{ |
|
"epoch": 0.7053165909698104, |
|
"grad_norm": 13.015902838409264, |
|
"learning_rate": 1.201734872092077e-07, |
|
"logits/chosen": 2.0204429626464844, |
|
"logits/rejected": 1.9889049530029297, |
|
"logps/chosen": -0.9624814987182617, |
|
"logps/rejected": -1.4148118495941162, |
|
"loss": 1.6247, |
|
"rewards/accuracies": 0.550000011920929, |
|
"rewards/chosen": -2.4062037467956543, |
|
"rewards/margins": 1.130826473236084, |
|
"rewards/rejected": -3.537029981613159, |
|
"step": 330 |
|
}, |
|
{ |
|
"epoch": 0.7160032059845044, |
|
"grad_norm": 10.457412577224535, |
|
"learning_rate": 1.1227575463697439e-07, |
|
"logits/chosen": 2.1002488136291504, |
|
"logits/rejected": 2.1578989028930664, |
|
"logps/chosen": -0.9219167828559875, |
|
"logps/rejected": -1.0541021823883057, |
|
"loss": 1.6126, |
|
"rewards/accuracies": 0.574999988079071, |
|
"rewards/chosen": -2.3047919273376465, |
|
"rewards/margins": 0.33046382665634155, |
|
"rewards/rejected": -2.635255813598633, |
|
"step": 335 |
|
}, |
|
{ |
|
"epoch": 0.7266898209991985, |
|
"grad_norm": 13.23573306912224, |
|
"learning_rate": 1.0457064187534861e-07, |
|
"logits/chosen": 2.2117228507995605, |
|
"logits/rejected": 2.291081666946411, |
|
"logps/chosen": -0.8333222270011902, |
|
"logps/rejected": -1.0662933588027954, |
|
"loss": 1.611, |
|
"rewards/accuracies": 0.574999988079071, |
|
"rewards/chosen": -2.083305597305298, |
|
"rewards/margins": 0.5824280977249146, |
|
"rewards/rejected": -2.665733575820923, |
|
"step": 340 |
|
}, |
|
{ |
|
"epoch": 0.7373764360138926, |
|
"grad_norm": 7.317920448430426, |
|
"learning_rate": 9.706892522124838e-08, |
|
"logits/chosen": 2.1216635704040527, |
|
"logits/rejected": 2.178093194961548, |
|
"logps/chosen": -1.0588462352752686, |
|
"logps/rejected": -1.0970056056976318, |
|
"loss": 1.5811, |
|
"rewards/accuracies": 0.512499988079071, |
|
"rewards/chosen": -2.647115468978882, |
|
"rewards/margins": 0.09539877623319626, |
|
"rewards/rejected": -2.742514133453369, |
|
"step": 345 |
|
}, |
|
{ |
|
"epoch": 0.7480630510285867, |
|
"grad_norm": 14.193162676817792, |
|
"learning_rate": 8.978109650374396e-08, |
|
"logits/chosen": 2.2546029090881348, |
|
"logits/rejected": 2.294689893722534, |
|
"logps/chosen": -0.9023507833480835, |
|
"logps/rejected": -1.162570595741272, |
|
"loss": 1.6171, |
|
"rewards/accuracies": 0.5874999761581421, |
|
"rewards/chosen": -2.2558767795562744, |
|
"rewards/margins": 0.6505497694015503, |
|
"rewards/rejected": -2.9064266681671143, |
|
"step": 350 |
|
}, |
|
{ |
|
"epoch": 0.7587496660432808, |
|
"grad_norm": 12.52863776377329, |
|
"learning_rate": 8.271734841028552e-08, |
|
"logits/chosen": 2.245272636413574, |
|
"logits/rejected": 2.2576751708984375, |
|
"logps/chosen": -1.005053997039795, |
|
"logps/rejected": -1.0007752180099487, |
|
"loss": 1.6536, |
|
"rewards/accuracies": 0.4749999940395355, |
|
"rewards/chosen": -2.512634754180908, |
|
"rewards/margins": -0.010696577839553356, |
|
"rewards/rejected": -2.5019383430480957, |
|
"step": 355 |
|
}, |
|
{ |
|
"epoch": 0.7694362810579749, |
|
"grad_norm": 10.369189534811127, |
|
"learning_rate": 7.588756023130833e-08, |
|
"logits/chosen": 2.318972587585449, |
|
"logits/rejected": 2.281419515609741, |
|
"logps/chosen": -1.0578750371932983, |
|
"logps/rejected": -1.3604905605316162, |
|
"loss": 1.6205, |
|
"rewards/accuracies": 0.625, |
|
"rewards/chosen": -2.6446874141693115, |
|
"rewards/margins": 0.7565388679504395, |
|
"rewards/rejected": -3.40122652053833, |
|
"step": 360 |
|
}, |
|
{ |
|
"epoch": 0.7801228960726689, |
|
"grad_norm": 6.075365195709392, |
|
"learning_rate": 6.930128404315214e-08, |
|
"logits/chosen": 2.0289673805236816, |
|
"logits/rejected": 2.0680978298187256, |
|
"logps/chosen": -0.976359486579895, |
|
"logps/rejected": -1.0074145793914795, |
|
"loss": 1.6246, |
|
"rewards/accuracies": 0.512499988079071, |
|
"rewards/chosen": -2.440898895263672, |
|
"rewards/margins": 0.07763750851154327, |
|
"rewards/rejected": -2.518536329269409, |
|
"step": 365 |
|
}, |
|
{ |
|
"epoch": 0.7908095110873631, |
|
"grad_norm": 16.214823251732284, |
|
"learning_rate": 6.296773134861824e-08, |
|
"logits/chosen": 2.2002553939819336, |
|
"logits/rejected": 2.2343714237213135, |
|
"logps/chosen": -0.9719416499137878, |
|
"logps/rejected": -1.023166537284851, |
|
"loss": 1.6894, |
|
"rewards/accuracies": 0.5874999761581421, |
|
"rewards/chosen": -2.429853916168213, |
|
"rewards/margins": 0.12806224822998047, |
|
"rewards/rejected": -2.5579161643981934, |
|
"step": 370 |
|
}, |
|
{ |
|
"epoch": 0.8014961261020572, |
|
"grad_norm": 10.840080143128308, |
|
"learning_rate": 5.6895760193850145e-08, |
|
"logits/chosen": 2.1092734336853027, |
|
"logits/rejected": 2.168839693069458, |
|
"logps/chosen": -1.0623986721038818, |
|
"logps/rejected": -1.1893246173858643, |
|
"loss": 1.6248, |
|
"rewards/accuracies": 0.675000011920929, |
|
"rewards/chosen": -2.6559970378875732, |
|
"rewards/margins": 0.31731492280960083, |
|
"rewards/rejected": -2.9733119010925293, |
|
"step": 375 |
|
}, |
|
{ |
|
"epoch": 0.8121827411167513, |
|
"grad_norm": 14.986128731201607, |
|
"learning_rate": 5.109386277955477e-08, |
|
"logits/chosen": 2.400944232940674, |
|
"logits/rejected": 2.424126625061035, |
|
"logps/chosen": -1.0006974935531616, |
|
"logps/rejected": -1.0918558835983276, |
|
"loss": 1.6152, |
|
"rewards/accuracies": 0.48750001192092896, |
|
"rewards/chosen": -2.501743793487549, |
|
"rewards/margins": 0.22789612412452698, |
|
"rewards/rejected": -2.729639768600464, |
|
"step": 380 |
|
}, |
|
{ |
|
"epoch": 0.8228693561314454, |
|
"grad_norm": 11.04509407630343, |
|
"learning_rate": 4.557015358389216e-08, |
|
"logits/chosen": 2.2161147594451904, |
|
"logits/rejected": 2.255049705505371, |
|
"logps/chosen": -0.9763911962509155, |
|
"logps/rejected": -1.2085295915603638, |
|
"loss": 1.6246, |
|
"rewards/accuracies": 0.5625, |
|
"rewards/chosen": -2.4409780502319336, |
|
"rewards/margins": 0.5803458094596863, |
|
"rewards/rejected": -3.0213241577148438, |
|
"step": 385 |
|
}, |
|
{ |
|
"epoch": 0.8335559711461394, |
|
"grad_norm": 17.08993723906078, |
|
"learning_rate": 4.0332358013644015e-08, |
|
"logits/chosen": 2.3219351768493652, |
|
"logits/rejected": 2.316385507583618, |
|
"logps/chosen": -0.8949235081672668, |
|
"logps/rejected": -1.2672293186187744, |
|
"loss": 1.6161, |
|
"rewards/accuracies": 0.5375000238418579, |
|
"rewards/chosen": -2.2373085021972656, |
|
"rewards/margins": 0.9307647943496704, |
|
"rewards/rejected": -3.1680736541748047, |
|
"step": 390 |
|
}, |
|
{ |
|
"epoch": 0.8442425861608336, |
|
"grad_norm": 7.486937950213667, |
|
"learning_rate": 3.538780159953347e-08, |
|
"logits/chosen": 2.171618700027466, |
|
"logits/rejected": 2.133547067642212, |
|
"logps/chosen": -0.913979172706604, |
|
"logps/rejected": -1.011819839477539, |
|
"loss": 1.6306, |
|
"rewards/accuracies": 0.512499988079071, |
|
"rewards/chosen": -2.284947633743286, |
|
"rewards/margins": 0.24460192024707794, |
|
"rewards/rejected": -2.5295493602752686, |
|
"step": 395 |
|
}, |
|
{ |
|
"epoch": 0.8549292011755276, |
|
"grad_norm": 12.731226626352624, |
|
"learning_rate": 3.074339975080836e-08, |
|
"logits/chosen": 2.207425594329834, |
|
"logits/rejected": 2.2549290657043457, |
|
"logps/chosen": -1.030874490737915, |
|
"logps/rejected": -1.2776422500610352, |
|
"loss": 1.6417, |
|
"rewards/accuracies": 0.5, |
|
"rewards/chosen": -2.577186107635498, |
|
"rewards/margins": 0.6169193983078003, |
|
"rewards/rejected": -3.194105863571167, |
|
"step": 400 |
|
}, |
|
{ |
|
"epoch": 0.8549292011755276, |
|
"eval_logits/chosen": 2.0145604610443115, |
|
"eval_logits/rejected": 2.0190441608428955, |
|
"eval_logps/chosen": -0.8955881595611572, |
|
"eval_logps/rejected": -1.058284044265747, |
|
"eval_loss": 1.6235809326171875, |
|
"eval_rewards/accuracies": 0.6209677457809448, |
|
"eval_rewards/chosen": -2.2389702796936035, |
|
"eval_rewards/margins": 0.40673962235450745, |
|
"eval_rewards/rejected": -2.645709753036499, |
|
"eval_runtime": 44.336, |
|
"eval_samples_per_second": 44.23, |
|
"eval_steps_per_second": 1.398, |
|
"step": 400 |
|
}, |
|
{ |
|
"epoch": 0.8656158161902218, |
|
"grad_norm": 10.867554141726604, |
|
"learning_rate": 2.6405648083415833e-08, |
|
"logits/chosen": 2.151744842529297, |
|
"logits/rejected": 2.200650691986084, |
|
"logps/chosen": -0.980237603187561, |
|
"logps/rejected": -1.0543644428253174, |
|
"loss": 1.7482, |
|
"rewards/accuracies": 0.5249999761581421, |
|
"rewards/chosen": -2.450594186782837, |
|
"rewards/margins": 0.18531720340251923, |
|
"rewards/rejected": -2.635911226272583, |
|
"step": 405 |
|
}, |
|
{ |
|
"epoch": 0.8763024312049158, |
|
"grad_norm": 8.929503879536968, |
|
"learning_rate": 2.2380613335296033e-08, |
|
"logits/chosen": 2.186063289642334, |
|
"logits/rejected": 2.2187790870666504, |
|
"logps/chosen": -0.9089058041572571, |
|
"logps/rejected": -1.0911632776260376, |
|
"loss": 1.5847, |
|
"rewards/accuracies": 0.612500011920929, |
|
"rewards/chosen": -2.272264242172241, |
|
"rewards/margins": 0.45564380288124084, |
|
"rewards/rejected": -2.727908134460449, |
|
"step": 410 |
|
}, |
|
{ |
|
"epoch": 0.88698904621961, |
|
"grad_norm": 7.217288770833436, |
|
"learning_rate": 1.8673924881500823e-08, |
|
"logits/chosen": 2.505709409713745, |
|
"logits/rejected": 2.470183849334717, |
|
"logps/chosen": -0.8387972116470337, |
|
"logps/rejected": -1.0221149921417236, |
|
"loss": 1.6029, |
|
"rewards/accuracies": 0.5, |
|
"rewards/chosen": -2.0969929695129395, |
|
"rewards/margins": 0.45829471945762634, |
|
"rewards/rejected": -2.5552875995635986, |
|
"step": 415 |
|
}, |
|
{ |
|
"epoch": 0.897675661234304, |
|
"grad_norm": 8.941360081202905, |
|
"learning_rate": 1.5290766861003475e-08, |
|
"logits/chosen": 2.191061019897461, |
|
"logits/rejected": 2.173102378845215, |
|
"logps/chosen": -0.9252120852470398, |
|
"logps/rejected": -1.03108811378479, |
|
"loss": 1.6027, |
|
"rewards/accuracies": 0.550000011920929, |
|
"rewards/chosen": -2.31303071975708, |
|
"rewards/margins": 0.2646896243095398, |
|
"rewards/rejected": -2.5777201652526855, |
|
"step": 420 |
|
}, |
|
{ |
|
"epoch": 0.9083622762489981, |
|
"grad_norm": 14.568857335011407, |
|
"learning_rate": 1.2235870926211616e-08, |
|
"logits/chosen": 2.0643928050994873, |
|
"logits/rejected": 2.0896859169006348, |
|
"logps/chosen": -0.9114057421684265, |
|
"logps/rejected": -1.0619531869888306, |
|
"loss": 1.6274, |
|
"rewards/accuracies": 0.5375000238418579, |
|
"rewards/chosen": -2.2785143852233887, |
|
"rewards/margins": 0.37636831402778625, |
|
"rewards/rejected": -2.6548826694488525, |
|
"step": 425 |
|
}, |
|
{ |
|
"epoch": 0.9190488912636923, |
|
"grad_norm": 12.761707033542287, |
|
"learning_rate": 9.513509625323518e-09, |
|
"logits/chosen": 2.1255576610565186, |
|
"logits/rejected": 2.181107759475708, |
|
"logps/chosen": -0.85987389087677, |
|
"logps/rejected": -1.0060930252075195, |
|
"loss": 1.5736, |
|
"rewards/accuracies": 0.7124999761581421, |
|
"rewards/chosen": -2.149684429168701, |
|
"rewards/margins": 0.3655479848384857, |
|
"rewards/rejected": -2.515232563018799, |
|
"step": 430 |
|
}, |
|
{ |
|
"epoch": 0.9297355062783863, |
|
"grad_norm": 16.68383920608418, |
|
"learning_rate": 7.127490426783123e-09, |
|
"logits/chosen": 2.2676453590393066, |
|
"logits/rejected": 2.224987506866455, |
|
"logps/chosen": -0.9531320333480835, |
|
"logps/rejected": -1.1150267124176025, |
|
"loss": 1.6026, |
|
"rewards/accuracies": 0.574999988079071, |
|
"rewards/chosen": -2.3828301429748535, |
|
"rewards/margins": 0.4047369956970215, |
|
"rewards/rejected": -2.787567138671875, |
|
"step": 435 |
|
}, |
|
{ |
|
"epoch": 0.9404221212930804, |
|
"grad_norm": 11.429700658399854, |
|
"learning_rate": 5.08115039419113e-09, |
|
"logits/chosen": 2.2096710205078125, |
|
"logits/rejected": 2.2330925464630127, |
|
"logps/chosen": -0.8609305620193481, |
|
"logps/rejected": -0.9673101305961609, |
|
"loss": 1.5978, |
|
"rewards/accuracies": 0.574999988079071, |
|
"rewards/chosen": -2.1523263454437256, |
|
"rewards/margins": 0.265949010848999, |
|
"rewards/rejected": -2.4182753562927246, |
|
"step": 440 |
|
}, |
|
{ |
|
"epoch": 0.9511087363077745, |
|
"grad_norm": 7.426544303240404, |
|
"learning_rate": 3.3773515191196646e-09, |
|
"logits/chosen": 2.20963716506958, |
|
"logits/rejected": 2.201622724533081, |
|
"logps/chosen": -0.8218202590942383, |
|
"logps/rejected": -0.8649258613586426, |
|
"loss": 1.6414, |
|
"rewards/accuracies": 0.512499988079071, |
|
"rewards/chosen": -2.0545506477355957, |
|
"rewards/margins": 0.10776408016681671, |
|
"rewards/rejected": -2.1623148918151855, |
|
"step": 445 |
|
}, |
|
{ |
|
"epoch": 0.9617953513224686, |
|
"grad_norm": 12.349821491663924, |
|
"learning_rate": 2.0184767183584474e-09, |
|
"logits/chosen": 2.325861930847168, |
|
"logits/rejected": 2.2664694786071777, |
|
"logps/chosen": -0.9952027201652527, |
|
"logps/rejected": -1.3025867938995361, |
|
"loss": 1.6015, |
|
"rewards/accuracies": 0.5874999761581421, |
|
"rewards/chosen": -2.488007068634033, |
|
"rewards/margins": 0.7684597969055176, |
|
"rewards/rejected": -3.2564666271209717, |
|
"step": 450 |
|
}, |
|
{ |
|
"epoch": 0.9724819663371627, |
|
"grad_norm": 10.41544260252462, |
|
"learning_rate": 1.0064265011902328e-09, |
|
"logits/chosen": 2.0219030380249023, |
|
"logits/rejected": 1.9868360757827759, |
|
"logps/chosen": -1.0134050846099854, |
|
"logps/rejected": -1.0333858728408813, |
|
"loss": 1.6377, |
|
"rewards/accuracies": 0.4000000059604645, |
|
"rewards/chosen": -2.533512592315674, |
|
"rewards/margins": 0.049951858818531036, |
|
"rewards/rejected": -2.5834643840789795, |
|
"step": 455 |
|
}, |
|
{ |
|
"epoch": 0.9831685813518568, |
|
"grad_norm": 9.044209064035641, |
|
"learning_rate": 3.4261631135654167e-10, |
|
"logits/chosen": 2.2376327514648438, |
|
"logits/rejected": 2.1572484970092773, |
|
"logps/chosen": -1.0239425897598267, |
|
"logps/rejected": -1.3165332078933716, |
|
"loss": 1.5875, |
|
"rewards/accuracies": 0.5375000238418579, |
|
"rewards/chosen": -2.559856414794922, |
|
"rewards/margins": 0.7314767241477966, |
|
"rewards/rejected": -3.291332960128784, |
|
"step": 460 |
|
}, |
|
{ |
|
"epoch": 0.9938551963665508, |
|
"grad_norm": 11.3294414057108, |
|
"learning_rate": 2.797454743164174e-11, |
|
"logits/chosen": 2.186420202255249, |
|
"logits/rejected": 2.1705265045166016, |
|
"logps/chosen": -0.875818133354187, |
|
"logps/rejected": -1.0201952457427979, |
|
"loss": 1.5967, |
|
"rewards/accuracies": 0.5249999761581421, |
|
"rewards/chosen": -2.1895456314086914, |
|
"rewards/margins": 0.36094242334365845, |
|
"rewards/rejected": -2.550488233566284, |
|
"step": 465 |
|
}, |
|
{ |
|
"epoch": 0.9981298423724285, |
|
"step": 467, |
|
"total_flos": 0.0, |
|
"train_loss": 1.6566840494470372, |
|
"train_runtime": 5417.7619, |
|
"train_samples_per_second": 11.052, |
|
"train_steps_per_second": 0.086 |
|
} |
|
], |
|
"logging_steps": 5, |
|
"max_steps": 467, |
|
"num_input_tokens_seen": 0, |
|
"num_train_epochs": 1, |
|
"save_steps": 1000000, |
|
"stateful_callbacks": { |
|
"TrainerControl": { |
|
"args": { |
|
"should_epoch_stop": false, |
|
"should_evaluate": false, |
|
"should_log": false, |
|
"should_save": false, |
|
"should_training_stop": false |
|
}, |
|
"attributes": {} |
|
} |
|
}, |
|
"total_flos": 0.0, |
|
"train_batch_size": 2, |
|
"trial_name": null, |
|
"trial_params": null |
|
} |
|
|