diff --git "a/trainer_state.json" "b/trainer_state.json" new file mode 100644--- /dev/null +++ "b/trainer_state.json" @@ -0,0 +1,6383 @@ +{ + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 1.0, + "eval_steps": 100, + "global_step": 3821, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.0, + "grad_norm": 3.8125, + "learning_rate": 1.3054830287206268e-08, + "logits/chosen": -2.377302885055542, + "logits/rejected": -2.2193148136138916, + "logps/chosen": -290.4185485839844, + "logps/rejected": -374.6668701171875, + "loss": 0.6931, + "rewards/accuracies": 0.0, + "rewards/chosen": 0.0, + "rewards/margins": 0.0, + "rewards/rejected": 0.0, + "step": 1 + }, + { + "epoch": 0.0, + "grad_norm": 2.40625, + "learning_rate": 1.3054830287206266e-07, + "logits/chosen": -2.2499454021453857, + "logits/rejected": -2.0522336959838867, + "logps/chosen": -279.5985107421875, + "logps/rejected": -245.43223571777344, + "loss": 0.6929, + "rewards/accuracies": 0.4861111044883728, + "rewards/chosen": 0.0009557433077134192, + "rewards/margins": 0.0005407779826782644, + "rewards/rejected": 0.0004149653250351548, + "step": 10 + }, + { + "epoch": 0.01, + "grad_norm": 2.5, + "learning_rate": 2.610966057441253e-07, + "logits/chosen": -2.2457704544067383, + "logits/rejected": -1.944566011428833, + "logps/chosen": -305.46026611328125, + "logps/rejected": -237.7046356201172, + "loss": 0.6927, + "rewards/accuracies": 0.5375000238418579, + "rewards/chosen": 0.004223807714879513, + "rewards/margins": 0.0008180936565622687, + "rewards/rejected": 0.0034057139419019222, + "step": 20 + }, + { + "epoch": 0.01, + "grad_norm": 2.3125, + "learning_rate": 3.9164490861618804e-07, + "logits/chosen": -2.2053937911987305, + "logits/rejected": -2.1369049549102783, + "logps/chosen": -251.22940063476562, + "logps/rejected": -251.3945770263672, + "loss": 0.6922, + "rewards/accuracies": 0.606249988079071, + "rewards/chosen": 0.011935219168663025, + "rewards/margins": 0.001991255208849907, + "rewards/rejected": 0.009943963959813118, + "step": 30 + }, + { + "epoch": 0.01, + "grad_norm": 1.9453125, + "learning_rate": 5.221932114882506e-07, + "logits/chosen": -2.0609099864959717, + "logits/rejected": -2.0232198238372803, + "logps/chosen": -216.2375030517578, + "logps/rejected": -221.68643188476562, + "loss": 0.6916, + "rewards/accuracies": 0.543749988079071, + "rewards/chosen": 0.019067076966166496, + "rewards/margins": 0.0031169778667390347, + "rewards/rejected": 0.015950100496411324, + "step": 40 + }, + { + "epoch": 0.01, + "grad_norm": 2.078125, + "learning_rate": 6.527415143603135e-07, + "logits/chosen": -2.1113991737365723, + "logits/rejected": -2.0994935035705566, + "logps/chosen": -266.8785095214844, + "logps/rejected": -234.3098907470703, + "loss": 0.6906, + "rewards/accuracies": 0.5874999761581421, + "rewards/chosen": 0.030090373009443283, + "rewards/margins": 0.005098854657262564, + "rewards/rejected": 0.024991516023874283, + "step": 50 + }, + { + "epoch": 0.02, + "grad_norm": 2.125, + "learning_rate": 7.832898172323761e-07, + "logits/chosen": -2.099452257156372, + "logits/rejected": -1.9423996210098267, + "logps/chosen": -252.3367919921875, + "logps/rejected": -226.71066284179688, + "loss": 0.69, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": 0.031631551682949066, + "rewards/margins": 0.006400656886398792, + "rewards/rejected": 0.025230899453163147, + "step": 60 + }, + { + "epoch": 0.02, + "grad_norm": 2.03125, + "learning_rate": 9.138381201044387e-07, + "logits/chosen": -2.243349552154541, + "logits/rejected": -2.035635471343994, + "logps/chosen": -272.0526428222656, + "logps/rejected": -246.71243286132812, + "loss": 0.6878, + "rewards/accuracies": 0.643750011920929, + "rewards/chosen": 0.04103558883070946, + "rewards/margins": 0.01095958799123764, + "rewards/rejected": 0.03007599711418152, + "step": 70 + }, + { + "epoch": 0.02, + "grad_norm": 2.40625, + "learning_rate": 1.0443864229765013e-06, + "logits/chosen": -2.154019355773926, + "logits/rejected": -1.9776532649993896, + "logps/chosen": -257.60150146484375, + "logps/rejected": -246.8785858154297, + "loss": 0.6873, + "rewards/accuracies": 0.6625000238418579, + "rewards/chosen": 0.03827068209648132, + "rewards/margins": 0.01215548999607563, + "rewards/rejected": 0.026115190237760544, + "step": 80 + }, + { + "epoch": 0.02, + "grad_norm": 2.203125, + "learning_rate": 1.1749347258485642e-06, + "logits/chosen": -2.135911703109741, + "logits/rejected": -2.000063419342041, + "logps/chosen": -250.10903930664062, + "logps/rejected": -234.52127075195312, + "loss": 0.6846, + "rewards/accuracies": 0.6812499761581421, + "rewards/chosen": 0.04227185994386673, + "rewards/margins": 0.017635947093367577, + "rewards/rejected": 0.024635912850499153, + "step": 90 + }, + { + "epoch": 0.03, + "grad_norm": 2.140625, + "learning_rate": 1.305483028720627e-06, + "logits/chosen": -2.180065631866455, + "logits/rejected": -2.069608449935913, + "logps/chosen": -247.0371551513672, + "logps/rejected": -230.7642822265625, + "loss": 0.6823, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": 0.04732387885451317, + "rewards/margins": 0.022458035498857498, + "rewards/rejected": 0.02486584149301052, + "step": 100 + }, + { + "epoch": 0.03, + "eval_logits/chosen": -2.095273017883301, + "eval_logits/rejected": -1.955887794494629, + "eval_logps/chosen": -259.675048828125, + "eval_logps/rejected": -241.93234252929688, + "eval_loss": 0.6821526885032654, + "eval_rewards/accuracies": 0.6610000133514404, + "eval_rewards/chosen": 0.04976964741945267, + "eval_rewards/margins": 0.02300717867910862, + "eval_rewards/rejected": 0.026762468740344048, + "eval_runtime": 384.6998, + "eval_samples_per_second": 5.199, + "eval_steps_per_second": 0.65, + "step": 100 + }, + { + "epoch": 0.03, + "grad_norm": 2.3125, + "learning_rate": 1.4360313315926894e-06, + "logits/chosen": -2.14589786529541, + "logits/rejected": -2.0023417472839355, + "logps/chosen": -284.3846130371094, + "logps/rejected": -238.9386444091797, + "loss": 0.6789, + "rewards/accuracies": 0.637499988079071, + "rewards/chosen": 0.04979206249117851, + "rewards/margins": 0.02978363260626793, + "rewards/rejected": 0.020008429884910583, + "step": 110 + }, + { + "epoch": 0.03, + "grad_norm": 2.15625, + "learning_rate": 1.5665796344647521e-06, + "logits/chosen": -2.192373275756836, + "logits/rejected": -2.052762985229492, + "logps/chosen": -287.498291015625, + "logps/rejected": -271.96441650390625, + "loss": 0.6727, + "rewards/accuracies": 0.6875, + "rewards/chosen": 0.05546677112579346, + "rewards/margins": 0.04281745105981827, + "rewards/rejected": 0.012649321928620338, + "step": 120 + }, + { + "epoch": 0.03, + "grad_norm": 2.640625, + "learning_rate": 1.6971279373368146e-06, + "logits/chosen": -2.2076873779296875, + "logits/rejected": -2.1181204319000244, + "logps/chosen": -250.1416015625, + "logps/rejected": -252.60836791992188, + "loss": 0.6701, + "rewards/accuracies": 0.7124999761581421, + "rewards/chosen": 0.05010415241122246, + "rewards/margins": 0.04891490936279297, + "rewards/rejected": 0.0011892480542883277, + "step": 130 + }, + { + "epoch": 0.04, + "grad_norm": 2.5, + "learning_rate": 1.8276762402088774e-06, + "logits/chosen": -2.2444560527801514, + "logits/rejected": -1.9101192951202393, + "logps/chosen": -270.51544189453125, + "logps/rejected": -226.2876739501953, + "loss": 0.6685, + "rewards/accuracies": 0.65625, + "rewards/chosen": 0.04222479462623596, + "rewards/margins": 0.05299673601984978, + "rewards/rejected": -0.010771943256258965, + "step": 140 + }, + { + "epoch": 0.04, + "grad_norm": 2.640625, + "learning_rate": 1.9582245430809403e-06, + "logits/chosen": -2.264216899871826, + "logits/rejected": -2.0381903648376465, + "logps/chosen": -280.3586120605469, + "logps/rejected": -242.8353271484375, + "loss": 0.6677, + "rewards/accuracies": 0.65625, + "rewards/chosen": 0.03652295097708702, + "rewards/margins": 0.05572789907455444, + "rewards/rejected": -0.019204948097467422, + "step": 150 + }, + { + "epoch": 0.04, + "grad_norm": 2.71875, + "learning_rate": 2.0887728459530026e-06, + "logits/chosen": -2.1548912525177, + "logits/rejected": -2.052673816680908, + "logps/chosen": -256.0513610839844, + "logps/rejected": -261.861328125, + "loss": 0.6688, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": 0.007725180592387915, + "rewards/margins": 0.055973686277866364, + "rewards/rejected": -0.04824850708246231, + "step": 160 + }, + { + "epoch": 0.04, + "grad_norm": 2.890625, + "learning_rate": 2.2193211488250653e-06, + "logits/chosen": -2.124185562133789, + "logits/rejected": -1.9691429138183594, + "logps/chosen": -220.9111328125, + "logps/rejected": -228.2654571533203, + "loss": 0.6707, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.0032043636310845613, + "rewards/margins": 0.051164913922548294, + "rewards/rejected": -0.054369281977415085, + "step": 170 + }, + { + "epoch": 0.05, + "grad_norm": 2.984375, + "learning_rate": 2.3498694516971284e-06, + "logits/chosen": -2.122745990753174, + "logits/rejected": -1.9872467517852783, + "logps/chosen": -259.26727294921875, + "logps/rejected": -252.7638702392578, + "loss": 0.6637, + "rewards/accuracies": 0.6312500238418579, + "rewards/chosen": -0.03562777489423752, + "rewards/margins": 0.06830445677042007, + "rewards/rejected": -0.1039322167634964, + "step": 180 + }, + { + "epoch": 0.05, + "grad_norm": 3.8125, + "learning_rate": 2.4804177545691907e-06, + "logits/chosen": -2.2456932067871094, + "logits/rejected": -2.029968738555908, + "logps/chosen": -275.09014892578125, + "logps/rejected": -256.80023193359375, + "loss": 0.65, + "rewards/accuracies": 0.65625, + "rewards/chosen": -0.050059832632541656, + "rewards/margins": 0.10052521526813507, + "rewards/rejected": -0.15058502554893494, + "step": 190 + }, + { + "epoch": 0.05, + "grad_norm": 3.40625, + "learning_rate": 2.610966057441254e-06, + "logits/chosen": -2.194169759750366, + "logits/rejected": -1.954360008239746, + "logps/chosen": -258.94854736328125, + "logps/rejected": -231.2574920654297, + "loss": 0.6492, + "rewards/accuracies": 0.668749988079071, + "rewards/chosen": -0.09239193052053452, + "rewards/margins": 0.10424431413412094, + "rewards/rejected": -0.19663624465465546, + "step": 200 + }, + { + "epoch": 0.05, + "eval_logits/chosen": -2.0706493854522705, + "eval_logits/rejected": -1.9337996244430542, + "eval_logps/chosen": -269.44000244140625, + "eval_logps/rejected": -259.9606018066406, + "eval_loss": 0.6490568518638611, + "eval_rewards/accuracies": 0.6815000176429749, + "eval_rewards/chosen": -0.04788003861904144, + "eval_rewards/margins": 0.10563990473747253, + "eval_rewards/rejected": -0.15351995825767517, + "eval_runtime": 408.5567, + "eval_samples_per_second": 4.895, + "eval_steps_per_second": 0.612, + "step": 200 + }, + { + "epoch": 0.05, + "grad_norm": 3.359375, + "learning_rate": 2.741514360313316e-06, + "logits/chosen": -2.196761131286621, + "logits/rejected": -1.9796245098114014, + "logps/chosen": -267.26092529296875, + "logps/rejected": -249.635986328125, + "loss": 0.6334, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.05040832608938217, + "rewards/margins": 0.14044944941997528, + "rewards/rejected": -0.19085776805877686, + "step": 210 + }, + { + "epoch": 0.06, + "grad_norm": 4.125, + "learning_rate": 2.872062663185379e-06, + "logits/chosen": -2.0959529876708984, + "logits/rejected": -1.9805145263671875, + "logps/chosen": -265.8911437988281, + "logps/rejected": -252.39990234375, + "loss": 0.6401, + "rewards/accuracies": 0.65625, + "rewards/chosen": -0.11875394731760025, + "rewards/margins": 0.1290549784898758, + "rewards/rejected": -0.24780890345573425, + "step": 220 + }, + { + "epoch": 0.06, + "grad_norm": 4.21875, + "learning_rate": 3.0026109660574416e-06, + "logits/chosen": -2.239375591278076, + "logits/rejected": -2.0525119304656982, + "logps/chosen": -317.2463684082031, + "logps/rejected": -291.793212890625, + "loss": 0.657, + "rewards/accuracies": 0.6187499761581421, + "rewards/chosen": -0.11077094078063965, + "rewards/margins": 0.10038264095783234, + "rewards/rejected": -0.21115358173847198, + "step": 230 + }, + { + "epoch": 0.06, + "grad_norm": 2.78125, + "learning_rate": 3.1331592689295043e-06, + "logits/chosen": -2.149392604827881, + "logits/rejected": -1.961692452430725, + "logps/chosen": -313.08331298828125, + "logps/rejected": -309.60198974609375, + "loss": 0.648, + "rewards/accuracies": 0.612500011920929, + "rewards/chosen": -0.16783350706100464, + "rewards/margins": 0.11663570255041122, + "rewards/rejected": -0.28446921706199646, + "step": 240 + }, + { + "epoch": 0.07, + "grad_norm": 2.9375, + "learning_rate": 3.263707571801567e-06, + "logits/chosen": -2.109787702560425, + "logits/rejected": -2.0100176334381104, + "logps/chosen": -297.9364929199219, + "logps/rejected": -283.5945129394531, + "loss": 0.6475, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.38356590270996094, + "rewards/margins": 0.11117170751094818, + "rewards/rejected": -0.49473756551742554, + "step": 250 + }, + { + "epoch": 0.07, + "grad_norm": 3.125, + "learning_rate": 3.3942558746736293e-06, + "logits/chosen": -2.1550464630126953, + "logits/rejected": -1.9509170055389404, + "logps/chosen": -313.568115234375, + "logps/rejected": -301.53759765625, + "loss": 0.654, + "rewards/accuracies": 0.6937500238418579, + "rewards/chosen": -0.4116322994232178, + "rewards/margins": 0.09985023736953735, + "rewards/rejected": -0.5114825367927551, + "step": 260 + }, + { + "epoch": 0.07, + "grad_norm": 3.328125, + "learning_rate": 3.524804177545692e-06, + "logits/chosen": -2.0624794960021973, + "logits/rejected": -1.9370944499969482, + "logps/chosen": -292.0105895996094, + "logps/rejected": -279.63177490234375, + "loss": 0.6173, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.3133612275123596, + "rewards/margins": 0.190281480550766, + "rewards/rejected": -0.5036426782608032, + "step": 270 + }, + { + "epoch": 0.07, + "grad_norm": 4.6875, + "learning_rate": 3.6553524804177547e-06, + "logits/chosen": -2.13100528717041, + "logits/rejected": -1.9591659307479858, + "logps/chosen": -289.3089904785156, + "logps/rejected": -282.8084411621094, + "loss": 0.641, + "rewards/accuracies": 0.6187499761581421, + "rewards/chosen": -0.3035983741283417, + "rewards/margins": 0.1496592015028, + "rewards/rejected": -0.45325756072998047, + "step": 280 + }, + { + "epoch": 0.08, + "grad_norm": 4.875, + "learning_rate": 3.7859007832898174e-06, + "logits/chosen": -2.057344675064087, + "logits/rejected": -1.9588420391082764, + "logps/chosen": -311.67510986328125, + "logps/rejected": -307.47015380859375, + "loss": 0.6204, + "rewards/accuracies": 0.668749988079071, + "rewards/chosen": -0.2604138255119324, + "rewards/margins": 0.209051251411438, + "rewards/rejected": -0.46946510672569275, + "step": 290 + }, + { + "epoch": 0.08, + "grad_norm": 5.15625, + "learning_rate": 3.9164490861618806e-06, + "logits/chosen": -2.0852608680725098, + "logits/rejected": -1.8826076984405518, + "logps/chosen": -279.9902038574219, + "logps/rejected": -289.1649169921875, + "loss": 0.6101, + "rewards/accuracies": 0.668749988079071, + "rewards/chosen": -0.35862046480178833, + "rewards/margins": 0.2289685755968094, + "rewards/rejected": -0.5875889658927917, + "step": 300 + }, + { + "epoch": 0.08, + "eval_logits/chosen": -2.0021328926086426, + "eval_logits/rejected": -1.8679821491241455, + "eval_logps/chosen": -298.72515869140625, + "eval_logps/rejected": -299.372802734375, + "eval_loss": 0.6216704249382019, + "eval_rewards/accuracies": 0.6769999861717224, + "eval_rewards/chosen": -0.3407318592071533, + "eval_rewards/margins": 0.20691031217575073, + "eval_rewards/rejected": -0.547642171382904, + "eval_runtime": 385.1163, + "eval_samples_per_second": 5.193, + "eval_steps_per_second": 0.649, + "step": 300 + }, + { + "epoch": 0.08, + "grad_norm": 4.15625, + "learning_rate": 4.046997389033943e-06, + "logits/chosen": -2.2290728092193604, + "logits/rejected": -2.029090642929077, + "logps/chosen": -319.5593566894531, + "logps/rejected": -296.0638732910156, + "loss": 0.5797, + "rewards/accuracies": 0.7437499761581421, + "rewards/chosen": -0.30653077363967896, + "rewards/margins": 0.3014482855796814, + "rewards/rejected": -0.6079790592193604, + "step": 310 + }, + { + "epoch": 0.08, + "grad_norm": 4.78125, + "learning_rate": 4.177545691906005e-06, + "logits/chosen": -2.1043484210968018, + "logits/rejected": -1.9178568124771118, + "logps/chosen": -299.31610107421875, + "logps/rejected": -301.1820068359375, + "loss": 0.6348, + "rewards/accuracies": 0.637499988079071, + "rewards/chosen": -0.41052132844924927, + "rewards/margins": 0.18894672393798828, + "rewards/rejected": -0.5994681119918823, + "step": 320 + }, + { + "epoch": 0.09, + "grad_norm": 4.625, + "learning_rate": 4.308093994778068e-06, + "logits/chosen": -2.0332717895507812, + "logits/rejected": -1.891761064529419, + "logps/chosen": -292.5372619628906, + "logps/rejected": -289.52130126953125, + "loss": 0.6187, + "rewards/accuracies": 0.668749988079071, + "rewards/chosen": -0.3294296860694885, + "rewards/margins": 0.21162664890289307, + "rewards/rejected": -0.5410563945770264, + "step": 330 + }, + { + "epoch": 0.09, + "grad_norm": 3.875, + "learning_rate": 4.4386422976501306e-06, + "logits/chosen": -2.096933364868164, + "logits/rejected": -1.9842134714126587, + "logps/chosen": -318.83001708984375, + "logps/rejected": -326.63128662109375, + "loss": 0.5927, + "rewards/accuracies": 0.737500011920929, + "rewards/chosen": -0.29144176840782166, + "rewards/margins": 0.2920153737068176, + "rewards/rejected": -0.5834571719169617, + "step": 340 + }, + { + "epoch": 0.09, + "grad_norm": 5.625, + "learning_rate": 4.569190600522193e-06, + "logits/chosen": -2.0155937671661377, + "logits/rejected": -1.8702361583709717, + "logps/chosen": -324.283447265625, + "logps/rejected": -333.35772705078125, + "loss": 0.6323, + "rewards/accuracies": 0.65625, + "rewards/chosen": -0.3502510190010071, + "rewards/margins": 0.20564763247966766, + "rewards/rejected": -0.5558986067771912, + "step": 350 + }, + { + "epoch": 0.09, + "grad_norm": 5.84375, + "learning_rate": 4.699738903394257e-06, + "logits/chosen": -1.9730371236801147, + "logits/rejected": -1.927020788192749, + "logps/chosen": -289.91351318359375, + "logps/rejected": -299.1167907714844, + "loss": 0.5914, + "rewards/accuracies": 0.737500011920929, + "rewards/chosen": -0.4892953038215637, + "rewards/margins": 0.3004259467124939, + "rewards/rejected": -0.7897213697433472, + "step": 360 + }, + { + "epoch": 0.1, + "grad_norm": 6.0625, + "learning_rate": 4.8302872062663196e-06, + "logits/chosen": -2.0422720909118652, + "logits/rejected": -1.8679052591323853, + "logps/chosen": -364.3427429199219, + "logps/rejected": -349.3939514160156, + "loss": 0.608, + "rewards/accuracies": 0.65625, + "rewards/chosen": -0.8744655847549438, + "rewards/margins": 0.2888025939464569, + "rewards/rejected": -1.1632683277130127, + "step": 370 + }, + { + "epoch": 0.1, + "grad_norm": 4.375, + "learning_rate": 4.9608355091383814e-06, + "logits/chosen": -1.9862794876098633, + "logits/rejected": -1.7763971090316772, + "logps/chosen": -339.8470153808594, + "logps/rejected": -339.43743896484375, + "loss": 0.5892, + "rewards/accuracies": 0.706250011920929, + "rewards/chosen": -0.41790348291397095, + "rewards/margins": 0.3297092318534851, + "rewards/rejected": -0.7476127743721008, + "step": 380 + }, + { + "epoch": 0.1, + "grad_norm": 5.78125, + "learning_rate": 4.9999488562447675e-06, + "logits/chosen": -1.9420530796051025, + "logits/rejected": -1.8238258361816406, + "logps/chosen": -318.894775390625, + "logps/rejected": -334.8386535644531, + "loss": 0.5721, + "rewards/accuracies": 0.7124999761581421, + "rewards/chosen": -0.3784860074520111, + "rewards/margins": 0.35959383845329285, + "rewards/rejected": -0.738079845905304, + "step": 390 + }, + { + "epoch": 0.1, + "grad_norm": 6.90625, + "learning_rate": 4.999698361256577e-06, + "logits/chosen": -1.9580987691879272, + "logits/rejected": -1.7359654903411865, + "logps/chosen": -302.38946533203125, + "logps/rejected": -285.8229675292969, + "loss": 0.6173, + "rewards/accuracies": 0.6937500238418579, + "rewards/chosen": -0.36912912130355835, + "rewards/margins": 0.24084654450416565, + "rewards/rejected": -0.6099756956100464, + "step": 400 + }, + { + "epoch": 0.1, + "eval_logits/chosen": -1.7878212928771973, + "eval_logits/rejected": -1.658238410949707, + "eval_logps/chosen": -314.925048828125, + "eval_logps/rejected": -327.9222412109375, + "eval_loss": 0.5952155590057373, + "eval_rewards/accuracies": 0.6834999918937683, + "eval_rewards/chosen": -0.5027302503585815, + "eval_rewards/margins": 0.33040642738342285, + "eval_rewards/rejected": -0.8331366777420044, + "eval_runtime": 384.9418, + "eval_samples_per_second": 5.196, + "eval_steps_per_second": 0.649, + "step": 400 + }, + { + "epoch": 0.11, + "grad_norm": 5.28125, + "learning_rate": 4.999239142174581e-06, + "logits/chosen": -1.791033148765564, + "logits/rejected": -1.727502465248108, + "logps/chosen": -305.58447265625, + "logps/rejected": -327.44293212890625, + "loss": 0.631, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.6326209306716919, + "rewards/margins": 0.2363467961549759, + "rewards/rejected": -0.8689676523208618, + "step": 410 + }, + { + "epoch": 0.11, + "grad_norm": 6.09375, + "learning_rate": 4.99857123734344e-06, + "logits/chosen": -1.758462905883789, + "logits/rejected": -1.625067114830017, + "logps/chosen": -289.7398376464844, + "logps/rejected": -331.71600341796875, + "loss": 0.5301, + "rewards/accuracies": 0.706250011920929, + "rewards/chosen": -0.6094896197319031, + "rewards/margins": 0.51044100522995, + "rewards/rejected": -1.119930624961853, + "step": 420 + }, + { + "epoch": 0.11, + "grad_norm": 8.1875, + "learning_rate": 4.997694702533016e-06, + "logits/chosen": -1.7083183526992798, + "logits/rejected": -1.6363675594329834, + "logps/chosen": -356.60418701171875, + "logps/rejected": -385.64410400390625, + "loss": 0.5625, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.790561318397522, + "rewards/margins": 0.5265924334526062, + "rewards/rejected": -1.317153811454773, + "step": 430 + }, + { + "epoch": 0.12, + "grad_norm": 7.09375, + "learning_rate": 4.996609610933713e-06, + "logits/chosen": -1.8173853158950806, + "logits/rejected": -1.73589289188385, + "logps/chosen": -327.7349548339844, + "logps/rejected": -338.24334716796875, + "loss": 0.5833, + "rewards/accuracies": 0.706250011920929, + "rewards/chosen": -0.5558608770370483, + "rewards/margins": 0.3886395990848541, + "rewards/rejected": -0.9445004463195801, + "step": 440 + }, + { + "epoch": 0.12, + "grad_norm": 7.4375, + "learning_rate": 4.995316053150366e-06, + "logits/chosen": -1.674541711807251, + "logits/rejected": -1.5631834268569946, + "logps/chosen": -329.3605651855469, + "logps/rejected": -351.2587890625, + "loss": 0.556, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.6152701377868652, + "rewards/margins": 0.4492935538291931, + "rewards/rejected": -1.0645637512207031, + "step": 450 + }, + { + "epoch": 0.12, + "grad_norm": 12.5, + "learning_rate": 4.9938141371946815e-06, + "logits/chosen": -1.616389513015747, + "logits/rejected": -1.5250511169433594, + "logps/chosen": -378.802978515625, + "logps/rejected": -416.6460876464844, + "loss": 0.5634, + "rewards/accuracies": 0.731249988079071, + "rewards/chosen": -1.1370322704315186, + "rewards/margins": 0.523708701133728, + "rewards/rejected": -1.6607410907745361, + "step": 460 + }, + { + "epoch": 0.12, + "grad_norm": 6.625, + "learning_rate": 4.992103988476206e-06, + "logits/chosen": -1.6163583993911743, + "logits/rejected": -1.4710776805877686, + "logps/chosen": -357.6758728027344, + "logps/rejected": -395.6207580566406, + "loss": 0.5805, + "rewards/accuracies": 0.7124999761581421, + "rewards/chosen": -1.1385869979858398, + "rewards/margins": 0.504925012588501, + "rewards/rejected": -1.6435120105743408, + "step": 470 + }, + { + "epoch": 0.13, + "grad_norm": 5.71875, + "learning_rate": 4.990185749791866e-06, + "logits/chosen": -1.6142799854278564, + "logits/rejected": -1.484505534172058, + "logps/chosen": -354.9118957519531, + "logps/rejected": -412.96539306640625, + "loss": 0.542, + "rewards/accuracies": 0.6937500238418579, + "rewards/chosen": -0.980329155921936, + "rewards/margins": 0.5626578330993652, + "rewards/rejected": -1.5429868698120117, + "step": 480 + }, + { + "epoch": 0.13, + "grad_norm": 8.25, + "learning_rate": 4.9880595813140395e-06, + "logits/chosen": -1.6605031490325928, + "logits/rejected": -1.519315242767334, + "logps/chosen": -388.9188537597656, + "logps/rejected": -418.9161071777344, + "loss": 0.5346, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -1.0402735471725464, + "rewards/margins": 0.603448212146759, + "rewards/rejected": -1.6437218189239502, + "step": 490 + }, + { + "epoch": 0.13, + "grad_norm": 6.21875, + "learning_rate": 4.985725660577184e-06, + "logits/chosen": -1.6577751636505127, + "logits/rejected": -1.5049307346343994, + "logps/chosen": -385.02813720703125, + "logps/rejected": -409.57379150390625, + "loss": 0.5435, + "rewards/accuracies": 0.706250011920929, + "rewards/chosen": -1.11268150806427, + "rewards/margins": 0.6577950119972229, + "rewards/rejected": -1.7704765796661377, + "step": 500 + }, + { + "epoch": 0.13, + "eval_logits/chosen": -1.5543876886367798, + "eval_logits/rejected": -1.427258014678955, + "eval_logps/chosen": -376.1609191894531, + "eval_logps/rejected": -405.3195495605469, + "eval_loss": 0.5753844976425171, + "eval_rewards/accuracies": 0.6890000104904175, + "eval_rewards/chosen": -1.115088939666748, + "eval_rewards/margins": 0.4920206665992737, + "eval_rewards/rejected": -1.607109785079956, + "eval_runtime": 388.587, + "eval_samples_per_second": 5.147, + "eval_steps_per_second": 0.643, + "step": 500 + }, + { + "epoch": 0.13, + "grad_norm": 6.5625, + "learning_rate": 4.983184182463009e-06, + "logits/chosen": -1.6563122272491455, + "logits/rejected": -1.5474936962127686, + "logps/chosen": -371.3856201171875, + "logps/rejected": -394.072021484375, + "loss": 0.5478, + "rewards/accuracies": 0.737500011920929, + "rewards/chosen": -0.9312448501586914, + "rewards/margins": 0.5972838401794434, + "rewards/rejected": -1.5285285711288452, + "step": 510 + }, + { + "epoch": 0.14, + "grad_norm": 10.1875, + "learning_rate": 4.980435359184203e-06, + "logits/chosen": -1.6844135522842407, + "logits/rejected": -1.6247421503067017, + "logps/chosen": -349.91943359375, + "logps/rejected": -377.5780029296875, + "loss": 0.5953, + "rewards/accuracies": 0.706250011920929, + "rewards/chosen": -0.768576443195343, + "rewards/margins": 0.43869537115097046, + "rewards/rejected": -1.2072718143463135, + "step": 520 + }, + { + "epoch": 0.14, + "grad_norm": 10.75, + "learning_rate": 4.9774794202667236e-06, + "logits/chosen": -1.68100106716156, + "logits/rejected": -1.63496994972229, + "logps/chosen": -316.89166259765625, + "logps/rejected": -371.69012451171875, + "loss": 0.577, + "rewards/accuracies": 0.71875, + "rewards/chosen": -0.5579014420509338, + "rewards/margins": 0.4634733200073242, + "rewards/rejected": -1.0213747024536133, + "step": 530 + }, + { + "epoch": 0.14, + "grad_norm": 10.9375, + "learning_rate": 4.974316612530615e-06, + "logits/chosen": -1.5825644731521606, + "logits/rejected": -1.4188969135284424, + "logps/chosen": -361.13397216796875, + "logps/rejected": -398.6629638671875, + "loss": 0.4772, + "rewards/accuracies": 0.8187500238418579, + "rewards/chosen": -0.8265112042427063, + "rewards/margins": 0.8431331515312195, + "rewards/rejected": -1.6696443557739258, + "step": 540 + }, + { + "epoch": 0.14, + "grad_norm": 15.25, + "learning_rate": 4.970947200069416e-06, + "logits/chosen": -1.460860252380371, + "logits/rejected": -1.3948581218719482, + "logps/chosen": -469.55615234375, + "logps/rejected": -493.6904296875, + "loss": 0.6643, + "rewards/accuracies": 0.668749988079071, + "rewards/chosen": -1.9104303121566772, + "rewards/margins": 0.43918126821517944, + "rewards/rejected": -2.349611759185791, + "step": 550 + }, + { + "epoch": 0.15, + "grad_norm": 7.6875, + "learning_rate": 4.967371464228096e-06, + "logits/chosen": -1.6625244617462158, + "logits/rejected": -1.5688579082489014, + "logps/chosen": -452.31256103515625, + "logps/rejected": -504.4009704589844, + "loss": 0.57, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -1.948133111000061, + "rewards/margins": 0.5168659687042236, + "rewards/rejected": -2.464999198913574, + "step": 560 + }, + { + "epoch": 0.15, + "grad_norm": 9.5, + "learning_rate": 4.963589703579569e-06, + "logits/chosen": -1.7668651342391968, + "logits/rejected": -1.6291354894638062, + "logps/chosen": -480.91107177734375, + "logps/rejected": -497.85650634765625, + "loss": 0.6004, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -1.8087114095687866, + "rewards/margins": 0.5162175297737122, + "rewards/rejected": -2.3249289989471436, + "step": 570 + }, + { + "epoch": 0.15, + "grad_norm": 8.4375, + "learning_rate": 4.9596022338997615e-06, + "logits/chosen": -1.7466462850570679, + "logits/rejected": -1.5261166095733643, + "logps/chosen": -454.7201232910156, + "logps/rejected": -477.67987060546875, + "loss": 0.5409, + "rewards/accuracies": 0.7124999761581421, + "rewards/chosen": -1.569333791732788, + "rewards/margins": 0.6403428316116333, + "rewards/rejected": -2.209676504135132, + "step": 580 + }, + { + "epoch": 0.15, + "grad_norm": 5.8125, + "learning_rate": 4.955409388141243e-06, + "logits/chosen": -1.620123267173767, + "logits/rejected": -1.504206895828247, + "logps/chosen": -378.0823669433594, + "logps/rejected": -403.84649658203125, + "loss": 0.5869, + "rewards/accuracies": 0.6875, + "rewards/chosen": -1.2078220844268799, + "rewards/margins": 0.49430006742477417, + "rewards/rejected": -1.7021223306655884, + "step": 590 + }, + { + "epoch": 0.16, + "grad_norm": 4.5, + "learning_rate": 4.951011516405429e-06, + "logits/chosen": -1.7101682424545288, + "logits/rejected": -1.6404016017913818, + "logps/chosen": -328.59051513671875, + "logps/rejected": -365.19866943359375, + "loss": 0.5547, + "rewards/accuracies": 0.71875, + "rewards/chosen": -0.7783375978469849, + "rewards/margins": 0.5154664516448975, + "rewards/rejected": -1.2938039302825928, + "step": 600 + }, + { + "epoch": 0.16, + "eval_logits/chosen": -1.5725802183151245, + "eval_logits/rejected": -1.439643383026123, + "eval_logps/chosen": -340.6527404785156, + "eval_logps/rejected": -371.2197570800781, + "eval_loss": 0.5694642066955566, + "eval_rewards/accuracies": 0.6984999775886536, + "eval_rewards/chosen": -0.7600072622299194, + "eval_rewards/margins": 0.5061042904853821, + "eval_rewards/rejected": -1.2661116123199463, + "eval_runtime": 384.8138, + "eval_samples_per_second": 5.197, + "eval_steps_per_second": 0.65, + "step": 600 + }, + { + "epoch": 0.16, + "grad_norm": 8.375, + "learning_rate": 4.946408985913344e-06, + "logits/chosen": -1.6002981662750244, + "logits/rejected": -1.5132791996002197, + "logps/chosen": -320.1644592285156, + "logps/rejected": -367.06707763671875, + "loss": 0.5344, + "rewards/accuracies": 0.706250011920929, + "rewards/chosen": -0.7430071830749512, + "rewards/margins": 0.6466056108474731, + "rewards/rejected": -1.3896129131317139, + "step": 610 + }, + { + "epoch": 0.16, + "grad_norm": 11.5, + "learning_rate": 4.941602180974958e-06, + "logits/chosen": -1.5505564212799072, + "logits/rejected": -1.3113032579421997, + "logps/chosen": -394.12786865234375, + "logps/rejected": -412.055908203125, + "loss": 0.5398, + "rewards/accuracies": 0.7562500238418579, + "rewards/chosen": -1.0630056858062744, + "rewards/margins": 0.7659745216369629, + "rewards/rejected": -1.8289800882339478, + "step": 620 + }, + { + "epoch": 0.16, + "grad_norm": 4.96875, + "learning_rate": 4.936591502957101e-06, + "logits/chosen": -1.4749863147735596, + "logits/rejected": -1.315354347229004, + "logps/chosen": -395.21185302734375, + "logps/rejected": -471.55157470703125, + "loss": 0.5375, + "rewards/accuracies": 0.731249988079071, + "rewards/chosen": -1.470820665359497, + "rewards/margins": 0.8663395643234253, + "rewards/rejected": -2.337160587310791, + "step": 630 + }, + { + "epoch": 0.17, + "grad_norm": 8.6875, + "learning_rate": 4.931377370249946e-06, + "logits/chosen": -1.4970736503601074, + "logits/rejected": -1.2784922122955322, + "logps/chosen": -469.60565185546875, + "logps/rejected": -513.3096923828125, + "loss": 0.5511, + "rewards/accuracies": 0.7124999761581421, + "rewards/chosen": -2.0428223609924316, + "rewards/margins": 0.6945194602012634, + "rewards/rejected": -2.73734188079834, + "step": 640 + }, + { + "epoch": 0.17, + "grad_norm": 12.1875, + "learning_rate": 4.925960218232073e-06, + "logits/chosen": -1.5300517082214355, + "logits/rejected": -1.4055113792419434, + "logps/chosen": -411.9557189941406, + "logps/rejected": -478.2559509277344, + "loss": 0.5422, + "rewards/accuracies": 0.731249988079071, + "rewards/chosen": -1.5665361881256104, + "rewards/margins": 0.7580282092094421, + "rewards/rejected": -2.324564218521118, + "step": 650 + }, + { + "epoch": 0.17, + "grad_norm": 8.9375, + "learning_rate": 4.920340499234116e-06, + "logits/chosen": -1.5464222431182861, + "logits/rejected": -1.322009563446045, + "logps/chosen": -396.75299072265625, + "logps/rejected": -416.83868408203125, + "loss": 0.5751, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -1.307492733001709, + "rewards/margins": 0.5529674291610718, + "rewards/rejected": -1.8604600429534912, + "step": 660 + }, + { + "epoch": 0.18, + "grad_norm": 6.875, + "learning_rate": 4.914518682500995e-06, + "logits/chosen": -1.7627025842666626, + "logits/rejected": -1.554469347000122, + "logps/chosen": -392.9539794921875, + "logps/rejected": -419.94952392578125, + "loss": 0.529, + "rewards/accuracies": 0.6812499761581421, + "rewards/chosen": -1.0941137075424194, + "rewards/margins": 0.6771708726882935, + "rewards/rejected": -1.7712844610214233, + "step": 670 + }, + { + "epoch": 0.18, + "grad_norm": 7.3125, + "learning_rate": 4.9084952541527315e-06, + "logits/chosen": -1.6612813472747803, + "logits/rejected": -1.459668755531311, + "logps/chosen": -392.9391784667969, + "logps/rejected": -423.2802734375, + "loss": 0.5137, + "rewards/accuracies": 0.71875, + "rewards/chosen": -1.1960499286651611, + "rewards/margins": 0.774462878704071, + "rewards/rejected": -1.9705129861831665, + "step": 680 + }, + { + "epoch": 0.18, + "grad_norm": 9.75, + "learning_rate": 4.902270717143858e-06, + "logits/chosen": -1.5917980670928955, + "logits/rejected": -1.4846012592315674, + "logps/chosen": -386.42498779296875, + "logps/rejected": -509.9639587402344, + "loss": 0.4446, + "rewards/accuracies": 0.8062499761581421, + "rewards/chosen": -1.5019272565841675, + "rewards/margins": 1.1186031103134155, + "rewards/rejected": -2.620530605316162, + "step": 690 + }, + { + "epoch": 0.18, + "grad_norm": 10.375, + "learning_rate": 4.895845591221427e-06, + "logits/chosen": -1.5352580547332764, + "logits/rejected": -1.4566528797149658, + "logps/chosen": -440.7987365722656, + "logps/rejected": -521.3024291992188, + "loss": 0.5282, + "rewards/accuracies": 0.762499988079071, + "rewards/chosen": -1.8748830556869507, + "rewards/margins": 0.8497939109802246, + "rewards/rejected": -2.7246768474578857, + "step": 700 + }, + { + "epoch": 0.18, + "eval_logits/chosen": -1.3803926706314087, + "eval_logits/rejected": -1.2514902353286743, + "eval_logps/chosen": -470.9231262207031, + "eval_logps/rejected": -536.3329467773438, + "eval_loss": 0.5560212135314941, + "eval_rewards/accuracies": 0.7164999842643738, + "eval_rewards/chosen": -2.062711000442505, + "eval_rewards/margins": 0.8545322418212891, + "eval_rewards/rejected": -2.917243242263794, + "eval_runtime": 385.2779, + "eval_samples_per_second": 5.191, + "eval_steps_per_second": 0.649, + "step": 700 + }, + { + "epoch": 0.19, + "grad_norm": 11.0, + "learning_rate": 4.8892204128816e-06, + "logits/chosen": -1.5460926294326782, + "logits/rejected": -1.4308459758758545, + "logps/chosen": -454.60797119140625, + "logps/rejected": -516.79296875, + "loss": 0.5244, + "rewards/accuracies": 0.731249988079071, + "rewards/chosen": -1.8704407215118408, + "rewards/margins": 0.7577110528945923, + "rewards/rejected": -2.6281516551971436, + "step": 710 + }, + { + "epoch": 0.19, + "grad_norm": 7.78125, + "learning_rate": 4.882395735324864e-06, + "logits/chosen": -1.5478827953338623, + "logits/rejected": -1.4022762775421143, + "logps/chosen": -418.3267517089844, + "logps/rejected": -491.538818359375, + "loss": 0.5, + "rewards/accuracies": 0.762499988079071, + "rewards/chosen": -1.5224473476409912, + "rewards/margins": 0.9039508104324341, + "rewards/rejected": -2.4263980388641357, + "step": 720 + }, + { + "epoch": 0.19, + "grad_norm": 7.28125, + "learning_rate": 4.87537212840983e-06, + "logits/chosen": -1.5722401142120361, + "logits/rejected": -1.4339630603790283, + "logps/chosen": -399.0245056152344, + "logps/rejected": -433.6438903808594, + "loss": 0.5526, + "rewards/accuracies": 0.71875, + "rewards/chosen": -1.3276931047439575, + "rewards/margins": 0.6516003608703613, + "rewards/rejected": -1.9792934656143188, + "step": 730 + }, + { + "epoch": 0.19, + "grad_norm": 9.8125, + "learning_rate": 4.8681501786056545e-06, + "logits/chosen": -1.4682786464691162, + "logits/rejected": -1.3141381740570068, + "logps/chosen": -346.26873779296875, + "logps/rejected": -393.406005859375, + "loss": 0.5037, + "rewards/accuracies": 0.762499988079071, + "rewards/chosen": -1.1805267333984375, + "rewards/margins": 0.7937687039375305, + "rewards/rejected": -1.9742956161499023, + "step": 740 + }, + { + "epoch": 0.2, + "grad_norm": 15.0, + "learning_rate": 4.860730488943068e-06, + "logits/chosen": -1.3719309568405151, + "logits/rejected": -1.326554536819458, + "logps/chosen": -396.23095703125, + "logps/rejected": -483.92169189453125, + "loss": 0.4852, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -1.6170562505722046, + "rewards/margins": 0.8960045576095581, + "rewards/rejected": -2.513061046600342, + "step": 750 + }, + { + "epoch": 0.2, + "grad_norm": 6.59375, + "learning_rate": 4.853113678964022e-06, + "logits/chosen": -1.3708162307739258, + "logits/rejected": -1.295253038406372, + "logps/chosen": -474.16522216796875, + "logps/rejected": -546.8492431640625, + "loss": 0.5003, + "rewards/accuracies": 0.7562500238418579, + "rewards/chosen": -2.0177154541015625, + "rewards/margins": 0.8057907223701477, + "rewards/rejected": -2.8235061168670654, + "step": 760 + }, + { + "epoch": 0.2, + "grad_norm": 6.875, + "learning_rate": 4.845300384669958e-06, + "logits/chosen": -1.4265882968902588, + "logits/rejected": -1.292400598526001, + "logps/chosen": -428.4332580566406, + "logps/rejected": -476.52471923828125, + "loss": 0.5517, + "rewards/accuracies": 0.6937500238418579, + "rewards/chosen": -1.7522939443588257, + "rewards/margins": 0.7091314196586609, + "rewards/rejected": -2.461425304412842, + "step": 770 + }, + { + "epoch": 0.2, + "grad_norm": 12.3125, + "learning_rate": 4.837291258468701e-06, + "logits/chosen": -1.4694699048995972, + "logits/rejected": -1.3255608081817627, + "logps/chosen": -471.6322326660156, + "logps/rejected": -542.5789794921875, + "loss": 0.5457, + "rewards/accuracies": 0.7124999761581421, + "rewards/chosen": -1.8628593683242798, + "rewards/margins": 0.9583638310432434, + "rewards/rejected": -2.821223258972168, + "step": 780 + }, + { + "epoch": 0.21, + "grad_norm": 9.8125, + "learning_rate": 4.829086969119984e-06, + "logits/chosen": -1.3006738424301147, + "logits/rejected": -1.2943049669265747, + "logps/chosen": -477.1316833496094, + "logps/rejected": -552.9581298828125, + "loss": 0.6434, + "rewards/accuracies": 0.6625000238418579, + "rewards/chosen": -2.2640693187713623, + "rewards/margins": 0.740705132484436, + "rewards/rejected": -3.004774570465088, + "step": 790 + }, + { + "epoch": 0.21, + "grad_norm": 9.625, + "learning_rate": 4.820688201679605e-06, + "logits/chosen": -1.5880753993988037, + "logits/rejected": -1.2926288843154907, + "logps/chosen": -464.5062561035156, + "logps/rejected": -476.73968505859375, + "loss": 0.5205, + "rewards/accuracies": 0.731249988079071, + "rewards/chosen": -2.0187087059020996, + "rewards/margins": 0.745971143245697, + "rewards/rejected": -2.7646796703338623, + "step": 800 + }, + { + "epoch": 0.21, + "eval_logits/chosen": -1.4040697813034058, + "eval_logits/rejected": -1.275604009628296, + "eval_logps/chosen": -434.3306884765625, + "eval_logps/rejected": -484.6470031738281, + "eval_loss": 0.5363825559616089, + "eval_rewards/accuracies": 0.7264999747276306, + "eval_rewards/chosen": -1.6967861652374268, + "eval_rewards/margins": 0.7035976052284241, + "eval_rewards/rejected": -2.400383472442627, + "eval_runtime": 385.6904, + "eval_samples_per_second": 5.186, + "eval_steps_per_second": 0.648, + "step": 800 + }, + { + "epoch": 0.21, + "grad_norm": 7.28125, + "learning_rate": 4.8120956574422315e-06, + "logits/chosen": -1.5584218502044678, + "logits/rejected": -1.5333359241485596, + "logps/chosen": -437.0270080566406, + "logps/rejected": -483.4334411621094, + "loss": 0.5977, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -1.6310217380523682, + "rewards/margins": 0.5348454117774963, + "rewards/rejected": -2.165867328643799, + "step": 810 + }, + { + "epoch": 0.21, + "grad_norm": 8.3125, + "learning_rate": 4.803310053882831e-06, + "logits/chosen": -1.5471457242965698, + "logits/rejected": -1.5452721118927002, + "logps/chosen": -370.4143981933594, + "logps/rejected": -443.0765686035156, + "loss": 0.5346, + "rewards/accuracies": 0.7124999761581421, + "rewards/chosen": -1.4108860492706299, + "rewards/margins": 0.6031405925750732, + "rewards/rejected": -2.014026641845703, + "step": 820 + }, + { + "epoch": 0.22, + "grad_norm": 7.65625, + "learning_rate": 4.794332124596775e-06, + "logits/chosen": -1.542186975479126, + "logits/rejected": -1.4280331134796143, + "logps/chosen": -428.5491638183594, + "logps/rejected": -479.2059631347656, + "loss": 0.5879, + "rewards/accuracies": 0.668749988079071, + "rewards/chosen": -1.6010816097259521, + "rewards/margins": 0.5433088541030884, + "rewards/rejected": -2.144390344619751, + "step": 830 + }, + { + "epoch": 0.22, + "grad_norm": 6.25, + "learning_rate": 4.785162619238575e-06, + "logits/chosen": -1.5202006101608276, + "logits/rejected": -1.357043743133545, + "logps/chosen": -382.9326171875, + "logps/rejected": -429.1929626464844, + "loss": 0.5204, + "rewards/accuracies": 0.737500011920929, + "rewards/chosen": -1.2908388376235962, + "rewards/margins": 0.7357919216156006, + "rewards/rejected": -2.0266308784484863, + "step": 840 + }, + { + "epoch": 0.22, + "grad_norm": 8.375, + "learning_rate": 4.775802303459288e-06, + "logits/chosen": -1.4242273569107056, + "logits/rejected": -1.3464834690093994, + "logps/chosen": -368.41864013671875, + "logps/rejected": -433.1285705566406, + "loss": 0.5601, + "rewards/accuracies": 0.65625, + "rewards/chosen": -1.2065433263778687, + "rewards/margins": 0.69434654712677, + "rewards/rejected": -1.9008897542953491, + "step": 850 + }, + { + "epoch": 0.23, + "grad_norm": 11.5625, + "learning_rate": 4.766251958842589e-06, + "logits/chosen": -1.3795868158340454, + "logits/rejected": -1.2619365453720093, + "logps/chosen": -401.754150390625, + "logps/rejected": -458.87786865234375, + "loss": 0.5436, + "rewards/accuracies": 0.71875, + "rewards/chosen": -1.2925317287445068, + "rewards/margins": 0.6759337186813354, + "rewards/rejected": -1.9684655666351318, + "step": 860 + }, + { + "epoch": 0.23, + "grad_norm": 7.375, + "learning_rate": 4.7565123828395066e-06, + "logits/chosen": -1.2665306329727173, + "logits/rejected": -1.1762630939483643, + "logps/chosen": -408.20220947265625, + "logps/rejected": -490.75634765625, + "loss": 0.5077, + "rewards/accuracies": 0.7437499761581421, + "rewards/chosen": -1.522733449935913, + "rewards/margins": 0.8316558599472046, + "rewards/rejected": -2.3543894290924072, + "step": 870 + }, + { + "epoch": 0.23, + "grad_norm": 7.71875, + "learning_rate": 4.746584388701831e-06, + "logits/chosen": -1.2851665019989014, + "logits/rejected": -1.2308191061019897, + "logps/chosen": -443.7979431152344, + "logps/rejected": -522.8231201171875, + "loss": 0.5035, + "rewards/accuracies": 0.7875000238418579, + "rewards/chosen": -1.8055862188339233, + "rewards/margins": 0.9323374629020691, + "rewards/rejected": -2.7379238605499268, + "step": 880 + }, + { + "epoch": 0.23, + "grad_norm": 10.5625, + "learning_rate": 4.736468805414218e-06, + "logits/chosen": -1.250001072883606, + "logits/rejected": -1.2183704376220703, + "logps/chosen": -407.4109802246094, + "logps/rejected": -504.15130615234375, + "loss": 0.5632, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -1.5997565984725952, + "rewards/margins": 0.8288668394088745, + "rewards/rejected": -2.428623676300049, + "step": 890 + }, + { + "epoch": 0.24, + "grad_norm": 10.375, + "learning_rate": 4.7261664776249595e-06, + "logits/chosen": -1.1423993110656738, + "logits/rejected": -1.050307273864746, + "logps/chosen": -386.7989807128906, + "logps/rejected": -467.6982421875, + "loss": 0.4983, + "rewards/accuracies": 0.75, + "rewards/chosen": -1.5788599252700806, + "rewards/margins": 0.8989045023918152, + "rewards/rejected": -2.477764368057251, + "step": 900 + }, + { + "epoch": 0.24, + "eval_logits/chosen": -1.2160884141921997, + "eval_logits/rejected": -1.095556616783142, + "eval_logps/chosen": -432.6338806152344, + "eval_logps/rejected": -489.990966796875, + "eval_loss": 0.5329138040542603, + "eval_rewards/accuracies": 0.7204999923706055, + "eval_rewards/chosen": -1.679819107055664, + "eval_rewards/margins": 0.7740048170089722, + "eval_rewards/rejected": -2.4538238048553467, + "eval_runtime": 384.8785, + "eval_samples_per_second": 5.196, + "eval_steps_per_second": 0.65, + "step": 900 + }, + { + "epoch": 0.24, + "grad_norm": 8.8125, + "learning_rate": 4.715678265575463e-06, + "logits/chosen": -1.3864091634750366, + "logits/rejected": -1.188279390335083, + "logps/chosen": -431.4292907714844, + "logps/rejected": -437.1918029785156, + "loss": 0.5621, + "rewards/accuracies": 0.71875, + "rewards/chosen": -1.4715720415115356, + "rewards/margins": 0.6131043434143066, + "rewards/rejected": -2.0846762657165527, + "step": 910 + }, + { + "epoch": 0.24, + "grad_norm": 9.1875, + "learning_rate": 4.705005045028415e-06, + "logits/chosen": -1.3089066743850708, + "logits/rejected": -1.1869691610336304, + "logps/chosen": -409.567626953125, + "logps/rejected": -467.461181640625, + "loss": 0.5395, + "rewards/accuracies": 0.737500011920929, + "rewards/chosen": -1.4643157720565796, + "rewards/margins": 0.7529044151306152, + "rewards/rejected": -2.2172200679779053, + "step": 920 + }, + { + "epoch": 0.24, + "grad_norm": 10.1875, + "learning_rate": 4.694147707194659e-06, + "logits/chosen": -1.3738486766815186, + "logits/rejected": -1.295534372329712, + "logps/chosen": -467.11102294921875, + "logps/rejected": -519.7848510742188, + "loss": 0.5253, + "rewards/accuracies": 0.71875, + "rewards/chosen": -1.9523290395736694, + "rewards/margins": 0.7249565720558167, + "rewards/rejected": -2.677285671234131, + "step": 930 + }, + { + "epoch": 0.25, + "grad_norm": 8.5, + "learning_rate": 4.683107158658782e-06, + "logits/chosen": -1.2867457866668701, + "logits/rejected": -1.202335000038147, + "logps/chosen": -514.80126953125, + "logps/rejected": -559.9244384765625, + "loss": 0.5364, + "rewards/accuracies": 0.7124999761581421, + "rewards/chosen": -2.245605707168579, + "rewards/margins": 0.7386397123336792, + "rewards/rejected": -2.9842450618743896, + "step": 940 + }, + { + "epoch": 0.25, + "grad_norm": 7.9375, + "learning_rate": 4.671884321303407e-06, + "logits/chosen": -1.4214376211166382, + "logits/rejected": -1.286566972732544, + "logps/chosen": -444.1116638183594, + "logps/rejected": -498.4622497558594, + "loss": 0.5246, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -1.9778563976287842, + "rewards/margins": 0.6870826482772827, + "rewards/rejected": -2.6649391651153564, + "step": 950 + }, + { + "epoch": 0.25, + "grad_norm": 7.625, + "learning_rate": 4.660480132232224e-06, + "logits/chosen": -1.5147006511688232, + "logits/rejected": -1.4103442430496216, + "logps/chosen": -423.77252197265625, + "logps/rejected": -456.38330078125, + "loss": 0.5784, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -1.5195940732955933, + "rewards/margins": 0.5671976804733276, + "rewards/rejected": -2.086791753768921, + "step": 960 + }, + { + "epoch": 0.25, + "grad_norm": 14.4375, + "learning_rate": 4.6488955436917414e-06, + "logits/chosen": -1.5512803792953491, + "logits/rejected": -1.333134412765503, + "logps/chosen": -416.9090270996094, + "logps/rejected": -457.83563232421875, + "loss": 0.5199, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -1.369279384613037, + "rewards/margins": 0.875058650970459, + "rewards/rejected": -2.244338274002075, + "step": 970 + }, + { + "epoch": 0.26, + "grad_norm": 5.71875, + "learning_rate": 4.6371315229917644e-06, + "logits/chosen": -1.4992878437042236, + "logits/rejected": -1.369624376296997, + "logps/chosen": -448.16314697265625, + "logps/rejected": -506.52191162109375, + "loss": 0.527, + "rewards/accuracies": 0.7124999761581421, + "rewards/chosen": -1.6474645137786865, + "rewards/margins": 0.7875878810882568, + "rewards/rejected": -2.4350523948669434, + "step": 980 + }, + { + "epoch": 0.26, + "grad_norm": 11.9375, + "learning_rate": 4.625189052424638e-06, + "logits/chosen": -1.4010595083236694, + "logits/rejected": -1.259636402130127, + "logps/chosen": -442.61590576171875, + "logps/rejected": -522.6851196289062, + "loss": 0.4455, + "rewards/accuracies": 0.8062499761581421, + "rewards/chosen": -2.035126209259033, + "rewards/margins": 1.0280003547668457, + "rewards/rejected": -3.063126802444458, + "step": 990 + }, + { + "epoch": 0.26, + "grad_norm": 9.5, + "learning_rate": 4.613069129183218e-06, + "logits/chosen": -1.4701205492019653, + "logits/rejected": -1.3206748962402344, + "logps/chosen": -506.3350524902344, + "logps/rejected": -548.3522338867188, + "loss": 0.5443, + "rewards/accuracies": 0.7437499761581421, + "rewards/chosen": -2.02931809425354, + "rewards/margins": 0.7819973826408386, + "rewards/rejected": -2.8113150596618652, + "step": 1000 + }, + { + "epoch": 0.26, + "eval_logits/chosen": -1.2533063888549805, + "eval_logits/rejected": -1.1263666152954102, + "eval_logps/chosen": -454.4657897949219, + "eval_logps/rejected": -521.2656860351562, + "eval_loss": 0.5279005169868469, + "eval_rewards/accuracies": 0.7239999771118164, + "eval_rewards/chosen": -1.8981376886367798, + "eval_rewards/margins": 0.8684335947036743, + "eval_rewards/rejected": -2.766571283340454, + "eval_runtime": 396.7737, + "eval_samples_per_second": 5.041, + "eval_steps_per_second": 0.63, + "step": 1000 + }, + { + "epoch": 0.26, + "grad_norm": 8.8125, + "learning_rate": 4.600772765277607e-06, + "logits/chosen": -1.295363187789917, + "logits/rejected": -1.21110200881958, + "logps/chosen": -406.8030700683594, + "logps/rejected": -491.1182556152344, + "loss": 0.4907, + "rewards/accuracies": 0.768750011920929, + "rewards/chosen": -1.7324583530426025, + "rewards/margins": 0.8947044610977173, + "rewards/rejected": -2.6271629333496094, + "step": 1010 + }, + { + "epoch": 0.27, + "grad_norm": 10.5625, + "learning_rate": 4.588300987450652e-06, + "logits/chosen": -1.3897264003753662, + "logits/rejected": -1.2775170803070068, + "logps/chosen": -400.9536437988281, + "logps/rejected": -446.50244140625, + "loss": 0.5402, + "rewards/accuracies": 0.706250011920929, + "rewards/chosen": -1.4720966815948486, + "rewards/margins": 0.7910727262496948, + "rewards/rejected": -2.263169050216675, + "step": 1020 + }, + { + "epoch": 0.27, + "grad_norm": 6.5, + "learning_rate": 4.5756548370922136e-06, + "logits/chosen": -1.3535692691802979, + "logits/rejected": -1.2674891948699951, + "logps/chosen": -364.35394287109375, + "logps/rejected": -435.6012268066406, + "loss": 0.5053, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -1.2365007400512695, + "rewards/margins": 0.8022448420524597, + "rewards/rejected": -2.038745641708374, + "step": 1030 + }, + { + "epoch": 0.27, + "grad_norm": 10.9375, + "learning_rate": 4.562835370152206e-06, + "logits/chosen": -1.4244534969329834, + "logits/rejected": -1.2097485065460205, + "logps/chosen": -458.898681640625, + "logps/rejected": -542.5496215820312, + "loss": 0.4608, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -1.5766918659210205, + "rewards/margins": 1.1474573612213135, + "rewards/rejected": -2.724148988723755, + "step": 1040 + }, + { + "epoch": 0.27, + "grad_norm": 6.8125, + "learning_rate": 4.54984365705243e-06, + "logits/chosen": -1.3653185367584229, + "logits/rejected": -1.2626806497573853, + "logps/chosen": -441.8795471191406, + "logps/rejected": -547.1009521484375, + "loss": 0.5005, + "rewards/accuracies": 0.731249988079071, + "rewards/chosen": -1.7340948581695557, + "rewards/margins": 1.117974042892456, + "rewards/rejected": -2.8520689010620117, + "step": 1050 + }, + { + "epoch": 0.28, + "grad_norm": 10.1875, + "learning_rate": 4.536680782597191e-06, + "logits/chosen": -1.2812556028366089, + "logits/rejected": -1.2028696537017822, + "logps/chosen": -422.71746826171875, + "logps/rejected": -500.39276123046875, + "loss": 0.5823, + "rewards/accuracies": 0.7124999761581421, + "rewards/chosen": -1.8669233322143555, + "rewards/margins": 0.8329373598098755, + "rewards/rejected": -2.6998608112335205, + "step": 1060 + }, + { + "epoch": 0.28, + "grad_norm": 20.625, + "learning_rate": 4.523347845882718e-06, + "logits/chosen": -1.4049158096313477, + "logits/rejected": -1.2168524265289307, + "logps/chosen": -458.36956787109375, + "logps/rejected": -519.8746337890625, + "loss": 0.4512, + "rewards/accuracies": 0.75, + "rewards/chosen": -1.7019942998886108, + "rewards/margins": 1.1078611612319946, + "rewards/rejected": -2.8098552227020264, + "step": 1070 + }, + { + "epoch": 0.28, + "grad_norm": 7.3125, + "learning_rate": 4.50984596020539e-06, + "logits/chosen": -1.2350142002105713, + "logits/rejected": -1.166898488998413, + "logps/chosen": -446.97833251953125, + "logps/rejected": -499.6363220214844, + "loss": 0.5662, + "rewards/accuracies": 0.6937500238418579, + "rewards/chosen": -1.6682651042938232, + "rewards/margins": 0.8490194082260132, + "rewards/rejected": -2.517284393310547, + "step": 1080 + }, + { + "epoch": 0.29, + "grad_norm": 7.125, + "learning_rate": 4.4961762529687745e-06, + "logits/chosen": -1.427812933921814, + "logits/rejected": -1.3044004440307617, + "logps/chosen": -407.113525390625, + "logps/rejected": -485.9717712402344, + "loss": 0.4963, + "rewards/accuracies": 0.768750011920929, + "rewards/chosen": -1.4671194553375244, + "rewards/margins": 0.9535934329032898, + "rewards/rejected": -2.420712947845459, + "step": 1090 + }, + { + "epoch": 0.29, + "grad_norm": 6.96875, + "learning_rate": 4.482339865589492e-06, + "logits/chosen": -1.4146265983581543, + "logits/rejected": -1.2579143047332764, + "logps/chosen": -434.85498046875, + "logps/rejected": -476.2469787597656, + "loss": 0.565, + "rewards/accuracies": 0.7124999761581421, + "rewards/chosen": -1.685683012008667, + "rewards/margins": 0.8535755276679993, + "rewards/rejected": -2.5392584800720215, + "step": 1100 + }, + { + "epoch": 0.29, + "eval_logits/chosen": -1.271475911140442, + "eval_logits/rejected": -1.1444941759109497, + "eval_logps/chosen": -415.9482727050781, + "eval_logps/rejected": -478.28485107421875, + "eval_loss": 0.5207434296607971, + "eval_rewards/accuracies": 0.7289999723434448, + "eval_rewards/chosen": -1.5129626989364624, + "eval_rewards/margins": 0.8237999677658081, + "eval_rewards/rejected": -2.3367626667022705, + "eval_runtime": 400.4213, + "eval_samples_per_second": 4.995, + "eval_steps_per_second": 0.624, + "step": 1100 + }, + { + "epoch": 0.29, + "grad_norm": 7.1875, + "learning_rate": 4.468337953401909e-06, + "logits/chosen": -1.4210216999053955, + "logits/rejected": -1.3622697591781616, + "logps/chosen": -422.3374938964844, + "logps/rejected": -482.82489013671875, + "loss": 0.5625, + "rewards/accuracies": 0.7124999761581421, + "rewards/chosen": -1.5198637247085571, + "rewards/margins": 0.6534841656684875, + "rewards/rejected": -2.1733479499816895, + "step": 1110 + }, + { + "epoch": 0.29, + "grad_norm": 8.5, + "learning_rate": 4.45417168556166e-06, + "logits/chosen": -1.328687310218811, + "logits/rejected": -1.2253139019012451, + "logps/chosen": -382.0760192871094, + "logps/rejected": -463.2789001464844, + "loss": 0.5021, + "rewards/accuracies": 0.7437499761581421, + "rewards/chosen": -1.4092481136322021, + "rewards/margins": 0.8184933662414551, + "rewards/rejected": -2.2277414798736572, + "step": 1120 + }, + { + "epoch": 0.3, + "grad_norm": 9.25, + "learning_rate": 4.439842244948036e-06, + "logits/chosen": -1.2945640087127686, + "logits/rejected": -1.1504161357879639, + "logps/chosen": -432.81524658203125, + "logps/rejected": -501.5387268066406, + "loss": 0.5614, + "rewards/accuracies": 0.706250011920929, + "rewards/chosen": -1.7175318002700806, + "rewards/margins": 0.742792010307312, + "rewards/rejected": -2.4603238105773926, + "step": 1130 + }, + { + "epoch": 0.3, + "grad_norm": 17.75, + "learning_rate": 4.425350828065204e-06, + "logits/chosen": -1.3531075716018677, + "logits/rejected": -1.145227074623108, + "logps/chosen": -450.9183654785156, + "logps/rejected": -492.70623779296875, + "loss": 0.495, + "rewards/accuracies": 0.7875000238418579, + "rewards/chosen": -1.6683238744735718, + "rewards/margins": 0.8965514898300171, + "rewards/rejected": -2.564875364303589, + "step": 1140 + }, + { + "epoch": 0.3, + "grad_norm": 10.375, + "learning_rate": 4.410698644942303e-06, + "logits/chosen": -1.3738346099853516, + "logits/rejected": -1.2316032648086548, + "logps/chosen": -427.7021484375, + "logps/rejected": -499.776611328125, + "loss": 0.4885, + "rewards/accuracies": 0.78125, + "rewards/chosen": -1.5538952350616455, + "rewards/margins": 0.9599501490592957, + "rewards/rejected": -2.513845443725586, + "step": 1150 + }, + { + "epoch": 0.3, + "grad_norm": 11.8125, + "learning_rate": 4.395886919032406e-06, + "logits/chosen": -1.240241527557373, + "logits/rejected": -1.113989233970642, + "logps/chosen": -439.04095458984375, + "logps/rejected": -497.86297607421875, + "loss": 0.5348, + "rewards/accuracies": 0.731249988079071, + "rewards/chosen": -1.7520458698272705, + "rewards/margins": 0.8531185984611511, + "rewards/rejected": -2.6051642894744873, + "step": 1160 + }, + { + "epoch": 0.31, + "grad_norm": 8.3125, + "learning_rate": 4.380916887110366e-06, + "logits/chosen": -1.3933711051940918, + "logits/rejected": -1.2005256414413452, + "logps/chosen": -448.8775329589844, + "logps/rejected": -503.5738220214844, + "loss": 0.5142, + "rewards/accuracies": 0.762499988079071, + "rewards/chosen": -1.897091269493103, + "rewards/margins": 0.9505263566970825, + "rewards/rejected": -2.8476176261901855, + "step": 1170 + }, + { + "epoch": 0.31, + "grad_norm": 6.71875, + "learning_rate": 4.365789799169539e-06, + "logits/chosen": -1.2259958982467651, + "logits/rejected": -1.2687208652496338, + "logps/chosen": -432.62786865234375, + "logps/rejected": -522.6715698242188, + "loss": 0.5228, + "rewards/accuracies": 0.731249988079071, + "rewards/chosen": -1.8601690530776978, + "rewards/margins": 0.8469558954238892, + "rewards/rejected": -2.707124710083008, + "step": 1180 + }, + { + "epoch": 0.31, + "grad_norm": 10.0625, + "learning_rate": 4.350506918317416e-06, + "logits/chosen": -1.4284965991973877, + "logits/rejected": -1.2679976224899292, + "logps/chosen": -413.3136291503906, + "logps/rejected": -491.58721923828125, + "loss": 0.4982, + "rewards/accuracies": 0.75, + "rewards/chosen": -1.6571779251098633, + "rewards/margins": 0.8545892834663391, + "rewards/rejected": -2.5117671489715576, + "step": 1190 + }, + { + "epoch": 0.31, + "grad_norm": 8.5625, + "learning_rate": 4.335069520670149e-06, + "logits/chosen": -1.2467167377471924, + "logits/rejected": -1.170364260673523, + "logps/chosen": -406.6671447753906, + "logps/rejected": -487.4442443847656, + "loss": 0.5837, + "rewards/accuracies": 0.6875, + "rewards/chosen": -1.7695426940917969, + "rewards/margins": 0.7259476780891418, + "rewards/rejected": -2.495490312576294, + "step": 1200 + }, + { + "epoch": 0.31, + "eval_logits/chosen": -1.2314388751983643, + "eval_logits/rejected": -1.10646390914917, + "eval_logps/chosen": -431.9436950683594, + "eval_logps/rejected": -498.354736328125, + "eval_loss": 0.5104092359542847, + "eval_rewards/accuracies": 0.7354999780654907, + "eval_rewards/chosen": -1.6729168891906738, + "eval_rewards/margins": 0.8645446300506592, + "eval_rewards/rejected": -2.537461280822754, + "eval_runtime": 385.0952, + "eval_samples_per_second": 5.194, + "eval_steps_per_second": 0.649, + "step": 1200 + }, + { + "epoch": 0.32, + "grad_norm": 8.375, + "learning_rate": 4.319478895246e-06, + "logits/chosen": -1.3048737049102783, + "logits/rejected": -1.1300756931304932, + "logps/chosen": -410.984375, + "logps/rejected": -473.4725036621094, + "loss": 0.4842, + "rewards/accuracies": 0.768750011920929, + "rewards/chosen": -1.638689637184143, + "rewards/margins": 0.871795654296875, + "rewards/rejected": -2.5104851722717285, + "step": 1210 + }, + { + "epoch": 0.32, + "grad_norm": 17.875, + "learning_rate": 4.303736343857704e-06, + "logits/chosen": -1.2923145294189453, + "logits/rejected": -1.1951522827148438, + "logps/chosen": -442.15472412109375, + "logps/rejected": -570.807861328125, + "loss": 0.4982, + "rewards/accuracies": 0.71875, + "rewards/chosen": -1.8504784107208252, + "rewards/margins": 1.1673939228057861, + "rewards/rejected": -3.0178723335266113, + "step": 1220 + }, + { + "epoch": 0.32, + "grad_norm": 6.03125, + "learning_rate": 4.287843181003772e-06, + "logits/chosen": -1.347501277923584, + "logits/rejected": -1.1933776140213013, + "logps/chosen": -496.73077392578125, + "logps/rejected": -529.7448120117188, + "loss": 0.5797, + "rewards/accuracies": 0.75, + "rewards/chosen": -2.02878999710083, + "rewards/margins": 0.7935463190078735, + "rewards/rejected": -2.822335958480835, + "step": 1230 + }, + { + "epoch": 0.32, + "grad_norm": 7.90625, + "learning_rate": 4.27180073375873e-06, + "logits/chosen": -1.411339521408081, + "logits/rejected": -1.2728986740112305, + "logps/chosen": -444.9125061035156, + "logps/rejected": -488.64794921875, + "loss": 0.5208, + "rewards/accuracies": 0.731249988079071, + "rewards/chosen": -1.5754519701004028, + "rewards/margins": 0.8528854250907898, + "rewards/rejected": -2.428337574005127, + "step": 1240 + }, + { + "epoch": 0.33, + "grad_norm": 5.59375, + "learning_rate": 4.255610341662304e-06, + "logits/chosen": -1.4314241409301758, + "logits/rejected": -1.2228209972381592, + "logps/chosen": -408.0103454589844, + "logps/rejected": -470.0995178222656, + "loss": 0.5539, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -1.5272233486175537, + "rewards/margins": 0.8200508952140808, + "rewards/rejected": -2.3472743034362793, + "step": 1250 + }, + { + "epoch": 0.33, + "grad_norm": 9.1875, + "learning_rate": 4.2392733566075764e-06, + "logits/chosen": -1.4001052379608154, + "logits/rejected": -1.2643131017684937, + "logps/chosen": -446.30975341796875, + "logps/rejected": -492.1767578125, + "loss": 0.5838, + "rewards/accuracies": 0.6812499761581421, + "rewards/chosen": -1.9030348062515259, + "rewards/margins": 0.6141453981399536, + "rewards/rejected": -2.5171799659729004, + "step": 1260 + }, + { + "epoch": 0.33, + "grad_norm": 8.4375, + "learning_rate": 4.2227911427280975e-06, + "logits/chosen": -1.3647363185882568, + "logits/rejected": -1.1842234134674072, + "logps/chosen": -421.44952392578125, + "logps/rejected": -471.9312438964844, + "loss": 0.5187, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -1.711439847946167, + "rewards/margins": 0.8332679867744446, + "rewards/rejected": -2.544707775115967, + "step": 1270 + }, + { + "epoch": 0.33, + "grad_norm": 11.1875, + "learning_rate": 4.206165076283983e-06, + "logits/chosen": -1.4086225032806396, + "logits/rejected": -1.258502721786499, + "logps/chosen": -433.7823791503906, + "logps/rejected": -515.7772827148438, + "loss": 0.4621, + "rewards/accuracies": 0.7875000238418579, + "rewards/chosen": -1.878491759300232, + "rewards/margins": 1.0307425260543823, + "rewards/rejected": -2.9092345237731934, + "step": 1280 + }, + { + "epoch": 0.34, + "grad_norm": 12.1875, + "learning_rate": 4.189396545546995e-06, + "logits/chosen": -1.3694835901260376, + "logits/rejected": -1.2587060928344727, + "logps/chosen": -476.4090270996094, + "logps/rejected": -558.2841796875, + "loss": 0.5179, + "rewards/accuracies": 0.762499988079071, + "rewards/chosen": -2.272345781326294, + "rewards/margins": 1.0181748867034912, + "rewards/rejected": -3.290520429611206, + "step": 1290 + }, + { + "epoch": 0.34, + "grad_norm": 11.875, + "learning_rate": 4.172486950684627e-06, + "logits/chosen": -1.3458369970321655, + "logits/rejected": -1.2686779499053955, + "logps/chosen": -503.9125061035156, + "logps/rejected": -597.0067138671875, + "loss": 0.5342, + "rewards/accuracies": 0.706250011920929, + "rewards/chosen": -2.502702236175537, + "rewards/margins": 0.962365448474884, + "rewards/rejected": -3.4650676250457764, + "step": 1300 + }, + { + "epoch": 0.34, + "eval_logits/chosen": -1.1852304935455322, + "eval_logits/rejected": -1.065573811531067, + "eval_logps/chosen": -541.4911499023438, + "eval_logps/rejected": -629.0701293945312, + "eval_loss": 0.5145753026008606, + "eval_rewards/accuracies": 0.7239999771118164, + "eval_rewards/chosen": -2.7683911323547363, + "eval_rewards/margins": 1.0762238502502441, + "eval_rewards/rejected": -3.8446152210235596, + "eval_runtime": 384.8604, + "eval_samples_per_second": 5.197, + "eval_steps_per_second": 0.65, + "step": 1300 + }, + { + "epoch": 0.34, + "grad_norm": 10.875, + "learning_rate": 4.155437703643182e-06, + "logits/chosen": -1.3424588441848755, + "logits/rejected": -1.1757004261016846, + "logps/chosen": -503.0860900878906, + "logps/rejected": -576.0645751953125, + "loss": 0.5175, + "rewards/accuracies": 0.75, + "rewards/chosen": -2.563424587249756, + "rewards/margins": 1.0206806659698486, + "rewards/rejected": -3.5841050148010254, + "step": 1310 + }, + { + "epoch": 0.35, + "grad_norm": 12.5625, + "learning_rate": 4.138250228029882e-06, + "logits/chosen": -1.2897417545318604, + "logits/rejected": -1.2073286771774292, + "logps/chosen": -513.29541015625, + "logps/rejected": -622.9282836914062, + "loss": 0.4944, + "rewards/accuracies": 0.762499988079071, + "rewards/chosen": -2.5636470317840576, + "rewards/margins": 1.061576008796692, + "rewards/rejected": -3.625222682952881, + "step": 1320 + }, + { + "epoch": 0.35, + "grad_norm": 6.40625, + "learning_rate": 4.120925958993994e-06, + "logits/chosen": -1.2789089679718018, + "logits/rejected": -1.1816047430038452, + "logps/chosen": -455.1483459472656, + "logps/rejected": -547.6185302734375, + "loss": 0.5614, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -2.212470293045044, + "rewards/margins": 0.9680255055427551, + "rewards/rejected": -3.180495500564575, + "step": 1330 + }, + { + "epoch": 0.35, + "grad_norm": 10.875, + "learning_rate": 4.103466343106999e-06, + "logits/chosen": -1.451428771018982, + "logits/rejected": -1.3242452144622803, + "logps/chosen": -471.908203125, + "logps/rejected": -530.3082275390625, + "loss": 0.531, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -2.0252251625061035, + "rewards/margins": 0.8427697420120239, + "rewards/rejected": -2.867994785308838, + "step": 1340 + }, + { + "epoch": 0.35, + "grad_norm": 13.375, + "learning_rate": 4.085872838241797e-06, + "logits/chosen": -1.3740856647491455, + "logits/rejected": -1.2394291162490845, + "logps/chosen": -466.03948974609375, + "logps/rejected": -517.3243408203125, + "loss": 0.5872, + "rewards/accuracies": 0.6812499761581421, + "rewards/chosen": -2.0367541313171387, + "rewards/margins": 0.7136788368225098, + "rewards/rejected": -2.7504329681396484, + "step": 1350 + }, + { + "epoch": 0.36, + "grad_norm": 11.625, + "learning_rate": 4.06814691345098e-06, + "logits/chosen": -1.3596174716949463, + "logits/rejected": -1.1983596086502075, + "logps/chosen": -435.5516052246094, + "logps/rejected": -502.25762939453125, + "loss": 0.4841, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -1.8000373840332031, + "rewards/margins": 0.8877509832382202, + "rewards/rejected": -2.687788486480713, + "step": 1360 + }, + { + "epoch": 0.36, + "grad_norm": 14.8125, + "learning_rate": 4.050290048844171e-06, + "logits/chosen": -1.4539238214492798, + "logits/rejected": -1.3574968576431274, + "logps/chosen": -447.8072204589844, + "logps/rejected": -525.6444091796875, + "loss": 0.5237, + "rewards/accuracies": 0.6875, + "rewards/chosen": -1.7919375896453857, + "rewards/margins": 0.829770565032959, + "rewards/rejected": -2.6217081546783447, + "step": 1370 + }, + { + "epoch": 0.36, + "grad_norm": 6.9375, + "learning_rate": 4.032303735464422e-06, + "logits/chosen": -1.498198390007019, + "logits/rejected": -1.3165781497955322, + "logps/chosen": -449.3099670410156, + "logps/rejected": -533.7264404296875, + "loss": 0.4426, + "rewards/accuracies": 0.793749988079071, + "rewards/chosen": -1.7732903957366943, + "rewards/margins": 1.075021505355835, + "rewards/rejected": -2.8483121395111084, + "step": 1380 + }, + { + "epoch": 0.36, + "grad_norm": 8.0625, + "learning_rate": 4.014189475163727e-06, + "logits/chosen": -1.2981091737747192, + "logits/rejected": -1.1843674182891846, + "logps/chosen": -436.0013732910156, + "logps/rejected": -536.7470703125, + "loss": 0.4771, + "rewards/accuracies": 0.7437499761581421, + "rewards/chosen": -1.8134443759918213, + "rewards/margins": 1.0886361598968506, + "rewards/rejected": -2.902080535888672, + "step": 1390 + }, + { + "epoch": 0.37, + "grad_norm": 16.5, + "learning_rate": 3.995948780477605e-06, + "logits/chosen": -1.4245179891586304, + "logits/rejected": -1.2672098875045776, + "logps/chosen": -442.8516540527344, + "logps/rejected": -501.25775146484375, + "loss": 0.5287, + "rewards/accuracies": 0.71875, + "rewards/chosen": -1.7409827709197998, + "rewards/margins": 0.8305804133415222, + "rewards/rejected": -2.571563243865967, + "step": 1400 + }, + { + "epoch": 0.37, + "eval_logits/chosen": -1.2506229877471924, + "eval_logits/rejected": -1.1252646446228027, + "eval_logps/chosen": -455.32855224609375, + "eval_logps/rejected": -530.7439575195312, + "eval_loss": 0.5196547508239746, + "eval_rewards/accuracies": 0.7235000133514404, + "eval_rewards/chosen": -1.9067655801773071, + "eval_rewards/margins": 0.9545875191688538, + "eval_rewards/rejected": -2.8613533973693848, + "eval_runtime": 384.8833, + "eval_samples_per_second": 5.196, + "eval_steps_per_second": 0.65, + "step": 1400 + }, + { + "epoch": 0.37, + "grad_norm": 10.0, + "learning_rate": 3.977583174498816e-06, + "logits/chosen": -1.296602487564087, + "logits/rejected": -1.1704097986221313, + "logps/chosen": -470.2802734375, + "logps/rejected": -584.6675415039062, + "loss": 0.3797, + "rewards/accuracies": 0.84375, + "rewards/chosen": -2.065967082977295, + "rewards/margins": 1.363657832145691, + "rewards/rejected": -3.4296250343322754, + "step": 1410 + }, + { + "epoch": 0.37, + "grad_norm": 12.75, + "learning_rate": 3.959094190750172e-06, + "logits/chosen": -1.2688853740692139, + "logits/rejected": -1.1229419708251953, + "logps/chosen": -514.8965454101562, + "logps/rejected": -601.8798217773438, + "loss": 0.5131, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -2.297360897064209, + "rewards/margins": 1.1389044523239136, + "rewards/rejected": -3.436265468597412, + "step": 1420 + }, + { + "epoch": 0.37, + "grad_norm": 23.625, + "learning_rate": 3.9404833730564975e-06, + "logits/chosen": -1.12982177734375, + "logits/rejected": -1.0124092102050781, + "logps/chosen": -485.57647705078125, + "logps/rejected": -589.0986328125, + "loss": 0.509, + "rewards/accuracies": 0.762499988079071, + "rewards/chosen": -2.324758529663086, + "rewards/margins": 1.1509692668914795, + "rewards/rejected": -3.4757277965545654, + "step": 1430 + }, + { + "epoch": 0.38, + "grad_norm": 18.5, + "learning_rate": 3.921752275415712e-06, + "logits/chosen": -1.1547746658325195, + "logits/rejected": -1.0837663412094116, + "logps/chosen": -519.43994140625, + "logps/rejected": -648.2784423828125, + "loss": 0.4382, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -2.689040184020996, + "rewards/margins": 1.4942899942398071, + "rewards/rejected": -4.1833295822143555, + "step": 1440 + }, + { + "epoch": 0.38, + "grad_norm": 7.125, + "learning_rate": 3.902902461869079e-06, + "logits/chosen": -1.122580885887146, + "logits/rejected": -0.9964879155158997, + "logps/chosen": -557.9876098632812, + "logps/rejected": -675.2171630859375, + "loss": 0.5761, + "rewards/accuracies": 0.731249988079071, + "rewards/chosen": -3.184739351272583, + "rewards/margins": 1.3369649648666382, + "rewards/rejected": -4.52170467376709, + "step": 1450 + }, + { + "epoch": 0.38, + "grad_norm": 14.4375, + "learning_rate": 3.883935506370605e-06, + "logits/chosen": -1.2318613529205322, + "logits/rejected": -1.1085087060928345, + "logps/chosen": -504.7203674316406, + "logps/rejected": -563.6139526367188, + "loss": 0.5652, + "rewards/accuracies": 0.7124999761581421, + "rewards/chosen": -2.5147933959960938, + "rewards/margins": 0.8672431111335754, + "rewards/rejected": -3.3820366859436035, + "step": 1460 + }, + { + "epoch": 0.38, + "grad_norm": 4.46875, + "learning_rate": 3.864852992655617e-06, + "logits/chosen": -1.3699067831039429, + "logits/rejected": -1.2695183753967285, + "logps/chosen": -446.06903076171875, + "logps/rejected": -533.8704833984375, + "loss": 0.4493, + "rewards/accuracies": 0.793749988079071, + "rewards/chosen": -1.9743432998657227, + "rewards/margins": 1.0008807182312012, + "rewards/rejected": -2.975224256515503, + "step": 1470 + }, + { + "epoch": 0.39, + "grad_norm": 7.40625, + "learning_rate": 3.845656514108516e-06, + "logits/chosen": -1.3080308437347412, + "logits/rejected": -1.1480329036712646, + "logps/chosen": -499.15087890625, + "logps/rejected": -540.1708984375, + "loss": 0.5052, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -2.400202512741089, + "rewards/margins": 0.9600592851638794, + "rewards/rejected": -3.360261917114258, + "step": 1480 + }, + { + "epoch": 0.39, + "grad_norm": 10.0625, + "learning_rate": 3.826347673629738e-06, + "logits/chosen": -1.3206231594085693, + "logits/rejected": -1.1406716108322144, + "logps/chosen": -456.264404296875, + "logps/rejected": -542.2021484375, + "loss": 0.4827, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -2.029609203338623, + "rewards/margins": 1.1219456195831299, + "rewards/rejected": -3.151554822921753, + "step": 1490 + }, + { + "epoch": 0.39, + "grad_norm": 16.375, + "learning_rate": 3.8069280835019062e-06, + "logits/chosen": -1.3296412229537964, + "logits/rejected": -1.1789557933807373, + "logps/chosen": -472.45135498046875, + "logps/rejected": -577.9127197265625, + "loss": 0.4634, + "rewards/accuracies": 0.7875000238418579, + "rewards/chosen": -2.0813910961151123, + "rewards/margins": 1.246517300605774, + "rewards/rejected": -3.3279082775115967, + "step": 1500 + }, + { + "epoch": 0.39, + "eval_logits/chosen": -1.2696473598480225, + "eval_logits/rejected": -1.1408381462097168, + "eval_logps/chosen": -478.6543884277344, + "eval_logps/rejected": -568.5231323242188, + "eval_loss": 0.516459047794342, + "eval_rewards/accuracies": 0.7294999957084656, + "eval_rewards/chosen": -2.1400234699249268, + "eval_rewards/margins": 1.0991216897964478, + "eval_rewards/rejected": -3.239145040512085, + "eval_runtime": 384.85, + "eval_samples_per_second": 5.197, + "eval_steps_per_second": 0.65, + "step": 1500 + }, + { + "epoch": 0.4, + "grad_norm": 18.75, + "learning_rate": 3.7873993652552077e-06, + "logits/chosen": -1.2893366813659668, + "logits/rejected": -1.1984317302703857, + "logps/chosen": -447.75994873046875, + "logps/rejected": -531.8748779296875, + "loss": 0.6089, + "rewards/accuracies": 0.668749988079071, + "rewards/chosen": -2.140458583831787, + "rewards/margins": 0.8995217084884644, + "rewards/rejected": -3.039980173110962, + "step": 1510 + }, + { + "epoch": 0.4, + "grad_norm": 9.375, + "learning_rate": 3.7677631495319953e-06, + "logits/chosen": -1.4299360513687134, + "logits/rejected": -1.3164293766021729, + "logps/chosen": -420.2286682128906, + "logps/rejected": -484.6932067871094, + "loss": 0.5232, + "rewards/accuracies": 0.737500011920929, + "rewards/chosen": -1.5654377937316895, + "rewards/margins": 0.824730396270752, + "rewards/rejected": -2.3901684284210205, + "step": 1520 + }, + { + "epoch": 0.4, + "grad_norm": 6.65625, + "learning_rate": 3.748021075950633e-06, + "logits/chosen": -1.4940413236618042, + "logits/rejected": -1.3882437944412231, + "logps/chosen": -430.9405822753906, + "logps/rejected": -477.81365966796875, + "loss": 0.5819, + "rewards/accuracies": 0.6937500238418579, + "rewards/chosen": -1.5629723072052002, + "rewards/margins": 0.5750766396522522, + "rewards/rejected": -2.1380488872528076, + "step": 1530 + }, + { + "epoch": 0.4, + "grad_norm": 9.5625, + "learning_rate": 3.7281747929685824e-06, + "logits/chosen": -1.3339357376098633, + "logits/rejected": -1.1862207651138306, + "logps/chosen": -417.4947814941406, + "logps/rejected": -475.46160888671875, + "loss": 0.5335, + "rewards/accuracies": 0.71875, + "rewards/chosen": -1.7983973026275635, + "rewards/margins": 0.7366180419921875, + "rewards/rejected": -2.535015106201172, + "step": 1540 + }, + { + "epoch": 0.41, + "grad_norm": 7.5625, + "learning_rate": 3.7082259577447604e-06, + "logits/chosen": -1.4374284744262695, + "logits/rejected": -1.3284789323806763, + "logps/chosen": -462.6572265625, + "logps/rejected": -520.775390625, + "loss": 0.4858, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -1.9146639108657837, + "rewards/margins": 0.7811635732650757, + "rewards/rejected": -2.6958274841308594, + "step": 1550 + }, + { + "epoch": 0.41, + "grad_norm": 9.875, + "learning_rate": 3.6881762360011688e-06, + "logits/chosen": -1.45357346534729, + "logits/rejected": -1.2578237056732178, + "logps/chosen": -482.557861328125, + "logps/rejected": -538.22705078125, + "loss": 0.5034, + "rewards/accuracies": 0.71875, + "rewards/chosen": -2.010615110397339, + "rewards/margins": 0.9266406297683716, + "rewards/rejected": -2.937255620956421, + "step": 1560 + }, + { + "epoch": 0.41, + "grad_norm": 9.3125, + "learning_rate": 3.668027301883802e-06, + "logits/chosen": -1.3933765888214111, + "logits/rejected": -1.2354159355163574, + "logps/chosen": -465.87628173828125, + "logps/rejected": -557.3919067382812, + "loss": 0.4966, + "rewards/accuracies": 0.7437499761581421, + "rewards/chosen": -2.145137310028076, + "rewards/margins": 1.0642468929290771, + "rewards/rejected": -3.209383726119995, + "step": 1570 + }, + { + "epoch": 0.41, + "grad_norm": 6.3125, + "learning_rate": 3.64778083782286e-06, + "logits/chosen": -1.283038854598999, + "logits/rejected": -1.264458417892456, + "logps/chosen": -504.4510803222656, + "logps/rejected": -629.853271484375, + "loss": 0.5256, + "rewards/accuracies": 0.7437499761581421, + "rewards/chosen": -2.5184431076049805, + "rewards/margins": 0.9640257954597473, + "rewards/rejected": -3.482469081878662, + "step": 1580 + }, + { + "epoch": 0.42, + "grad_norm": 12.1875, + "learning_rate": 3.627438534392268e-06, + "logits/chosen": -1.3820140361785889, + "logits/rejected": -1.3510745763778687, + "logps/chosen": -486.08721923828125, + "logps/rejected": -596.9356689453125, + "loss": 0.5014, + "rewards/accuracies": 0.762499988079071, + "rewards/chosen": -2.4804224967956543, + "rewards/margins": 1.0413092374801636, + "rewards/rejected": -3.5217316150665283, + "step": 1590 + }, + { + "epoch": 0.42, + "grad_norm": 7.9375, + "learning_rate": 3.607002090168506e-06, + "logits/chosen": -1.2637202739715576, + "logits/rejected": -1.1885454654693604, + "logps/chosen": -528.8330688476562, + "logps/rejected": -598.1657104492188, + "loss": 0.5551, + "rewards/accuracies": 0.6937500238418579, + "rewards/chosen": -2.6452443599700928, + "rewards/margins": 0.9074187278747559, + "rewards/rejected": -3.5526630878448486, + "step": 1600 + }, + { + "epoch": 0.42, + "eval_logits/chosen": -1.2402483224868774, + "eval_logits/rejected": -1.1162159442901611, + "eval_logps/chosen": -512.13427734375, + "eval_logps/rejected": -599.2672119140625, + "eval_loss": 0.5056775212287903, + "eval_rewards/accuracies": 0.7310000061988831, + "eval_rewards/chosen": -2.474822521209717, + "eval_rewards/margins": 1.0717631578445435, + "eval_rewards/rejected": -3.5465855598449707, + "eval_runtime": 384.9561, + "eval_samples_per_second": 5.195, + "eval_steps_per_second": 0.649, + "step": 1600 + }, + { + "epoch": 0.42, + "grad_norm": 4.5625, + "learning_rate": 3.586473211588787e-06, + "logits/chosen": -1.3531196117401123, + "logits/rejected": -1.2485313415527344, + "logps/chosen": -474.69183349609375, + "logps/rejected": -598.9015502929688, + "loss": 0.4461, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -2.299967050552368, + "rewards/margins": 1.1753101348876953, + "rewards/rejected": -3.4752774238586426, + "step": 1610 + }, + { + "epoch": 0.42, + "grad_norm": 16.125, + "learning_rate": 3.5658536128085623e-06, + "logits/chosen": -1.397863745689392, + "logits/rejected": -1.2203514575958252, + "logps/chosen": -538.4635009765625, + "logps/rejected": -599.5867919921875, + "loss": 0.5809, + "rewards/accuracies": 0.6875, + "rewards/chosen": -2.7555203437805176, + "rewards/margins": 0.9067786931991577, + "rewards/rejected": -3.6622989177703857, + "step": 1620 + }, + { + "epoch": 0.43, + "grad_norm": 8.0, + "learning_rate": 3.545145015558399e-06, + "logits/chosen": -1.198150873184204, + "logits/rejected": -1.1703031063079834, + "logps/chosen": -492.5457458496094, + "logps/rejected": -591.6585083007812, + "loss": 0.484, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -2.609083652496338, + "rewards/margins": 1.1332842111587524, + "rewards/rejected": -3.7423675060272217, + "step": 1630 + }, + { + "epoch": 0.43, + "grad_norm": 5.375, + "learning_rate": 3.5243491490002056e-06, + "logits/chosen": -1.3408269882202148, + "logits/rejected": -1.2534765005111694, + "logps/chosen": -525.4117431640625, + "logps/rejected": -613.4912109375, + "loss": 0.5814, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -2.717186450958252, + "rewards/margins": 0.9326400756835938, + "rewards/rejected": -3.649826765060425, + "step": 1640 + }, + { + "epoch": 0.43, + "grad_norm": 10.5, + "learning_rate": 3.503467749582857e-06, + "logits/chosen": -1.4275528192520142, + "logits/rejected": -1.233086109161377, + "logps/chosen": -476.9230041503906, + "logps/rejected": -512.470703125, + "loss": 0.594, + "rewards/accuracies": 0.6875, + "rewards/chosen": -2.256998300552368, + "rewards/margins": 0.6955486536026001, + "rewards/rejected": -2.9525468349456787, + "step": 1650 + }, + { + "epoch": 0.43, + "grad_norm": 11.375, + "learning_rate": 3.4825025608971947e-06, + "logits/chosen": -1.347503662109375, + "logits/rejected": -1.2736327648162842, + "logps/chosen": -408.91351318359375, + "logps/rejected": -484.05926513671875, + "loss": 0.5229, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -1.8696982860565186, + "rewards/margins": 0.7134873270988464, + "rewards/rejected": -2.583185911178589, + "step": 1660 + }, + { + "epoch": 0.44, + "grad_norm": 6.375, + "learning_rate": 3.4614553335304407e-06, + "logits/chosen": -1.3952078819274902, + "logits/rejected": -1.1842278242111206, + "logps/chosen": -461.0892028808594, + "logps/rejected": -531.4512939453125, + "loss": 0.4605, + "rewards/accuracies": 0.75, + "rewards/chosen": -1.9217849969863892, + "rewards/margins": 1.02143132686615, + "rewards/rejected": -2.9432168006896973, + "step": 1670 + }, + { + "epoch": 0.44, + "grad_norm": 10.875, + "learning_rate": 3.4403278249200222e-06, + "logits/chosen": -1.3681023120880127, + "logits/rejected": -1.159461259841919, + "logps/chosen": -495.4026794433594, + "logps/rejected": -574.2462768554688, + "loss": 0.4479, + "rewards/accuracies": 0.737500011920929, + "rewards/chosen": -2.127955913543701, + "rewards/margins": 1.2076514959335327, + "rewards/rejected": -3.3356070518493652, + "step": 1680 + }, + { + "epoch": 0.44, + "grad_norm": 15.1875, + "learning_rate": 3.4191217992068293e-06, + "logits/chosen": -1.4043166637420654, + "logits/rejected": -1.2147849798202515, + "logps/chosen": -536.7705688476562, + "logps/rejected": -593.9422607421875, + "loss": 0.5436, + "rewards/accuracies": 0.737500011920929, + "rewards/chosen": -2.586280345916748, + "rewards/margins": 1.0274693965911865, + "rewards/rejected": -3.6137497425079346, + "step": 1690 + }, + { + "epoch": 0.44, + "grad_norm": 11.5625, + "learning_rate": 3.3978390270879056e-06, + "logits/chosen": -1.2982449531555176, + "logits/rejected": -1.202523946762085, + "logps/chosen": -495.7208557128906, + "logps/rejected": -613.0609130859375, + "loss": 0.5183, + "rewards/accuracies": 0.768750011920929, + "rewards/chosen": -2.8570523262023926, + "rewards/margins": 1.105515480041504, + "rewards/rejected": -3.9625678062438965, + "step": 1700 + }, + { + "epoch": 0.44, + "eval_logits/chosen": -1.2783551216125488, + "eval_logits/rejected": -1.1493244171142578, + "eval_logps/chosen": -543.2153930664062, + "eval_logps/rejected": -629.583251953125, + "eval_loss": 0.49932965636253357, + "eval_rewards/accuracies": 0.7390000224113464, + "eval_rewards/chosen": -2.7856333255767822, + "eval_rewards/margins": 1.0641134977340698, + "eval_rewards/rejected": -3.8497471809387207, + "eval_runtime": 384.9856, + "eval_samples_per_second": 5.195, + "eval_steps_per_second": 0.649, + "step": 1700 + }, + { + "epoch": 0.45, + "grad_norm": 12.3125, + "learning_rate": 3.3764812856685995e-06, + "logits/chosen": -1.3749181032180786, + "logits/rejected": -1.3575098514556885, + "logps/chosen": -486.59808349609375, + "logps/rejected": -601.6511840820312, + "loss": 0.5154, + "rewards/accuracies": 0.71875, + "rewards/chosen": -2.6204190254211426, + "rewards/margins": 1.015373945236206, + "rewards/rejected": -3.6357929706573486, + "step": 1710 + }, + { + "epoch": 0.45, + "grad_norm": 9.5625, + "learning_rate": 3.3550503583141726e-06, + "logits/chosen": -1.4978001117706299, + "logits/rejected": -1.3547831773757935, + "logps/chosen": -509.7710876464844, + "logps/rejected": -603.9198608398438, + "loss": 0.4785, + "rewards/accuracies": 0.78125, + "rewards/chosen": -2.409210205078125, + "rewards/margins": 1.0928349494934082, + "rewards/rejected": -3.5020453929901123, + "step": 1720 + }, + { + "epoch": 0.45, + "grad_norm": 9.8125, + "learning_rate": 3.3335480345008907e-06, + "logits/chosen": -1.359413743019104, + "logits/rejected": -1.2617199420928955, + "logps/chosen": -466.9374084472656, + "logps/rejected": -555.6993408203125, + "loss": 0.4525, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -2.067884922027588, + "rewards/margins": 1.134508728981018, + "rewards/rejected": -3.2023932933807373, + "step": 1730 + }, + { + "epoch": 0.46, + "grad_norm": 7.875, + "learning_rate": 3.3119761096666055e-06, + "logits/chosen": -1.4102518558502197, + "logits/rejected": -1.2553739547729492, + "logps/chosen": -483.46356201171875, + "logps/rejected": -535.0630493164062, + "loss": 0.5686, + "rewards/accuracies": 0.7124999761581421, + "rewards/chosen": -2.0844430923461914, + "rewards/margins": 0.8192558288574219, + "rewards/rejected": -2.9036991596221924, + "step": 1740 + }, + { + "epoch": 0.46, + "grad_norm": 8.875, + "learning_rate": 3.290336385060832e-06, + "logits/chosen": -1.573932409286499, + "logits/rejected": -1.365490436553955, + "logps/chosen": -463.2930603027344, + "logps/rejected": -536.6669921875, + "loss": 0.5262, + "rewards/accuracies": 0.71875, + "rewards/chosen": -2.1443819999694824, + "rewards/margins": 0.9354622960090637, + "rewards/rejected": -3.0798439979553223, + "step": 1750 + }, + { + "epoch": 0.46, + "grad_norm": 10.125, + "learning_rate": 3.268630667594348e-06, + "logits/chosen": -1.4119102954864502, + "logits/rejected": -1.375628113746643, + "logps/chosen": -442.0873107910156, + "logps/rejected": -511.404296875, + "loss": 0.5134, + "rewards/accuracies": 0.6875, + "rewards/chosen": -1.8643802404403687, + "rewards/margins": 0.9355432391166687, + "rewards/rejected": -2.7999236583709717, + "step": 1760 + }, + { + "epoch": 0.46, + "grad_norm": 8.5625, + "learning_rate": 3.2468607696883147e-06, + "logits/chosen": -1.4483537673950195, + "logits/rejected": -1.3931195735931396, + "logps/chosen": -452.58056640625, + "logps/rejected": -550.0877075195312, + "loss": 0.4914, + "rewards/accuracies": 0.7437499761581421, + "rewards/chosen": -1.9940522909164429, + "rewards/margins": 0.9315992593765259, + "rewards/rejected": -2.9256515502929688, + "step": 1770 + }, + { + "epoch": 0.47, + "grad_norm": 6.40625, + "learning_rate": 3.225028509122944e-06, + "logits/chosen": -1.5014052391052246, + "logits/rejected": -1.3815762996673584, + "logps/chosen": -431.58447265625, + "logps/rejected": -498.1240234375, + "loss": 0.5295, + "rewards/accuracies": 0.731249988079071, + "rewards/chosen": -1.9506728649139404, + "rewards/margins": 0.7785587310791016, + "rewards/rejected": -2.729231595993042, + "step": 1780 + }, + { + "epoch": 0.47, + "grad_norm": 11.8125, + "learning_rate": 3.2031357088857083e-06, + "logits/chosen": -1.4752798080444336, + "logits/rejected": -1.3948938846588135, + "logps/chosen": -494.77056884765625, + "logps/rejected": -584.5908813476562, + "loss": 0.5126, + "rewards/accuracies": 0.737500011920929, + "rewards/chosen": -2.1956658363342285, + "rewards/margins": 0.9330615997314453, + "rewards/rejected": -3.128727674484253, + "step": 1790 + }, + { + "epoch": 0.47, + "grad_norm": 14.125, + "learning_rate": 3.181184197019127e-06, + "logits/chosen": -1.227853775024414, + "logits/rejected": -1.1199489831924438, + "logps/chosen": -480.1280212402344, + "logps/rejected": -640.9280395507812, + "loss": 0.478, + "rewards/accuracies": 0.731249988079071, + "rewards/chosen": -2.4991328716278076, + "rewards/margins": 1.3459218740463257, + "rewards/rejected": -3.845055103302002, + "step": 1800 + }, + { + "epoch": 0.47, + "eval_logits/chosen": -1.2418839931488037, + "eval_logits/rejected": -1.1179745197296143, + "eval_logps/chosen": -533.2012329101562, + "eval_logps/rejected": -618.8510131835938, + "eval_loss": 0.5059686303138733, + "eval_rewards/accuracies": 0.7390000224113464, + "eval_rewards/chosen": -2.685492753982544, + "eval_rewards/margins": 1.0569311380386353, + "eval_rewards/rejected": -3.7424237728118896, + "eval_runtime": 385.0443, + "eval_samples_per_second": 5.194, + "eval_steps_per_second": 0.649, + "step": 1800 + }, + { + "epoch": 0.47, + "grad_norm": 22.25, + "learning_rate": 3.159175806468126e-06, + "logits/chosen": -1.206802248954773, + "logits/rejected": -1.0140331983566284, + "logps/chosen": -536.1395263671875, + "logps/rejected": -618.5433959960938, + "loss": 0.4884, + "rewards/accuracies": 0.78125, + "rewards/chosen": -2.874638795852661, + "rewards/margins": 1.125908374786377, + "rewards/rejected": -4.000546932220459, + "step": 1810 + }, + { + "epoch": 0.48, + "grad_norm": 18.125, + "learning_rate": 3.1371123749269804e-06, + "logits/chosen": -1.27443265914917, + "logits/rejected": -1.2016910314559937, + "logps/chosen": -608.0801391601562, + "logps/rejected": -680.9607543945312, + "loss": 0.6001, + "rewards/accuracies": 0.731249988079071, + "rewards/chosen": -3.246840000152588, + "rewards/margins": 0.8893483877182007, + "rewards/rejected": -4.13618803024292, + "step": 1820 + }, + { + "epoch": 0.48, + "grad_norm": 10.125, + "learning_rate": 3.114995744685877e-06, + "logits/chosen": -1.238593339920044, + "logits/rejected": -1.2031556367874146, + "logps/chosen": -542.8019409179688, + "logps/rejected": -626.592041015625, + "loss": 0.5158, + "rewards/accuracies": 0.731249988079071, + "rewards/chosen": -2.95475697517395, + "rewards/margins": 0.9683350324630737, + "rewards/rejected": -3.9230918884277344, + "step": 1830 + }, + { + "epoch": 0.48, + "grad_norm": 5.8125, + "learning_rate": 3.0928277624770743e-06, + "logits/chosen": -1.4476687908172607, + "logits/rejected": -1.2830606698989868, + "logps/chosen": -545.8017578125, + "logps/rejected": -637.268798828125, + "loss": 0.5053, + "rewards/accuracies": 0.7875000238418579, + "rewards/chosen": -2.61057710647583, + "rewards/margins": 1.1988862752914429, + "rewards/rejected": -3.8094639778137207, + "step": 1840 + }, + { + "epoch": 0.48, + "grad_norm": 6.0625, + "learning_rate": 3.070610279320708e-06, + "logits/chosen": -1.469509243965149, + "logits/rejected": -1.2941926717758179, + "logps/chosen": -530.1407470703125, + "logps/rejected": -617.324462890625, + "loss": 0.4534, + "rewards/accuracies": 0.7875000238418579, + "rewards/chosen": -2.4305920600891113, + "rewards/margins": 1.112181544303894, + "rewards/rejected": -3.542773723602295, + "step": 1850 + }, + { + "epoch": 0.49, + "grad_norm": 6.125, + "learning_rate": 3.0483451503702264e-06, + "logits/chosen": -1.3953410387039185, + "logits/rejected": -1.319723129272461, + "logps/chosen": -553.1173706054688, + "logps/rejected": -624.0015258789062, + "loss": 0.5575, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -2.6702237129211426, + "rewards/margins": 0.9092646837234497, + "rewards/rejected": -3.5794882774353027, + "step": 1860 + }, + { + "epoch": 0.49, + "grad_norm": 7.40625, + "learning_rate": 3.0260342347574916e-06, + "logits/chosen": -1.366349697113037, + "logits/rejected": -1.2165526151657104, + "logps/chosen": -525.2376098632812, + "logps/rejected": -642.2418823242188, + "loss": 0.4285, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": -2.52901291847229, + "rewards/margins": 1.3603582382202148, + "rewards/rejected": -3.889371156692505, + "step": 1870 + }, + { + "epoch": 0.49, + "grad_norm": 11.0625, + "learning_rate": 3.0036793954375358e-06, + "logits/chosen": -1.3298574686050415, + "logits/rejected": -1.175115942955017, + "logps/chosen": -573.416015625, + "logps/rejected": -657.1756591796875, + "loss": 0.4426, + "rewards/accuracies": 0.793749988079071, + "rewards/chosen": -3.033194065093994, + "rewards/margins": 1.259937047958374, + "rewards/rejected": -4.293131351470947, + "step": 1880 + }, + { + "epoch": 0.49, + "grad_norm": 16.625, + "learning_rate": 2.981282499033009e-06, + "logits/chosen": -1.3132747411727905, + "logits/rejected": -1.1860190629959106, + "logps/chosen": -596.8779296875, + "logps/rejected": -696.3871459960938, + "loss": 0.5166, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -3.2053794860839844, + "rewards/margins": 1.2494663000106812, + "rewards/rejected": -4.454846382141113, + "step": 1890 + }, + { + "epoch": 0.5, + "grad_norm": 14.5, + "learning_rate": 2.9588454156783163e-06, + "logits/chosen": -1.3445230722427368, + "logits/rejected": -1.1744747161865234, + "logps/chosen": -582.3587646484375, + "logps/rejected": -712.7427978515625, + "loss": 0.4325, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -3.0100293159484863, + "rewards/margins": 1.5214197635650635, + "rewards/rejected": -4.531449317932129, + "step": 1900 + }, + { + "epoch": 0.5, + "eval_logits/chosen": -1.2515246868133545, + "eval_logits/rejected": -1.1244795322418213, + "eval_logps/chosen": -567.7128295898438, + "eval_logps/rejected": -665.8478393554688, + "eval_loss": 0.4995792806148529, + "eval_rewards/accuracies": 0.7369999885559082, + "eval_rewards/chosen": -3.0306081771850586, + "eval_rewards/margins": 1.1817845106124878, + "eval_rewards/rejected": -4.2123918533325195, + "eval_runtime": 384.958, + "eval_samples_per_second": 5.195, + "eval_steps_per_second": 0.649, + "step": 1900 + }, + { + "epoch": 0.5, + "grad_norm": 10.6875, + "learning_rate": 2.9363700188634597e-06, + "logits/chosen": -1.3585935831069946, + "logits/rejected": -1.2228021621704102, + "logps/chosen": -564.982666015625, + "logps/rejected": -625.9069213867188, + "loss": 0.5244, + "rewards/accuracies": 0.737500011920929, + "rewards/chosen": -2.999126434326172, + "rewards/margins": 0.9794772863388062, + "rewards/rejected": -3.9786033630371094, + "step": 1910 + }, + { + "epoch": 0.5, + "grad_norm": 10.625, + "learning_rate": 2.9138581852776053e-06, + "logits/chosen": -1.378015160560608, + "logits/rejected": -1.2671245336532593, + "logps/chosen": -547.2576904296875, + "logps/rejected": -642.3396606445312, + "loss": 0.5133, + "rewards/accuracies": 0.737500011920929, + "rewards/chosen": -2.8754656314849854, + "rewards/margins": 1.0803815126419067, + "rewards/rejected": -3.9558472633361816, + "step": 1920 + }, + { + "epoch": 0.51, + "grad_norm": 6.71875, + "learning_rate": 2.8913117946523805e-06, + "logits/chosen": -1.4040184020996094, + "logits/rejected": -1.2129814624786377, + "logps/chosen": -538.7586669921875, + "logps/rejected": -607.2654418945312, + "loss": 0.4589, + "rewards/accuracies": 0.78125, + "rewards/chosen": -2.72344970703125, + "rewards/margins": 1.065987229347229, + "rewards/rejected": -3.7894368171691895, + "step": 1930 + }, + { + "epoch": 0.51, + "grad_norm": 12.1875, + "learning_rate": 2.8687327296049126e-06, + "logits/chosen": -1.3795297145843506, + "logits/rejected": -1.2698510885238647, + "logps/chosen": -529.3519897460938, + "logps/rejected": -627.5960693359375, + "loss": 0.5143, + "rewards/accuracies": 0.706250011920929, + "rewards/chosen": -2.702291965484619, + "rewards/margins": 1.0797038078308105, + "rewards/rejected": -3.781996250152588, + "step": 1940 + }, + { + "epoch": 0.51, + "grad_norm": 17.0, + "learning_rate": 2.8461228754806376e-06, + "logits/chosen": -1.4074313640594482, + "logits/rejected": -1.231386661529541, + "logps/chosen": -545.1365966796875, + "logps/rejected": -611.1114501953125, + "loss": 0.5387, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -2.6934621334075928, + "rewards/margins": 0.9177573323249817, + "rewards/rejected": -3.611219882965088, + "step": 1950 + }, + { + "epoch": 0.51, + "grad_norm": 8.0625, + "learning_rate": 2.823484120195865e-06, + "logits/chosen": -1.51656174659729, + "logits/rejected": -1.2999058961868286, + "logps/chosen": -509.6175231933594, + "logps/rejected": -593.9664306640625, + "loss": 0.4301, + "rewards/accuracies": 0.7875000238418579, + "rewards/chosen": -2.3180630207061768, + "rewards/margins": 1.187303900718689, + "rewards/rejected": -3.5053672790527344, + "step": 1960 + }, + { + "epoch": 0.52, + "grad_norm": 8.8125, + "learning_rate": 2.8008183540801486e-06, + "logits/chosen": -1.366084337234497, + "logits/rejected": -1.211987853050232, + "logps/chosen": -522.1681518554688, + "logps/rejected": -573.3563232421875, + "loss": 0.5128, + "rewards/accuracies": 0.6937500238418579, + "rewards/chosen": -2.4853363037109375, + "rewards/margins": 0.965847373008728, + "rewards/rejected": -3.451184034347534, + "step": 1970 + }, + { + "epoch": 0.52, + "grad_norm": 14.3125, + "learning_rate": 2.7781274697184353e-06, + "logits/chosen": -1.2134374380111694, + "logits/rejected": -1.2537566423416138, + "logps/chosen": -503.17462158203125, + "logps/rejected": -637.951171875, + "loss": 0.5379, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -2.738171100616455, + "rewards/margins": 1.1446306705474854, + "rewards/rejected": -3.8828015327453613, + "step": 1980 + }, + { + "epoch": 0.52, + "grad_norm": 6.96875, + "learning_rate": 2.7554133617930397e-06, + "logits/chosen": -1.3368358612060547, + "logits/rejected": -1.2081658840179443, + "logps/chosen": -505.2743225097656, + "logps/rejected": -597.1849365234375, + "loss": 0.5015, + "rewards/accuracies": 0.737500011920929, + "rewards/chosen": -2.5743813514709473, + "rewards/margins": 1.0698320865631104, + "rewards/rejected": -3.644213914871216, + "step": 1990 + }, + { + "epoch": 0.52, + "grad_norm": 10.125, + "learning_rate": 2.7326779269254363e-06, + "logits/chosen": -1.5063138008117676, + "logits/rejected": -1.330679178237915, + "logps/chosen": -549.5060424804688, + "logps/rejected": -596.6401977539062, + "loss": 0.4926, + "rewards/accuracies": 0.737500011920929, + "rewards/chosen": -2.6017112731933594, + "rewards/margins": 1.0307661294937134, + "rewards/rejected": -3.6324775218963623, + "step": 2000 + }, + { + "epoch": 0.52, + "eval_logits/chosen": -1.2879465818405151, + "eval_logits/rejected": -1.1607391834259033, + "eval_logps/chosen": -531.1353759765625, + "eval_logps/rejected": -612.32275390625, + "eval_loss": 0.4934316873550415, + "eval_rewards/accuracies": 0.7404999732971191, + "eval_rewards/chosen": -2.664834499359131, + "eval_rewards/margins": 1.0123074054718018, + "eval_rewards/rejected": -3.6771416664123535, + "eval_runtime": 385.0864, + "eval_samples_per_second": 5.194, + "eval_steps_per_second": 0.649, + "step": 2000 + }, + { + "epoch": 0.53, + "grad_norm": 12.0625, + "learning_rate": 2.7099230635178954e-06, + "logits/chosen": -1.32606840133667, + "logits/rejected": -1.284746766090393, + "logps/chosen": -530.9010009765625, + "logps/rejected": -617.3204956054688, + "loss": 0.5175, + "rewards/accuracies": 0.737500011920929, + "rewards/chosen": -2.6605124473571777, + "rewards/margins": 0.9288894534111023, + "rewards/rejected": -3.589401960372925, + "step": 2010 + }, + { + "epoch": 0.53, + "grad_norm": 12.6875, + "learning_rate": 2.6871506715949608e-06, + "logits/chosen": -1.4429805278778076, + "logits/rejected": -1.309429407119751, + "logps/chosen": -509.5052795410156, + "logps/rejected": -599.0560913085938, + "loss": 0.4724, + "rewards/accuracies": 0.75, + "rewards/chosen": -2.565361976623535, + "rewards/margins": 1.0760364532470703, + "rewards/rejected": -3.6413981914520264, + "step": 2020 + }, + { + "epoch": 0.53, + "grad_norm": 11.125, + "learning_rate": 2.6643626526448063e-06, + "logits/chosen": -1.516875982284546, + "logits/rejected": -1.3450555801391602, + "logps/chosen": -554.38427734375, + "logps/rejected": -638.1011352539062, + "loss": 0.4402, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -2.594825267791748, + "rewards/margins": 1.260938048362732, + "rewards/rejected": -3.8557631969451904, + "step": 2030 + }, + { + "epoch": 0.53, + "grad_norm": 9.375, + "learning_rate": 2.6415609094604562e-06, + "logits/chosen": -1.331624150276184, + "logits/rejected": -1.273493766784668, + "logps/chosen": -572.4739990234375, + "logps/rejected": -668.96484375, + "loss": 0.4552, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -3.0190746784210205, + "rewards/margins": 1.1551822423934937, + "rewards/rejected": -4.174256801605225, + "step": 2040 + }, + { + "epoch": 0.54, + "grad_norm": 15.5, + "learning_rate": 2.618747345980904e-06, + "logits/chosen": -1.3320283889770508, + "logits/rejected": -1.138301134109497, + "logps/chosen": -604.2484130859375, + "logps/rejected": -656.2549438476562, + "loss": 0.5591, + "rewards/accuracies": 0.7124999761581421, + "rewards/chosen": -3.5585105419158936, + "rewards/margins": 1.0249335765838623, + "rewards/rejected": -4.583444118499756, + "step": 2050 + }, + { + "epoch": 0.54, + "grad_norm": 5.78125, + "learning_rate": 2.595923867132136e-06, + "logits/chosen": -1.3874107599258423, + "logits/rejected": -1.25830078125, + "logps/chosen": -597.45751953125, + "logps/rejected": -698.0640258789062, + "loss": 0.482, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -3.1653714179992676, + "rewards/margins": 1.2528908252716064, + "rewards/rejected": -4.418262481689453, + "step": 2060 + }, + { + "epoch": 0.54, + "grad_norm": 7.625, + "learning_rate": 2.5730923786680672e-06, + "logits/chosen": -1.3206942081451416, + "logits/rejected": -1.2852368354797363, + "logps/chosen": -542.5220336914062, + "logps/rejected": -643.3800048828125, + "loss": 0.5358, + "rewards/accuracies": 0.6937500238418579, + "rewards/chosen": -2.9099090099334717, + "rewards/margins": 0.932235836982727, + "rewards/rejected": -3.842144727706909, + "step": 2070 + }, + { + "epoch": 0.54, + "grad_norm": 9.75, + "learning_rate": 2.5502547870114137e-06, + "logits/chosen": -1.3924452066421509, + "logits/rejected": -1.2653155326843262, + "logps/chosen": -522.9793090820312, + "logps/rejected": -588.5538940429688, + "loss": 0.5191, + "rewards/accuracies": 0.7562500238418579, + "rewards/chosen": -2.6759915351867676, + "rewards/margins": 0.9582807421684265, + "rewards/rejected": -3.634272336959839, + "step": 2080 + }, + { + "epoch": 0.55, + "grad_norm": 11.5, + "learning_rate": 2.527412999094507e-06, + "logits/chosen": -1.3764816522598267, + "logits/rejected": -1.2036128044128418, + "logps/chosen": -582.7906494140625, + "logps/rejected": -684.7365112304688, + "loss": 0.4795, + "rewards/accuracies": 0.7437499761581421, + "rewards/chosen": -2.890341281890869, + "rewards/margins": 1.1449689865112305, + "rewards/rejected": -4.0353102684021, + "step": 2090 + }, + { + "epoch": 0.55, + "grad_norm": 11.0625, + "learning_rate": 2.504568922200064e-06, + "logits/chosen": -1.3423351049423218, + "logits/rejected": -1.1891554594039917, + "logps/chosen": -519.73388671875, + "logps/rejected": -617.8820190429688, + "loss": 0.5009, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -2.7839534282684326, + "rewards/margins": 1.1430864334106445, + "rewards/rejected": -3.9270401000976562, + "step": 2100 + }, + { + "epoch": 0.55, + "eval_logits/chosen": -1.3098708391189575, + "eval_logits/rejected": -1.1825140714645386, + "eval_logps/chosen": -547.086669921875, + "eval_logps/rejected": -630.5530395507812, + "eval_loss": 0.4915066063404083, + "eval_rewards/accuracies": 0.7509999871253967, + "eval_rewards/chosen": -2.8243465423583984, + "eval_rewards/margins": 1.0350984334945679, + "eval_rewards/rejected": -3.859445095062256, + "eval_runtime": 384.9833, + "eval_samples_per_second": 5.195, + "eval_steps_per_second": 0.649, + "step": 2100 + }, + { + "epoch": 0.55, + "grad_norm": 7.25, + "learning_rate": 2.4817244638019333e-06, + "logits/chosen": -1.4199883937835693, + "logits/rejected": -1.2606356143951416, + "logps/chosen": -566.4400634765625, + "logps/rejected": -621.7224731445312, + "loss": 0.5296, + "rewards/accuracies": 0.762499988079071, + "rewards/chosen": -2.906280040740967, + "rewards/margins": 0.9649537205696106, + "rewards/rejected": -3.8712337017059326, + "step": 2110 + }, + { + "epoch": 0.55, + "grad_norm": 14.75, + "learning_rate": 2.4588815314058155e-06, + "logits/chosen": -1.3702977895736694, + "logits/rejected": -1.307600736618042, + "logps/chosen": -536.4708862304688, + "logps/rejected": -595.4000244140625, + "loss": 0.4918, + "rewards/accuracies": 0.7562500238418579, + "rewards/chosen": -2.909069776535034, + "rewards/margins": 0.9321239590644836, + "rewards/rejected": -3.841193675994873, + "step": 2120 + }, + { + "epoch": 0.56, + "grad_norm": 11.9375, + "learning_rate": 2.4360420323899922e-06, + "logits/chosen": -1.4435946941375732, + "logits/rejected": -1.3134028911590576, + "logps/chosen": -554.8719482421875, + "logps/rejected": -603.7503051757812, + "loss": 0.5711, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -2.7905924320220947, + "rewards/margins": 0.8145803213119507, + "rewards/rejected": -3.605172634124756, + "step": 2130 + }, + { + "epoch": 0.56, + "grad_norm": 8.5625, + "learning_rate": 2.4132078738460585e-06, + "logits/chosen": -1.4984867572784424, + "logits/rejected": -1.3388562202453613, + "logps/chosen": -511.22344970703125, + "logps/rejected": -575.4027099609375, + "loss": 0.4819, + "rewards/accuracies": 0.768750011920929, + "rewards/chosen": -2.4675207138061523, + "rewards/margins": 1.027344822883606, + "rewards/rejected": -3.4948654174804688, + "step": 2140 + }, + { + "epoch": 0.56, + "grad_norm": 13.8125, + "learning_rate": 2.3903809624196826e-06, + "logits/chosen": -1.4616576433181763, + "logits/rejected": -1.3179928064346313, + "logps/chosen": -468.075439453125, + "logps/rejected": -520.423095703125, + "loss": 0.5464, + "rewards/accuracies": 0.7124999761581421, + "rewards/chosen": -2.3437161445617676, + "rewards/margins": 0.8483073115348816, + "rewards/rejected": -3.192023515701294, + "step": 2150 + }, + { + "epoch": 0.57, + "grad_norm": 12.375, + "learning_rate": 2.3675632041513978e-06, + "logits/chosen": -1.5807464122772217, + "logits/rejected": -1.3334547281265259, + "logps/chosen": -532.7571411132812, + "logps/rejected": -571.2667846679688, + "loss": 0.4903, + "rewards/accuracies": 0.75, + "rewards/chosen": -2.4575352668762207, + "rewards/margins": 1.0318888425827026, + "rewards/rejected": -3.489424228668213, + "step": 2160 + }, + { + "epoch": 0.57, + "grad_norm": 14.125, + "learning_rate": 2.3447565043174533e-06, + "logits/chosen": -1.42782461643219, + "logits/rejected": -1.2667099237442017, + "logps/chosen": -528.2996826171875, + "logps/rejected": -579.03173828125, + "loss": 0.525, + "rewards/accuracies": 0.6937500238418579, + "rewards/chosen": -2.6998863220214844, + "rewards/margins": 0.8944419026374817, + "rewards/rejected": -3.5943286418914795, + "step": 2170 + }, + { + "epoch": 0.57, + "grad_norm": 9.9375, + "learning_rate": 2.321962767270724e-06, + "logits/chosen": -1.4696300029754639, + "logits/rejected": -1.3142036199569702, + "logps/chosen": -505.57843017578125, + "logps/rejected": -546.5269165039062, + "loss": 0.5521, + "rewards/accuracies": 0.71875, + "rewards/chosen": -2.556256055831909, + "rewards/margins": 0.7811610698699951, + "rewards/rejected": -3.3374171257019043, + "step": 2180 + }, + { + "epoch": 0.57, + "grad_norm": 14.1875, + "learning_rate": 2.299183896281692e-06, + "logits/chosen": -1.4029340744018555, + "logits/rejected": -1.2631093263626099, + "logps/chosen": -490.07208251953125, + "logps/rejected": -572.7850952148438, + "loss": 0.5183, + "rewards/accuracies": 0.731249988079071, + "rewards/chosen": -2.3514010906219482, + "rewards/margins": 0.850308895111084, + "rewards/rejected": -3.2017102241516113, + "step": 2190 + }, + { + "epoch": 0.58, + "grad_norm": 8.125, + "learning_rate": 2.2764217933795297e-06, + "logits/chosen": -1.497179627418518, + "logits/rejected": -1.3680561780929565, + "logps/chosen": -482.985595703125, + "logps/rejected": -569.66552734375, + "loss": 0.4777, + "rewards/accuracies": 0.737500011920929, + "rewards/chosen": -2.2155346870422363, + "rewards/margins": 1.0720819234848022, + "rewards/rejected": -3.2876172065734863, + "step": 2200 + }, + { + "epoch": 0.58, + "eval_logits/chosen": -1.377992033958435, + "eval_logits/rejected": -1.2484394311904907, + "eval_logps/chosen": -498.22637939453125, + "eval_logps/rejected": -575.8182983398438, + "eval_loss": 0.49137604236602783, + "eval_rewards/accuracies": 0.7475000023841858, + "eval_rewards/chosen": -2.3357439041137695, + "eval_rewards/margins": 0.9763532876968384, + "eval_rewards/rejected": -3.3120970726013184, + "eval_runtime": 384.8882, + "eval_samples_per_second": 5.196, + "eval_steps_per_second": 0.65, + "step": 2200 + }, + { + "epoch": 0.58, + "grad_norm": 8.1875, + "learning_rate": 2.2536783591932786e-06, + "logits/chosen": -1.5414104461669922, + "logits/rejected": -1.380723237991333, + "logps/chosen": -521.5967407226562, + "logps/rejected": -590.3738403320312, + "loss": 0.5316, + "rewards/accuracies": 0.7562500238418579, + "rewards/chosen": -2.460937023162842, + "rewards/margins": 0.8691738843917847, + "rewards/rejected": -3.330111026763916, + "step": 2210 + }, + { + "epoch": 0.58, + "grad_norm": 7.71875, + "learning_rate": 2.230955492793149e-06, + "logits/chosen": -1.3435004949569702, + "logits/rejected": -1.285659909248352, + "logps/chosen": -540.8585815429688, + "logps/rejected": -614.8424072265625, + "loss": 0.591, + "rewards/accuracies": 0.737500011920929, + "rewards/chosen": -2.6706583499908447, + "rewards/margins": 0.8687313199043274, + "rewards/rejected": -3.5393898487091064, + "step": 2220 + }, + { + "epoch": 0.58, + "grad_norm": 6.53125, + "learning_rate": 2.208255091531947e-06, + "logits/chosen": -1.3562614917755127, + "logits/rejected": -1.267709493637085, + "logps/chosen": -524.9986572265625, + "logps/rejected": -600.77880859375, + "loss": 0.4914, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -2.480012893676758, + "rewards/margins": 1.107474684715271, + "rewards/rejected": -3.5874874591827393, + "step": 2230 + }, + { + "epoch": 0.59, + "grad_norm": 12.25, + "learning_rate": 2.1855790508866435e-06, + "logits/chosen": -1.4271819591522217, + "logits/rejected": -1.3109095096588135, + "logps/chosen": -548.445556640625, + "logps/rejected": -636.0255126953125, + "loss": 0.4951, + "rewards/accuracies": 0.7437499761581421, + "rewards/chosen": -2.51424241065979, + "rewards/margins": 1.0523946285247803, + "rewards/rejected": -3.5666370391845703, + "step": 2240 + }, + { + "epoch": 0.59, + "grad_norm": 10.0, + "learning_rate": 2.162929264300107e-06, + "logits/chosen": -1.4263137578964233, + "logits/rejected": -1.329329252243042, + "logps/chosen": -499.01947021484375, + "logps/rejected": -603.9342041015625, + "loss": 0.4131, + "rewards/accuracies": 0.793749988079071, + "rewards/chosen": -2.300555944442749, + "rewards/margins": 1.2727267742156982, + "rewards/rejected": -3.5732827186584473, + "step": 2250 + }, + { + "epoch": 0.59, + "grad_norm": 12.0, + "learning_rate": 2.1403076230230006e-06, + "logits/chosen": -1.3517494201660156, + "logits/rejected": -1.2308024168014526, + "logps/chosen": -553.6866455078125, + "logps/rejected": -609.8203125, + "loss": 0.5893, + "rewards/accuracies": 0.706250011920929, + "rewards/chosen": -2.825040817260742, + "rewards/margins": 0.8394268155097961, + "rewards/rejected": -3.6644675731658936, + "step": 2260 + }, + { + "epoch": 0.59, + "grad_norm": 6.90625, + "learning_rate": 2.11771601595586e-06, + "logits/chosen": -1.420016884803772, + "logits/rejected": -1.302247405052185, + "logps/chosen": -558.3585205078125, + "logps/rejected": -605.810791015625, + "loss": 0.5185, + "rewards/accuracies": 0.7562500238418579, + "rewards/chosen": -2.7631030082702637, + "rewards/margins": 0.9972630739212036, + "rewards/rejected": -3.760366439819336, + "step": 2270 + }, + { + "epoch": 0.6, + "grad_norm": 10.1875, + "learning_rate": 2.0951563294913737e-06, + "logits/chosen": -1.4174630641937256, + "logits/rejected": -1.2032339572906494, + "logps/chosen": -525.8551025390625, + "logps/rejected": -591.8328247070312, + "loss": 0.4771, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -2.643219232559204, + "rewards/margins": 0.9647111892700195, + "rewards/rejected": -3.6079304218292236, + "step": 2280 + }, + { + "epoch": 0.6, + "grad_norm": 8.9375, + "learning_rate": 2.0726304473568693e-06, + "logits/chosen": -1.3942341804504395, + "logits/rejected": -1.2679177522659302, + "logps/chosen": -523.4191284179688, + "logps/rejected": -589.2734985351562, + "loss": 0.492, + "rewards/accuracies": 0.75, + "rewards/chosen": -2.6448018550872803, + "rewards/margins": 0.9818207621574402, + "rewards/rejected": -3.6266231536865234, + "step": 2290 + }, + { + "epoch": 0.6, + "grad_norm": 9.1875, + "learning_rate": 2.050140250457023e-06, + "logits/chosen": -1.4893324375152588, + "logits/rejected": -1.2684142589569092, + "logps/chosen": -571.5191650390625, + "logps/rejected": -653.9939575195312, + "loss": 0.4655, + "rewards/accuracies": 0.78125, + "rewards/chosen": -2.9395713806152344, + "rewards/margins": 1.165346384048462, + "rewards/rejected": -4.104917526245117, + "step": 2300 + }, + { + "epoch": 0.6, + "eval_logits/chosen": -1.2897459268569946, + "eval_logits/rejected": -1.1627777814865112, + "eval_logps/chosen": -571.7406616210938, + "eval_logps/rejected": -672.1651000976562, + "eval_loss": 0.4928034543991089, + "eval_rewards/accuracies": 0.7450000047683716, + "eval_rewards/chosen": -3.0708866119384766, + "eval_rewards/margins": 1.2046781778335571, + "eval_rewards/rejected": -4.2755656242370605, + "eval_runtime": 384.9685, + "eval_samples_per_second": 5.195, + "eval_steps_per_second": 0.649, + "step": 2300 + }, + { + "epoch": 0.6, + "grad_norm": 17.75, + "learning_rate": 2.0276876167168042e-06, + "logits/chosen": -1.2386648654937744, + "logits/rejected": -1.1422879695892334, + "logps/chosen": -528.5611572265625, + "logps/rejected": -600.717529296875, + "loss": 0.5928, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -3.040616273880005, + "rewards/margins": 1.0473132133483887, + "rewards/rejected": -4.087929725646973, + "step": 2310 + }, + { + "epoch": 0.61, + "grad_norm": 8.875, + "learning_rate": 2.0052744209246682e-06, + "logits/chosen": -1.3995951414108276, + "logits/rejected": -1.2767788171768188, + "logps/chosen": -552.1273193359375, + "logps/rejected": -622.7492065429688, + "loss": 0.5088, + "rewards/accuracies": 0.737500011920929, + "rewards/chosen": -2.9678916931152344, + "rewards/margins": 1.0349812507629395, + "rewards/rejected": -4.002872943878174, + "step": 2320 + }, + { + "epoch": 0.61, + "grad_norm": 12.1875, + "learning_rate": 1.9829025345760127e-06, + "logits/chosen": -1.405669093132019, + "logits/rejected": -1.3700844049453735, + "logps/chosen": -559.1365966796875, + "logps/rejected": -644.623779296875, + "loss": 0.5386, + "rewards/accuracies": 0.706250011920929, + "rewards/chosen": -2.8013358116149902, + "rewards/margins": 0.9156203269958496, + "rewards/rejected": -3.7169559001922607, + "step": 2330 + }, + { + "epoch": 0.61, + "grad_norm": 9.125, + "learning_rate": 1.9605738257169115e-06, + "logits/chosen": -1.3817434310913086, + "logits/rejected": -1.2056105136871338, + "logps/chosen": -506.63201904296875, + "logps/rejected": -614.6943969726562, + "loss": 0.4856, + "rewards/accuracies": 0.737500011920929, + "rewards/chosen": -2.7365362644195557, + "rewards/margins": 1.1604435443878174, + "rewards/rejected": -3.896979808807373, + "step": 2340 + }, + { + "epoch": 0.62, + "grad_norm": 14.5, + "learning_rate": 1.9382901587881275e-06, + "logits/chosen": -1.4400955438613892, + "logits/rejected": -1.3102456331253052, + "logps/chosen": -521.36767578125, + "logps/rejected": -608.1643676757812, + "loss": 0.4185, + "rewards/accuracies": 0.8062499761581421, + "rewards/chosen": -2.6261146068573, + "rewards/margins": 1.1909650564193726, + "rewards/rejected": -3.817080020904541, + "step": 2350 + }, + { + "epoch": 0.62, + "grad_norm": 11.0625, + "learning_rate": 1.916053394469437e-06, + "logits/chosen": -1.4624320268630981, + "logits/rejected": -1.2485642433166504, + "logps/chosen": -547.3516845703125, + "logps/rejected": -640.5094604492188, + "loss": 0.5228, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -2.8405938148498535, + "rewards/margins": 1.0770297050476074, + "rewards/rejected": -3.917623996734619, + "step": 2360 + }, + { + "epoch": 0.62, + "grad_norm": 9.75, + "learning_rate": 1.8938653895242604e-06, + "logits/chosen": -1.4186649322509766, + "logits/rejected": -1.2279117107391357, + "logps/chosen": -544.6094970703125, + "logps/rejected": -642.7184448242188, + "loss": 0.4335, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": -2.803039073944092, + "rewards/margins": 1.269425630569458, + "rewards/rejected": -4.072464942932129, + "step": 2370 + }, + { + "epoch": 0.62, + "grad_norm": 11.8125, + "learning_rate": 1.8717279966446267e-06, + "logits/chosen": -1.2762877941131592, + "logits/rejected": -1.1935598850250244, + "logps/chosen": -549.9871826171875, + "logps/rejected": -655.0281982421875, + "loss": 0.4571, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -3.014120101928711, + "rewards/margins": 1.1458585262298584, + "rewards/rejected": -4.15997838973999, + "step": 2380 + }, + { + "epoch": 0.63, + "grad_norm": 7.8125, + "learning_rate": 1.8496430642964698e-06, + "logits/chosen": -1.351825475692749, + "logits/rejected": -1.2405725717544556, + "logps/chosen": -560.7874145507812, + "logps/rejected": -648.8487548828125, + "loss": 0.4959, + "rewards/accuracies": 0.7562500238418579, + "rewards/chosen": -2.8837532997131348, + "rewards/margins": 1.1171314716339111, + "rewards/rejected": -4.000885009765625, + "step": 2390 + }, + { + "epoch": 0.63, + "grad_norm": 7.40625, + "learning_rate": 1.827612436565286e-06, + "logits/chosen": -1.359593391418457, + "logits/rejected": -1.2036197185516357, + "logps/chosen": -551.798828125, + "logps/rejected": -647.3968505859375, + "loss": 0.47, + "rewards/accuracies": 0.75, + "rewards/chosen": -2.88834285736084, + "rewards/margins": 1.1616257429122925, + "rewards/rejected": -4.049968242645264, + "step": 2400 + }, + { + "epoch": 0.63, + "eval_logits/chosen": -1.2773113250732422, + "eval_logits/rejected": -1.1517482995986938, + "eval_logps/chosen": -557.9854125976562, + "eval_logps/rejected": -651.6221923828125, + "eval_loss": 0.4908619225025177, + "eval_rewards/accuracies": 0.7409999966621399, + "eval_rewards/chosen": -2.9333345890045166, + "eval_rewards/margins": 1.1368014812469482, + "eval_rewards/rejected": -4.070136070251465, + "eval_runtime": 385.0042, + "eval_samples_per_second": 5.195, + "eval_steps_per_second": 0.649, + "step": 2400 + }, + { + "epoch": 0.63, + "grad_norm": 16.5, + "learning_rate": 1.8056379530021492e-06, + "logits/chosen": -1.4074201583862305, + "logits/rejected": -1.3206876516342163, + "logps/chosen": -535.7772216796875, + "logps/rejected": -602.4798583984375, + "loss": 0.5365, + "rewards/accuracies": 0.7124999761581421, + "rewards/chosen": -2.915788173675537, + "rewards/margins": 0.915139377117157, + "rewards/rejected": -3.830927610397339, + "step": 2410 + }, + { + "epoch": 0.63, + "grad_norm": 8.125, + "learning_rate": 1.7837214484701154e-06, + "logits/chosen": -1.4410103559494019, + "logits/rejected": -1.3098431825637817, + "logps/chosen": -498.352294921875, + "logps/rejected": -587.9461669921875, + "loss": 0.4705, + "rewards/accuracies": 0.8125, + "rewards/chosen": -2.460021495819092, + "rewards/margins": 1.1296519041061401, + "rewards/rejected": -3.5896732807159424, + "step": 2420 + }, + { + "epoch": 0.64, + "grad_norm": 15.25, + "learning_rate": 1.7618647529910043e-06, + "logits/chosen": -1.4378907680511475, + "logits/rejected": -1.3039597272872925, + "logps/chosen": -500.2588806152344, + "logps/rejected": -604.9884033203125, + "loss": 0.4869, + "rewards/accuracies": 0.762499988079071, + "rewards/chosen": -2.434854030609131, + "rewards/margins": 1.166461706161499, + "rewards/rejected": -3.601315975189209, + "step": 2430 + }, + { + "epoch": 0.64, + "grad_norm": 9.6875, + "learning_rate": 1.7400696915925996e-06, + "logits/chosen": -1.442056655883789, + "logits/rejected": -1.2497450113296509, + "logps/chosen": -522.6090698242188, + "logps/rejected": -571.2970581054688, + "loss": 0.5167, + "rewards/accuracies": 0.7437499761581421, + "rewards/chosen": -2.519986391067505, + "rewards/margins": 1.0696182250976562, + "rewards/rejected": -3.589604139328003, + "step": 2440 + }, + { + "epoch": 0.64, + "grad_norm": 11.0, + "learning_rate": 1.718338084156254e-06, + "logits/chosen": -1.4057360887527466, + "logits/rejected": -1.2487547397613525, + "logps/chosen": -521.5921020507812, + "logps/rejected": -594.0106201171875, + "loss": 0.4502, + "rewards/accuracies": 0.78125, + "rewards/chosen": -2.3480098247528076, + "rewards/margins": 1.1080354452133179, + "rewards/rejected": -3.456045627593994, + "step": 2450 + }, + { + "epoch": 0.64, + "grad_norm": 11.3125, + "learning_rate": 1.6966717452649372e-06, + "logits/chosen": -1.5112414360046387, + "logits/rejected": -1.3404825925827026, + "logps/chosen": -511.281005859375, + "logps/rejected": -574.0786743164062, + "loss": 0.4402, + "rewards/accuracies": 0.7437499761581421, + "rewards/chosen": -2.34338641166687, + "rewards/margins": 1.1465994119644165, + "rewards/rejected": -3.489985704421997, + "step": 2460 + }, + { + "epoch": 0.65, + "grad_norm": 8.375, + "learning_rate": 1.6750724840517103e-06, + "logits/chosen": -1.4542334079742432, + "logits/rejected": -1.3746201992034912, + "logps/chosen": -482.7185974121094, + "logps/rejected": -584.7723999023438, + "loss": 0.504, + "rewards/accuracies": 0.75, + "rewards/chosen": -2.2991232872009277, + "rewards/margins": 0.9794435501098633, + "rewards/rejected": -3.278566837310791, + "step": 2470 + }, + { + "epoch": 0.65, + "grad_norm": 12.0625, + "learning_rate": 1.6535421040486686e-06, + "logits/chosen": -1.2772949934005737, + "logits/rejected": -1.177643895149231, + "logps/chosen": -497.01861572265625, + "logps/rejected": -593.6337890625, + "loss": 0.424, + "rewards/accuracies": 0.831250011920929, + "rewards/chosen": -2.4366674423217773, + "rewards/margins": 1.3086668252944946, + "rewards/rejected": -3.7453346252441406, + "step": 2480 + }, + { + "epoch": 0.65, + "grad_norm": 15.5, + "learning_rate": 1.6320824030363458e-06, + "logits/chosen": -1.351653814315796, + "logits/rejected": -1.2928129434585571, + "logps/chosen": -490.3855895996094, + "logps/rejected": -596.65380859375, + "loss": 0.465, + "rewards/accuracies": 0.7562500238418579, + "rewards/chosen": -2.5555453300476074, + "rewards/margins": 1.217242956161499, + "rewards/rejected": -3.7727882862091064, + "step": 2490 + }, + { + "epoch": 0.65, + "grad_norm": 14.5625, + "learning_rate": 1.6106951728936028e-06, + "logits/chosen": -1.4520965814590454, + "logits/rejected": -1.318485975265503, + "logps/chosen": -513.4244995117188, + "logps/rejected": -615.5191040039062, + "loss": 0.4963, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -2.5622081756591797, + "rewards/margins": 1.0996941328048706, + "rewards/rejected": -3.661902666091919, + "step": 2500 + }, + { + "epoch": 0.65, + "eval_logits/chosen": -1.3239047527313232, + "eval_logits/rejected": -1.1944924592971802, + "eval_logps/chosen": -525.2288208007812, + "eval_logps/rejected": -621.9060668945312, + "eval_loss": 0.49328407645225525, + "eval_rewards/accuracies": 0.7390000224113464, + "eval_rewards/chosen": -2.6057679653167725, + "eval_rewards/margins": 1.1672067642211914, + "eval_rewards/rejected": -3.7729744911193848, + "eval_runtime": 384.9389, + "eval_samples_per_second": 5.196, + "eval_steps_per_second": 0.649, + "step": 2500 + }, + { + "epoch": 0.66, + "grad_norm": 10.6875, + "learning_rate": 1.5893821994479996e-06, + "logits/chosen": -1.4442546367645264, + "logits/rejected": -1.3286397457122803, + "logps/chosen": -525.9605102539062, + "logps/rejected": -608.4495239257812, + "loss": 0.475, + "rewards/accuracies": 0.7562500238418579, + "rewards/chosen": -2.4855237007141113, + "rewards/margins": 1.2105185985565186, + "rewards/rejected": -3.696042537689209, + "step": 2510 + }, + { + "epoch": 0.66, + "grad_norm": 7.90625, + "learning_rate": 1.5681452623266868e-06, + "logits/chosen": -1.4192955493927002, + "logits/rejected": -1.174787998199463, + "logps/chosen": -559.93603515625, + "logps/rejected": -632.15234375, + "loss": 0.4838, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -2.7089059352874756, + "rewards/margins": 1.304880976676941, + "rewards/rejected": -4.013787269592285, + "step": 2520 + }, + { + "epoch": 0.66, + "grad_norm": 5.9375, + "learning_rate": 1.5469861348078014e-06, + "logits/chosen": -1.420716643333435, + "logits/rejected": -1.2741087675094604, + "logps/chosen": -522.9864501953125, + "logps/rejected": -641.4890747070312, + "loss": 0.4193, + "rewards/accuracies": 0.768750011920929, + "rewards/chosen": -2.7621614933013916, + "rewards/margins": 1.2914037704467773, + "rewards/rejected": -4.05356502532959, + "step": 2530 + }, + { + "epoch": 0.66, + "grad_norm": 9.4375, + "learning_rate": 1.5259065836724035e-06, + "logits/chosen": -1.2834079265594482, + "logits/rejected": -1.2232940196990967, + "logps/chosen": -522.1956176757812, + "logps/rejected": -655.6646728515625, + "loss": 0.4116, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -2.773864269256592, + "rewards/margins": 1.3960521221160889, + "rewards/rejected": -4.16991662979126, + "step": 2540 + }, + { + "epoch": 0.67, + "grad_norm": 19.5, + "learning_rate": 1.5049083690569456e-06, + "logits/chosen": -1.3350251913070679, + "logits/rejected": -1.2278568744659424, + "logps/chosen": -531.40380859375, + "logps/rejected": -651.0161743164062, + "loss": 0.5176, + "rewards/accuracies": 0.6937500238418579, + "rewards/chosen": -2.939335584640503, + "rewards/margins": 1.2402127981185913, + "rewards/rejected": -4.179548740386963, + "step": 2550 + }, + { + "epoch": 0.67, + "grad_norm": 14.0625, + "learning_rate": 1.4839932443063057e-06, + "logits/chosen": -1.339280366897583, + "logits/rejected": -1.1727148294448853, + "logps/chosen": -578.1463623046875, + "logps/rejected": -648.4544677734375, + "loss": 0.4648, + "rewards/accuracies": 0.78125, + "rewards/chosen": -2.86167573928833, + "rewards/margins": 1.2659341096878052, + "rewards/rejected": -4.127610206604004, + "step": 2560 + }, + { + "epoch": 0.67, + "grad_norm": 22.875, + "learning_rate": 1.4631629558273803e-06, + "logits/chosen": -1.3597378730773926, + "logits/rejected": -1.2507550716400146, + "logps/chosen": -532.3685302734375, + "logps/rejected": -612.3403930664062, + "loss": 0.6361, + "rewards/accuracies": 0.668749988079071, + "rewards/chosen": -2.9064340591430664, + "rewards/margins": 0.9161791801452637, + "rewards/rejected": -3.822613477706909, + "step": 2570 + }, + { + "epoch": 0.68, + "grad_norm": 5.09375, + "learning_rate": 1.4424192429432657e-06, + "logits/chosen": -1.4379812479019165, + "logits/rejected": -1.3553606271743774, + "logps/chosen": -497.770263671875, + "logps/rejected": -619.8333129882812, + "loss": 0.4692, + "rewards/accuracies": 0.7124999761581421, + "rewards/chosen": -2.4189367294311523, + "rewards/margins": 1.1874587535858154, + "rewards/rejected": -3.606395721435547, + "step": 2580 + }, + { + "epoch": 0.68, + "grad_norm": 14.9375, + "learning_rate": 1.421763837748016e-06, + "logits/chosen": -1.4066466093063354, + "logits/rejected": -1.3101098537445068, + "logps/chosen": -501.1058654785156, + "logps/rejected": -613.1395263671875, + "loss": 0.4538, + "rewards/accuracies": 0.7875000238418579, + "rewards/chosen": -2.5047993659973145, + "rewards/margins": 1.2053807973861694, + "rewards/rejected": -3.7101802825927734, + "step": 2590 + }, + { + "epoch": 0.68, + "grad_norm": 10.125, + "learning_rate": 1.401198464962021e-06, + "logits/chosen": -1.4462471008300781, + "logits/rejected": -1.2622243165969849, + "logps/chosen": -535.8546142578125, + "logps/rejected": -603.2532348632812, + "loss": 0.4663, + "rewards/accuracies": 0.762499988079071, + "rewards/chosen": -2.672712564468384, + "rewards/margins": 1.0602437257766724, + "rewards/rejected": -3.7329559326171875, + "step": 2600 + }, + { + "epoch": 0.68, + "eval_logits/chosen": -1.3264377117156982, + "eval_logits/rejected": -1.1991208791732788, + "eval_logps/chosen": -532.6129760742188, + "eval_logps/rejected": -628.5565795898438, + "eval_loss": 0.4950037896633148, + "eval_rewards/accuracies": 0.7450000047683716, + "eval_rewards/chosen": -2.67961049079895, + "eval_rewards/margins": 1.1598690748214722, + "eval_rewards/rejected": -3.839479446411133, + "eval_runtime": 384.8048, + "eval_samples_per_second": 5.197, + "eval_steps_per_second": 0.65, + "step": 2600 + }, + { + "epoch": 0.68, + "grad_norm": 10.9375, + "learning_rate": 1.3807248417879896e-06, + "logits/chosen": -1.4817050695419312, + "logits/rejected": -1.368260145187378, + "logps/chosen": -536.787109375, + "logps/rejected": -646.0281982421875, + "loss": 0.4362, + "rewards/accuracies": 0.8062499761581421, + "rewards/chosen": -2.6372110843658447, + "rewards/margins": 1.2936350107192993, + "rewards/rejected": -3.9308464527130127, + "step": 2610 + }, + { + "epoch": 0.69, + "grad_norm": 22.75, + "learning_rate": 1.3603446777675665e-06, + "logits/chosen": -1.3277854919433594, + "logits/rejected": -1.207275629043579, + "logps/chosen": -556.1530151367188, + "logps/rejected": -648.6631469726562, + "loss": 0.5338, + "rewards/accuracies": 0.75, + "rewards/chosen": -2.9295554161071777, + "rewards/margins": 1.1459153890609741, + "rewards/rejected": -4.075470924377441, + "step": 2620 + }, + { + "epoch": 0.69, + "grad_norm": 8.3125, + "learning_rate": 1.3400596746385817e-06, + "logits/chosen": -1.4622247219085693, + "logits/rejected": -1.293874979019165, + "logps/chosen": -548.2469482421875, + "logps/rejected": -635.0680541992188, + "loss": 0.5024, + "rewards/accuracies": 0.768750011920929, + "rewards/chosen": -2.7906343936920166, + "rewards/margins": 1.13905930519104, + "rewards/rejected": -3.9296936988830566, + "step": 2630 + }, + { + "epoch": 0.69, + "grad_norm": 9.125, + "learning_rate": 1.3198715261929587e-06, + "logits/chosen": -1.4278955459594727, + "logits/rejected": -1.2781603336334229, + "logps/chosen": -530.7630615234375, + "logps/rejected": -637.7789306640625, + "loss": 0.4257, + "rewards/accuracies": 0.768750011920929, + "rewards/chosen": -2.943067789077759, + "rewards/margins": 1.2065026760101318, + "rewards/rejected": -4.149571418762207, + "step": 2640 + }, + { + "epoch": 0.69, + "grad_norm": 8.1875, + "learning_rate": 1.2997819181352823e-06, + "logits/chosen": -1.4591927528381348, + "logits/rejected": -1.3012304306030273, + "logps/chosen": -570.8213500976562, + "logps/rejected": -686.0819091796875, + "loss": 0.432, + "rewards/accuracies": 0.7875000238418579, + "rewards/chosen": -2.7398808002471924, + "rewards/margins": 1.3976715803146362, + "rewards/rejected": -4.137551784515381, + "step": 2650 + }, + { + "epoch": 0.7, + "grad_norm": 21.375, + "learning_rate": 1.2797925279420454e-06, + "logits/chosen": -1.4334865808486938, + "logits/rejected": -1.2914705276489258, + "logps/chosen": -569.4708862304688, + "logps/rejected": -681.2939453125, + "loss": 0.4801, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -2.9758732318878174, + "rewards/margins": 1.2384196519851685, + "rewards/rejected": -4.214293003082275, + "step": 2660 + }, + { + "epoch": 0.7, + "grad_norm": 13.625, + "learning_rate": 1.2599050247215764e-06, + "logits/chosen": -1.3719279766082764, + "logits/rejected": -1.2636692523956299, + "logps/chosen": -553.9847412109375, + "logps/rejected": -650.590576171875, + "loss": 0.4906, + "rewards/accuracies": 0.8062499761581421, + "rewards/chosen": -2.9605300426483154, + "rewards/margins": 1.196462869644165, + "rewards/rejected": -4.1569929122924805, + "step": 2670 + }, + { + "epoch": 0.7, + "grad_norm": 13.5625, + "learning_rate": 1.2401210690746705e-06, + "logits/chosen": -1.407362699508667, + "logits/rejected": -1.252087950706482, + "logps/chosen": -551.7230834960938, + "logps/rejected": -632.700439453125, + "loss": 0.4983, + "rewards/accuracies": 0.7562500238418579, + "rewards/chosen": -2.845663547515869, + "rewards/margins": 1.1418297290802002, + "rewards/rejected": -3.9874930381774902, + "step": 2680 + }, + { + "epoch": 0.7, + "grad_norm": 13.0, + "learning_rate": 1.2204423129559306e-06, + "logits/chosen": -1.4424464702606201, + "logits/rejected": -1.3769545555114746, + "logps/chosen": -529.952880859375, + "logps/rejected": -642.1946411132812, + "loss": 0.5035, + "rewards/accuracies": 0.75, + "rewards/chosen": -2.6443989276885986, + "rewards/margins": 1.1635662317276, + "rewards/rejected": -3.807965040206909, + "step": 2690 + }, + { + "epoch": 0.71, + "grad_norm": 18.625, + "learning_rate": 1.20087039953583e-06, + "logits/chosen": -1.4672467708587646, + "logits/rejected": -1.3367975950241089, + "logps/chosen": -524.6241455078125, + "logps/rejected": -619.1414794921875, + "loss": 0.5286, + "rewards/accuracies": 0.768750011920929, + "rewards/chosen": -2.604055881500244, + "rewards/margins": 1.2088748216629028, + "rewards/rejected": -3.8129310607910156, + "step": 2700 + }, + { + "epoch": 0.71, + "eval_logits/chosen": -1.330871820449829, + "eval_logits/rejected": -1.2033063173294067, + "eval_logps/chosen": -528.7828979492188, + "eval_logps/rejected": -622.6273193359375, + "eval_loss": 0.49607598781585693, + "eval_rewards/accuracies": 0.7379999756813049, + "eval_rewards/chosen": -2.6413092613220215, + "eval_rewards/margins": 1.1388777494430542, + "eval_rewards/rejected": -3.7801873683929443, + "eval_runtime": 384.8793, + "eval_samples_per_second": 5.196, + "eval_steps_per_second": 0.65, + "step": 2700 + }, + { + "epoch": 0.71, + "grad_norm": 12.1875, + "learning_rate": 1.181406963063507e-06, + "logits/chosen": -1.3730335235595703, + "logits/rejected": -1.3149739503860474, + "logps/chosen": -519.0841064453125, + "logps/rejected": -629.9993896484375, + "loss": 0.5014, + "rewards/accuracies": 0.731249988079071, + "rewards/chosen": -2.532372236251831, + "rewards/margins": 1.1152244806289673, + "rewards/rejected": -3.647596836090088, + "step": 2710 + }, + { + "epoch": 0.71, + "grad_norm": 5.875, + "learning_rate": 1.1620536287303052e-06, + "logits/chosen": -1.4739983081817627, + "logits/rejected": -1.3339478969573975, + "logps/chosen": -543.446533203125, + "logps/rejected": -608.2566528320312, + "loss": 0.5447, + "rewards/accuracies": 0.7562500238418579, + "rewards/chosen": -2.522928476333618, + "rewards/margins": 0.9496552348136902, + "rewards/rejected": -3.472583770751953, + "step": 2720 + }, + { + "epoch": 0.71, + "grad_norm": 10.5, + "learning_rate": 1.1428120125340717e-06, + "logits/chosen": -1.4052727222442627, + "logits/rejected": -1.2576491832733154, + "logps/chosen": -496.2265625, + "logps/rejected": -606.283203125, + "loss": 0.4235, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -2.4208767414093018, + "rewards/margins": 1.5151195526123047, + "rewards/rejected": -3.9359962940216064, + "step": 2730 + }, + { + "epoch": 0.72, + "grad_norm": 10.4375, + "learning_rate": 1.123683721144223e-06, + "logits/chosen": -1.4060310125350952, + "logits/rejected": -1.3006138801574707, + "logps/chosen": -531.5321655273438, + "logps/rejected": -636.8844604492188, + "loss": 0.4255, + "rewards/accuracies": 0.762499988079071, + "rewards/chosen": -2.4785878658294678, + "rewards/margins": 1.376922845840454, + "rewards/rejected": -3.85551118850708, + "step": 2740 + }, + { + "epoch": 0.72, + "grad_norm": 7.375, + "learning_rate": 1.1046703517675848e-06, + "logits/chosen": -1.4380762577056885, + "logits/rejected": -1.3517663478851318, + "logps/chosen": -497.983154296875, + "logps/rejected": -609.3760986328125, + "loss": 0.5127, + "rewards/accuracies": 0.737500011920929, + "rewards/chosen": -2.428668975830078, + "rewards/margins": 1.0526988506317139, + "rewards/rejected": -3.481367588043213, + "step": 2750 + }, + { + "epoch": 0.72, + "grad_norm": 11.3125, + "learning_rate": 1.085773492015028e-06, + "logits/chosen": -1.422131896018982, + "logits/rejected": -1.2450854778289795, + "logps/chosen": -488.3145446777344, + "logps/rejected": -581.1353149414062, + "loss": 0.4374, + "rewards/accuracies": 0.78125, + "rewards/chosen": -2.4193508625030518, + "rewards/margins": 1.271308183670044, + "rewards/rejected": -3.6906590461730957, + "step": 2760 + }, + { + "epoch": 0.72, + "grad_norm": 11.1875, + "learning_rate": 1.0669947197689034e-06, + "logits/chosen": -1.3928449153900146, + "logits/rejected": -1.2680118083953857, + "logps/chosen": -527.7369995117188, + "logps/rejected": -608.68408203125, + "loss": 0.4956, + "rewards/accuracies": 0.7562500238418579, + "rewards/chosen": -2.5510122776031494, + "rewards/margins": 1.0964926481246948, + "rewards/rejected": -3.6475048065185547, + "step": 2770 + }, + { + "epoch": 0.73, + "grad_norm": 10.5, + "learning_rate": 1.048335603051291e-06, + "logits/chosen": -1.3895783424377441, + "logits/rejected": -1.2499104738235474, + "logps/chosen": -566.0472412109375, + "logps/rejected": -672.4845581054688, + "loss": 0.4484, + "rewards/accuracies": 0.7875000238418579, + "rewards/chosen": -2.7824342250823975, + "rewards/margins": 1.3463845252990723, + "rewards/rejected": -4.128818511962891, + "step": 2780 + }, + { + "epoch": 0.73, + "grad_norm": 19.25, + "learning_rate": 1.0297976998930665e-06, + "logits/chosen": -1.3868653774261475, + "logits/rejected": -1.2727091312408447, + "logps/chosen": -521.1253662109375, + "logps/rejected": -633.4533081054688, + "loss": 0.44, + "rewards/accuracies": 0.768750011920929, + "rewards/chosen": -2.605109453201294, + "rewards/margins": 1.4010969400405884, + "rewards/rejected": -4.006206035614014, + "step": 2790 + }, + { + "epoch": 0.73, + "grad_norm": 9.4375, + "learning_rate": 1.0113825582038078e-06, + "logits/chosen": -1.4213166236877441, + "logits/rejected": -1.304223656654358, + "logps/chosen": -538.0697021484375, + "logps/rejected": -635.2910766601562, + "loss": 0.4564, + "rewards/accuracies": 0.78125, + "rewards/chosen": -2.727999210357666, + "rewards/margins": 1.1396801471710205, + "rewards/rejected": -3.8676788806915283, + "step": 2800 + }, + { + "epoch": 0.73, + "eval_logits/chosen": -1.3305258750915527, + "eval_logits/rejected": -1.2037560939788818, + "eval_logps/chosen": -532.7353515625, + "eval_logps/rejected": -627.1752319335938, + "eval_loss": 0.49248310923576355, + "eval_rewards/accuracies": 0.7404999732971191, + "eval_rewards/chosen": -2.680833578109741, + "eval_rewards/margins": 1.1448326110839844, + "eval_rewards/rejected": -3.8256664276123047, + "eval_runtime": 384.8109, + "eval_samples_per_second": 5.197, + "eval_steps_per_second": 0.65, + "step": 2800 + }, + { + "epoch": 0.74, + "grad_norm": 10.5, + "learning_rate": 9.930917156425477e-07, + "logits/chosen": -1.4014348983764648, + "logits/rejected": -1.2874269485473633, + "logps/chosen": -547.7164306640625, + "logps/rejected": -650.8362426757812, + "loss": 0.5363, + "rewards/accuracies": 0.768750011920929, + "rewards/chosen": -2.8659284114837646, + "rewards/margins": 1.0600517988204956, + "rewards/rejected": -3.9259800910949707, + "step": 2810 + }, + { + "epoch": 0.74, + "grad_norm": 23.5, + "learning_rate": 9.749266994893756e-07, + "logits/chosen": -1.3308923244476318, + "logits/rejected": -1.20591139793396, + "logps/chosen": -509.127685546875, + "logps/rejected": -585.4031982421875, + "loss": 0.5619, + "rewards/accuracies": 0.7437499761581421, + "rewards/chosen": -2.684295177459717, + "rewards/margins": 0.8970636129379272, + "rewards/rejected": -3.5813584327697754, + "step": 2820 + }, + { + "epoch": 0.74, + "grad_norm": 12.0625, + "learning_rate": 9.56889026517913e-07, + "logits/chosen": -1.388494610786438, + "logits/rejected": -1.2768661975860596, + "logps/chosen": -543.159423828125, + "logps/rejected": -621.0648193359375, + "loss": 0.5044, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -2.818211078643799, + "rewards/margins": 1.039044737815857, + "rewards/rejected": -3.857255458831787, + "step": 2830 + }, + { + "epoch": 0.74, + "grad_norm": 6.90625, + "learning_rate": 9.389802028686617e-07, + "logits/chosen": -1.4692569971084595, + "logits/rejected": -1.3720782995224, + "logps/chosen": -529.861328125, + "logps/rejected": -575.4440307617188, + "loss": 0.5904, + "rewards/accuracies": 0.71875, + "rewards/chosen": -2.690932035446167, + "rewards/margins": 0.7855546474456787, + "rewards/rejected": -3.4764866828918457, + "step": 2840 + }, + { + "epoch": 0.75, + "grad_norm": 10.5, + "learning_rate": 9.212017239232427e-07, + "logits/chosen": -1.4099429845809937, + "logits/rejected": -1.2403053045272827, + "logps/chosen": -530.2276611328125, + "logps/rejected": -631.5889892578125, + "loss": 0.4731, + "rewards/accuracies": 0.78125, + "rewards/chosen": -2.5727314949035645, + "rewards/margins": 1.2429524660110474, + "rewards/rejected": -3.8156840801239014, + "step": 2850 + }, + { + "epoch": 0.75, + "grad_norm": 9.625, + "learning_rate": 9.03555074179533e-07, + "logits/chosen": -1.3713786602020264, + "logits/rejected": -1.3497127294540405, + "logps/chosen": -513.812744140625, + "logps/rejected": -644.496826171875, + "loss": 0.4514, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -2.570328950881958, + "rewards/margins": 1.2485467195510864, + "rewards/rejected": -3.818875551223755, + "step": 2860 + }, + { + "epoch": 0.75, + "grad_norm": 20.0, + "learning_rate": 8.860417271277067e-07, + "logits/chosen": -1.4884029626846313, + "logits/rejected": -1.4525179862976074, + "logps/chosen": -530.30712890625, + "logps/rejected": -616.304931640625, + "loss": 0.4918, + "rewards/accuracies": 0.7562500238418579, + "rewards/chosen": -2.6107208728790283, + "rewards/margins": 0.9398597478866577, + "rewards/rejected": -3.5505805015563965, + "step": 2870 + }, + { + "epoch": 0.75, + "grad_norm": 8.25, + "learning_rate": 8.686631451272029e-07, + "logits/chosen": -1.4650144577026367, + "logits/rejected": -1.3026695251464844, + "logps/chosen": -529.6368408203125, + "logps/rejected": -623.4691162109375, + "loss": 0.4882, + "rewards/accuracies": 0.7124999761581421, + "rewards/chosen": -2.762089252471924, + "rewards/margins": 1.1926974058151245, + "rewards/rejected": -3.954786777496338, + "step": 2880 + }, + { + "epoch": 0.76, + "grad_norm": 7.03125, + "learning_rate": 8.514207792846168e-07, + "logits/chosen": -1.4712326526641846, + "logits/rejected": -1.3452935218811035, + "logps/chosen": -522.875244140625, + "logps/rejected": -606.8411254882812, + "loss": 0.5049, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -2.720153570175171, + "rewards/margins": 1.1239979267120361, + "rewards/rejected": -3.844151735305786, + "step": 2890 + }, + { + "epoch": 0.76, + "grad_norm": 9.6875, + "learning_rate": 8.343160693325356e-07, + "logits/chosen": -1.3627361059188843, + "logits/rejected": -1.2458826303482056, + "logps/chosen": -535.0531005859375, + "logps/rejected": -643.9103393554688, + "loss": 0.5166, + "rewards/accuracies": 0.7124999761581421, + "rewards/chosen": -2.795009136199951, + "rewards/margins": 1.1317135095596313, + "rewards/rejected": -3.926722764968872, + "step": 2900 + }, + { + "epoch": 0.76, + "eval_logits/chosen": -1.3309776782989502, + "eval_logits/rejected": -1.2046185731887817, + "eval_logps/chosen": -542.6776733398438, + "eval_logps/rejected": -634.599365234375, + "eval_loss": 0.4903542995452881, + "eval_rewards/accuracies": 0.7415000200271606, + "eval_rewards/chosen": -2.7802560329437256, + "eval_rewards/margins": 1.1196515560150146, + "eval_rewards/rejected": -3.8999080657958984, + "eval_runtime": 384.9371, + "eval_samples_per_second": 5.196, + "eval_steps_per_second": 0.649, + "step": 2900 + }, + { + "epoch": 0.76, + "grad_norm": 7.65625, + "learning_rate": 8.173504435093174e-07, + "logits/chosen": -1.3742562532424927, + "logits/rejected": -1.193378210067749, + "logps/chosen": -514.3562622070312, + "logps/rejected": -601.44091796875, + "loss": 0.4823, + "rewards/accuracies": 0.75, + "rewards/chosen": -2.727877378463745, + "rewards/margins": 1.1920711994171143, + "rewards/rejected": -3.9199485778808594, + "step": 2910 + }, + { + "epoch": 0.76, + "grad_norm": 6.6875, + "learning_rate": 8.00525318439836e-07, + "logits/chosen": -1.4089447259902954, + "logits/rejected": -1.2591418027877808, + "logps/chosen": -555.609375, + "logps/rejected": -642.9563598632812, + "loss": 0.5406, + "rewards/accuracies": 0.7562500238418579, + "rewards/chosen": -2.7878005504608154, + "rewards/margins": 0.9498197436332703, + "rewards/rejected": -3.7376205921173096, + "step": 2920 + }, + { + "epoch": 0.77, + "grad_norm": 8.625, + "learning_rate": 7.838420990171927e-07, + "logits/chosen": -1.4908835887908936, + "logits/rejected": -1.3264166116714478, + "logps/chosen": -539.0242919921875, + "logps/rejected": -614.5499267578125, + "loss": 0.5213, + "rewards/accuracies": 0.737500011920929, + "rewards/chosen": -2.6965396404266357, + "rewards/margins": 1.0111539363861084, + "rewards/rejected": -3.7076938152313232, + "step": 2930 + }, + { + "epoch": 0.77, + "grad_norm": 7.90625, + "learning_rate": 7.673021782854084e-07, + "logits/chosen": -1.3615717887878418, + "logits/rejected": -1.2126728296279907, + "logps/chosen": -533.7975463867188, + "logps/rejected": -619.9442138671875, + "loss": 0.4664, + "rewards/accuracies": 0.7437499761581421, + "rewards/chosen": -2.7310891151428223, + "rewards/margins": 1.2802739143371582, + "rewards/rejected": -4.0113630294799805, + "step": 2940 + }, + { + "epoch": 0.77, + "grad_norm": 8.0625, + "learning_rate": 7.509069373231039e-07, + "logits/chosen": -1.364745855331421, + "logits/rejected": -1.2478384971618652, + "logps/chosen": -535.0496215820312, + "logps/rejected": -594.5074462890625, + "loss": 0.5768, + "rewards/accuracies": 0.731249988079071, + "rewards/chosen": -2.823659658432007, + "rewards/margins": 0.8424208760261536, + "rewards/rejected": -3.666080951690674, + "step": 2950 + }, + { + "epoch": 0.77, + "grad_norm": 7.25, + "learning_rate": 7.346577451281822e-07, + "logits/chosen": -1.3784279823303223, + "logits/rejected": -1.2886309623718262, + "logps/chosen": -533.7701416015625, + "logps/rejected": -641.3207397460938, + "loss": 0.4516, + "rewards/accuracies": 0.762499988079071, + "rewards/chosen": -2.715505361557007, + "rewards/margins": 1.3165885210037231, + "rewards/rejected": -4.032094478607178, + "step": 2960 + }, + { + "epoch": 0.78, + "grad_norm": 22.0, + "learning_rate": 7.185559585035138e-07, + "logits/chosen": -1.413137674331665, + "logits/rejected": -1.2511926889419556, + "logps/chosen": -575.2888793945312, + "logps/rejected": -671.4484252929688, + "loss": 0.4863, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -2.9251255989074707, + "rewards/margins": 1.120355248451233, + "rewards/rejected": -4.045480728149414, + "step": 2970 + }, + { + "epoch": 0.78, + "grad_norm": 9.25, + "learning_rate": 7.026029219436504e-07, + "logits/chosen": -1.4352123737335205, + "logits/rejected": -1.2644484043121338, + "logps/chosen": -532.7044677734375, + "logps/rejected": -639.6867065429688, + "loss": 0.479, + "rewards/accuracies": 0.7562500238418579, + "rewards/chosen": -2.804140090942383, + "rewards/margins": 1.197808861732483, + "rewards/rejected": -4.001949310302734, + "step": 2980 + }, + { + "epoch": 0.78, + "grad_norm": 6.46875, + "learning_rate": 6.867999675225523e-07, + "logits/chosen": -1.4778783321380615, + "logits/rejected": -1.3396053314208984, + "logps/chosen": -498.43896484375, + "logps/rejected": -601.5620727539062, + "loss": 0.4659, + "rewards/accuracies": 0.7562500238418579, + "rewards/chosen": -2.675211191177368, + "rewards/margins": 1.1733391284942627, + "rewards/rejected": -3.8485500812530518, + "step": 2990 + }, + { + "epoch": 0.79, + "grad_norm": 9.75, + "learning_rate": 6.711484147823663e-07, + "logits/chosen": -1.3854588270187378, + "logits/rejected": -1.3027629852294922, + "logps/chosen": -500.9271545410156, + "logps/rejected": -624.5755615234375, + "loss": 0.4653, + "rewards/accuracies": 0.75, + "rewards/chosen": -2.667020797729492, + "rewards/margins": 1.1879808902740479, + "rewards/rejected": -3.855001449584961, + "step": 3000 + }, + { + "epoch": 0.79, + "eval_logits/chosen": -1.3332931995391846, + "eval_logits/rejected": -1.2066706418991089, + "eval_logps/chosen": -544.357421875, + "eval_logps/rejected": -633.0811157226562, + "eval_loss": 0.48963090777397156, + "eval_rewards/accuracies": 0.7425000071525574, + "eval_rewards/chosen": -2.7970540523529053, + "eval_rewards/margins": 1.0876713991165161, + "eval_rewards/rejected": -3.884725332260132, + "eval_runtime": 384.4736, + "eval_samples_per_second": 5.202, + "eval_steps_per_second": 0.65, + "step": 3000 + }, + { + "epoch": 0.79, + "grad_norm": 12.6875, + "learning_rate": 6.556495706232413e-07, + "logits/chosen": -1.3909966945648193, + "logits/rejected": -1.2924126386642456, + "logps/chosen": -558.5319213867188, + "logps/rejected": -642.2532958984375, + "loss": 0.5411, + "rewards/accuracies": 0.731249988079071, + "rewards/chosen": -2.8983654975891113, + "rewards/margins": 1.0214240550994873, + "rewards/rejected": -3.9197897911071777, + "step": 3010 + }, + { + "epoch": 0.79, + "grad_norm": 9.75, + "learning_rate": 6.403047291942057e-07, + "logits/chosen": -1.316248893737793, + "logits/rejected": -1.1616556644439697, + "logps/chosen": -507.49603271484375, + "logps/rejected": -592.3514404296875, + "loss": 0.4966, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -2.8122828006744385, + "rewards/margins": 1.092459797859192, + "rewards/rejected": -3.904742479324341, + "step": 3020 + }, + { + "epoch": 0.79, + "grad_norm": 13.1875, + "learning_rate": 6.251151717851023e-07, + "logits/chosen": -1.3809759616851807, + "logits/rejected": -1.300438642501831, + "logps/chosen": -501.9111328125, + "logps/rejected": -600.9503784179688, + "loss": 0.4907, + "rewards/accuracies": 0.7437499761581421, + "rewards/chosen": -2.685105323791504, + "rewards/margins": 1.123300313949585, + "rewards/rejected": -3.8084053993225098, + "step": 3030 + }, + { + "epoch": 0.8, + "grad_norm": 5.96875, + "learning_rate": 6.100821667196041e-07, + "logits/chosen": -1.5608792304992676, + "logits/rejected": -1.2841638326644897, + "logps/chosen": -542.8932495117188, + "logps/rejected": -584.8834228515625, + "loss": 0.4858, + "rewards/accuracies": 0.762499988079071, + "rewards/chosen": -2.672549247741699, + "rewards/margins": 1.0747697353363037, + "rewards/rejected": -3.747319459915161, + "step": 3040 + }, + { + "epoch": 0.8, + "grad_norm": 96.5, + "learning_rate": 5.952069692493062e-07, + "logits/chosen": -1.364072561264038, + "logits/rejected": -1.2446839809417725, + "logps/chosen": -493.5611267089844, + "logps/rejected": -623.0722045898438, + "loss": 0.4197, + "rewards/accuracies": 0.8062499761581421, + "rewards/chosen": -2.652458906173706, + "rewards/margins": 1.2607505321502686, + "rewards/rejected": -3.9132094383239746, + "step": 3050 + }, + { + "epoch": 0.8, + "grad_norm": 9.25, + "learning_rate": 5.80490821448918e-07, + "logits/chosen": -1.3073358535766602, + "logits/rejected": -1.3144903182983398, + "logps/chosen": -533.8553466796875, + "logps/rejected": -709.3834838867188, + "loss": 0.4358, + "rewards/accuracies": 0.78125, + "rewards/chosen": -2.7133326530456543, + "rewards/margins": 1.3306279182434082, + "rewards/rejected": -4.043961048126221, + "step": 3060 + }, + { + "epoch": 0.8, + "grad_norm": 9.0625, + "learning_rate": 5.659349521125459e-07, + "logits/chosen": -1.5068944692611694, + "logits/rejected": -1.4476134777069092, + "logps/chosen": -550.5355224609375, + "logps/rejected": -633.8616943359375, + "loss": 0.5084, + "rewards/accuracies": 0.7437499761581421, + "rewards/chosen": -2.6911702156066895, + "rewards/margins": 1.0392811298370361, + "rewards/rejected": -3.7304511070251465, + "step": 3070 + }, + { + "epoch": 0.81, + "grad_norm": 8.0, + "learning_rate": 5.5154057665109e-07, + "logits/chosen": -1.4646342992782593, + "logits/rejected": -1.306074857711792, + "logps/chosen": -547.6588745117188, + "logps/rejected": -648.3570556640625, + "loss": 0.4836, + "rewards/accuracies": 0.768750011920929, + "rewards/chosen": -2.8676917552948, + "rewards/margins": 1.2712510824203491, + "rewards/rejected": -4.138943672180176, + "step": 3080 + }, + { + "epoch": 0.81, + "grad_norm": 8.3125, + "learning_rate": 5.373088969907586e-07, + "logits/chosen": -1.4882913827896118, + "logits/rejected": -1.3172584772109985, + "logps/chosen": -559.1968994140625, + "logps/rejected": -619.5064697265625, + "loss": 0.4568, + "rewards/accuracies": 0.768750011920929, + "rewards/chosen": -2.8238325119018555, + "rewards/margins": 1.1052324771881104, + "rewards/rejected": -3.929064989089966, + "step": 3090 + }, + { + "epoch": 0.81, + "grad_norm": 6.53125, + "learning_rate": 5.23241101472709e-07, + "logits/chosen": -1.4091695547103882, + "logits/rejected": -1.2783794403076172, + "logps/chosen": -545.93359375, + "logps/rejected": -631.8021240234375, + "loss": 0.4808, + "rewards/accuracies": 0.8187500238418579, + "rewards/chosen": -2.696319580078125, + "rewards/margins": 1.0787622928619385, + "rewards/rejected": -3.7750816345214844, + "step": 3100 + }, + { + "epoch": 0.81, + "eval_logits/chosen": -1.32783842086792, + "eval_logits/rejected": -1.2009104490280151, + "eval_logps/chosen": -546.65625, + "eval_logps/rejected": -639.3413696289062, + "eval_loss": 0.4900914132595062, + "eval_rewards/accuracies": 0.7409999966621399, + "eval_rewards/chosen": -2.820042133331299, + "eval_rewards/margins": 1.1272854804992676, + "eval_rewards/rejected": -3.9473278522491455, + "eval_runtime": 384.996, + "eval_samples_per_second": 5.195, + "eval_steps_per_second": 0.649, + "step": 3100 + }, + { + "epoch": 0.81, + "grad_norm": 8.625, + "learning_rate": 5.09338364753818e-07, + "logits/chosen": -1.4804376363754272, + "logits/rejected": -1.3111730813980103, + "logps/chosen": -562.015625, + "logps/rejected": -653.310546875, + "loss": 0.5286, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -2.7659342288970947, + "rewards/margins": 1.0708248615264893, + "rewards/rejected": -3.836758852005005, + "step": 3110 + }, + { + "epoch": 0.82, + "grad_norm": 8.8125, + "learning_rate": 4.956018477086005e-07, + "logits/chosen": -1.444879174232483, + "logits/rejected": -1.2762501239776611, + "logps/chosen": -557.318115234375, + "logps/rejected": -641.5279541015625, + "loss": 0.5268, + "rewards/accuracies": 0.6937500238418579, + "rewards/chosen": -2.881852626800537, + "rewards/margins": 1.1156418323516846, + "rewards/rejected": -3.9974944591522217, + "step": 3120 + }, + { + "epoch": 0.82, + "grad_norm": 10.8125, + "learning_rate": 4.820326973322764e-07, + "logits/chosen": -1.3559176921844482, + "logits/rejected": -1.2783609628677368, + "logps/chosen": -545.87255859375, + "logps/rejected": -644.6527099609375, + "loss": 0.5488, + "rewards/accuracies": 0.75, + "rewards/chosen": -2.9490904808044434, + "rewards/margins": 1.0479673147201538, + "rewards/rejected": -3.997058153152466, + "step": 3130 + }, + { + "epoch": 0.82, + "grad_norm": 16.0, + "learning_rate": 4.686320466449981e-07, + "logits/chosen": -1.3531897068023682, + "logits/rejected": -1.1676143407821655, + "logps/chosen": -509.6865234375, + "logps/rejected": -646.2823486328125, + "loss": 0.4513, + "rewards/accuracies": 0.8062499761581421, + "rewards/chosen": -2.704374313354492, + "rewards/margins": 1.3875735998153687, + "rewards/rejected": -4.091948509216309, + "step": 3140 + }, + { + "epoch": 0.82, + "grad_norm": 7.53125, + "learning_rate": 4.554010145972418e-07, + "logits/chosen": -1.5110399723052979, + "logits/rejected": -1.3339247703552246, + "logps/chosen": -550.0721435546875, + "logps/rejected": -648.1046752929688, + "loss": 0.5631, + "rewards/accuracies": 0.731249988079071, + "rewards/chosen": -2.862560749053955, + "rewards/margins": 1.0935637950897217, + "rewards/rejected": -3.956124782562256, + "step": 3150 + }, + { + "epoch": 0.83, + "grad_norm": 10.0, + "learning_rate": 4.4234070597637455e-07, + "logits/chosen": -1.3611904382705688, + "logits/rejected": -1.270994782447815, + "logps/chosen": -550.0131225585938, + "logps/rejected": -645.00341796875, + "loss": 0.5164, + "rewards/accuracies": 0.7562500238418579, + "rewards/chosen": -2.744797468185425, + "rewards/margins": 1.0690962076187134, + "rewards/rejected": -3.8138937950134277, + "step": 3160 + }, + { + "epoch": 0.83, + "grad_norm": 8.6875, + "learning_rate": 4.2945221131440783e-07, + "logits/chosen": -1.3394265174865723, + "logits/rejected": -1.1380027532577515, + "logps/chosen": -532.6522216796875, + "logps/rejected": -629.9595336914062, + "loss": 0.4172, + "rewards/accuracies": 0.84375, + "rewards/chosen": -2.6694254875183105, + "rewards/margins": 1.289540410041809, + "rewards/rejected": -3.958966016769409, + "step": 3170 + }, + { + "epoch": 0.83, + "grad_norm": 9.8125, + "learning_rate": 4.167366067969381e-07, + "logits/chosen": -1.420111060142517, + "logits/rejected": -1.356261134147644, + "logps/chosen": -500.45135498046875, + "logps/rejected": -622.8685302734375, + "loss": 0.4999, + "rewards/accuracies": 0.737500011920929, + "rewards/chosen": -2.775409698486328, + "rewards/margins": 0.9858112335205078, + "rewards/rejected": -3.7612204551696777, + "step": 3180 + }, + { + "epoch": 0.83, + "grad_norm": 5.78125, + "learning_rate": 4.041949541732826e-07, + "logits/chosen": -1.4244636297225952, + "logits/rejected": -1.3622827529907227, + "logps/chosen": -547.3236694335938, + "logps/rejected": -641.5424194335938, + "loss": 0.5021, + "rewards/accuracies": 0.737500011920929, + "rewards/chosen": -2.83439302444458, + "rewards/margins": 1.0849192142486572, + "rewards/rejected": -3.9193122386932373, + "step": 3190 + }, + { + "epoch": 0.84, + "grad_norm": 18.5, + "learning_rate": 3.9182830066782614e-07, + "logits/chosen": -1.342223882675171, + "logits/rejected": -1.326421856880188, + "logps/chosen": -541.6400756835938, + "logps/rejected": -670.80126953125, + "loss": 0.4882, + "rewards/accuracies": 0.78125, + "rewards/chosen": -2.8389620780944824, + "rewards/margins": 1.1673671007156372, + "rewards/rejected": -4.006329536437988, + "step": 3200 + }, + { + "epoch": 0.84, + "eval_logits/chosen": -1.3354921340942383, + "eval_logits/rejected": -1.2087724208831787, + "eval_logps/chosen": -541.2137451171875, + "eval_logps/rejected": -633.5067749023438, + "eval_loss": 0.48956310749053955, + "eval_rewards/accuracies": 0.7440000176429749, + "eval_rewards/chosen": -2.7656171321868896, + "eval_rewards/margins": 1.1233649253845215, + "eval_rewards/rejected": -3.8889822959899902, + "eval_runtime": 384.9676, + "eval_samples_per_second": 5.195, + "eval_steps_per_second": 0.649, + "step": 3200 + }, + { + "epoch": 0.84, + "grad_norm": 7.09375, + "learning_rate": 3.796376788925771e-07, + "logits/chosen": -1.3472093343734741, + "logits/rejected": -1.27553129196167, + "logps/chosen": -528.8458251953125, + "logps/rejected": -600.9093627929688, + "loss": 0.5081, + "rewards/accuracies": 0.737500011920929, + "rewards/chosen": -2.674180507659912, + "rewards/margins": 0.9610416293144226, + "rewards/rejected": -3.6352221965789795, + "step": 3210 + }, + { + "epoch": 0.84, + "grad_norm": 7.8125, + "learning_rate": 3.676241067609465e-07, + "logits/chosen": -1.4311882257461548, + "logits/rejected": -1.3133292198181152, + "logps/chosen": -566.4583740234375, + "logps/rejected": -629.7789306640625, + "loss": 0.516, + "rewards/accuracies": 0.768750011920929, + "rewards/chosen": -2.7275829315185547, + "rewards/margins": 1.0640867948532104, + "rewards/rejected": -3.791670322418213, + "step": 3220 + }, + { + "epoch": 0.85, + "grad_norm": 13.3125, + "learning_rate": 3.5578858740274976e-07, + "logits/chosen": -1.3556554317474365, + "logits/rejected": -1.2464677095413208, + "logps/chosen": -546.6044311523438, + "logps/rejected": -628.5745849609375, + "loss": 0.5223, + "rewards/accuracies": 0.737500011920929, + "rewards/chosen": -2.848706007003784, + "rewards/margins": 0.9455618858337402, + "rewards/rejected": -3.7942676544189453, + "step": 3230 + }, + { + "epoch": 0.85, + "grad_norm": 10.875, + "learning_rate": 3.44132109080447e-07, + "logits/chosen": -1.5402499437332153, + "logits/rejected": -1.3627904653549194, + "logps/chosen": -532.7853393554688, + "logps/rejected": -620.8280029296875, + "loss": 0.4349, + "rewards/accuracies": 0.78125, + "rewards/chosen": -2.6756184101104736, + "rewards/margins": 1.2552212476730347, + "rewards/rejected": -3.9308395385742188, + "step": 3240 + }, + { + "epoch": 0.85, + "grad_norm": 11.625, + "learning_rate": 3.3265564510662344e-07, + "logits/chosen": -1.4746288061141968, + "logits/rejected": -1.3366063833236694, + "logps/chosen": -556.5170288085938, + "logps/rejected": -653.8280029296875, + "loss": 0.4486, + "rewards/accuracies": 0.793749988079071, + "rewards/chosen": -2.631749391555786, + "rewards/margins": 1.22456693649292, + "rewards/rejected": -3.856316328048706, + "step": 3250 + }, + { + "epoch": 0.85, + "grad_norm": 13.4375, + "learning_rate": 3.213601537627195e-07, + "logits/chosen": -1.3857685327529907, + "logits/rejected": -1.2827407121658325, + "logps/chosen": -551.5367431640625, + "logps/rejected": -642.1528930664062, + "loss": 0.5368, + "rewards/accuracies": 0.7124999761581421, + "rewards/chosen": -2.944179058074951, + "rewards/margins": 1.0771982669830322, + "rewards/rejected": -4.021376609802246, + "step": 3260 + }, + { + "epoch": 0.86, + "grad_norm": 13.1875, + "learning_rate": 3.1024657821901063e-07, + "logits/chosen": -1.4493802785873413, + "logits/rejected": -1.3605637550354004, + "logps/chosen": -506.7554626464844, + "logps/rejected": -613.6812744140625, + "loss": 0.4775, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -2.590136766433716, + "rewards/margins": 1.2277183532714844, + "rewards/rejected": -3.8178551197052, + "step": 3270 + }, + { + "epoch": 0.86, + "grad_norm": 14.1875, + "learning_rate": 2.9931584645585654e-07, + "logits/chosen": -1.3915107250213623, + "logits/rejected": -1.3630131483078003, + "logps/chosen": -540.7659912109375, + "logps/rejected": -647.734375, + "loss": 0.5041, + "rewards/accuracies": 0.7562500238418579, + "rewards/chosen": -2.6625149250030518, + "rewards/margins": 1.0405315160751343, + "rewards/rejected": -3.7030467987060547, + "step": 3280 + }, + { + "epoch": 0.86, + "grad_norm": 5.5625, + "learning_rate": 2.885688711862136e-07, + "logits/chosen": -1.4039568901062012, + "logits/rejected": -1.4016002416610718, + "logps/chosen": -542.4031372070312, + "logps/rejected": -669.4109497070312, + "loss": 0.4979, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -2.813516139984131, + "rewards/margins": 1.2897857427597046, + "rewards/rejected": -4.103302001953125, + "step": 3290 + }, + { + "epoch": 0.86, + "grad_norm": 13.9375, + "learning_rate": 2.7800654977942486e-07, + "logits/chosen": -1.3982176780700684, + "logits/rejected": -1.2750059366226196, + "logps/chosen": -529.8203735351562, + "logps/rejected": -631.67431640625, + "loss": 0.5123, + "rewards/accuracies": 0.7437499761581421, + "rewards/chosen": -2.7246382236480713, + "rewards/margins": 1.056668996810913, + "rewards/rejected": -3.7813076972961426, + "step": 3300 + }, + { + "epoch": 0.86, + "eval_logits/chosen": -1.3351857662200928, + "eval_logits/rejected": -1.208348274230957, + "eval_logps/chosen": -542.1024780273438, + "eval_logps/rejected": -634.3662109375, + "eval_loss": 0.4894912838935852, + "eval_rewards/accuracies": 0.7434999942779541, + "eval_rewards/chosen": -2.7745048999786377, + "eval_rewards/margins": 1.123070478439331, + "eval_rewards/rejected": -3.8975753784179688, + "eval_runtime": 384.9043, + "eval_samples_per_second": 5.196, + "eval_steps_per_second": 0.65, + "step": 3300 + }, + { + "epoch": 0.87, + "grad_norm": 14.3125, + "learning_rate": 2.6762976418628797e-07, + "logits/chosen": -1.434815764427185, + "logits/rejected": -1.2825909852981567, + "logps/chosen": -497.67559814453125, + "logps/rejected": -557.65478515625, + "loss": 0.5421, + "rewards/accuracies": 0.6937500238418579, + "rewards/chosen": -2.67899751663208, + "rewards/margins": 1.0358701944351196, + "rewards/rejected": -3.714867353439331, + "step": 3310 + }, + { + "epoch": 0.87, + "grad_norm": 17.0, + "learning_rate": 2.5743938086541354e-07, + "logits/chosen": -1.4120391607284546, + "logits/rejected": -1.282286524772644, + "logps/chosen": -537.319580078125, + "logps/rejected": -624.9915771484375, + "loss": 0.4846, + "rewards/accuracies": 0.75, + "rewards/chosen": -2.730372428894043, + "rewards/margins": 1.1250317096710205, + "rewards/rejected": -3.8554039001464844, + "step": 3320 + }, + { + "epoch": 0.87, + "grad_norm": 13.125, + "learning_rate": 2.4743625071087574e-07, + "logits/chosen": -1.5484386682510376, + "logits/rejected": -1.370973825454712, + "logps/chosen": -544.7464599609375, + "logps/rejected": -646.3916625976562, + "loss": 0.479, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -2.6717612743377686, + "rewards/margins": 1.311167597770691, + "rewards/rejected": -3.982929229736328, + "step": 3330 + }, + { + "epoch": 0.87, + "grad_norm": 10.5, + "learning_rate": 2.3762120898116498e-07, + "logits/chosen": -1.428130865097046, + "logits/rejected": -1.3167780637741089, + "logps/chosen": -561.8053588867188, + "logps/rejected": -654.1575927734375, + "loss": 0.4989, + "rewards/accuracies": 0.737500011920929, + "rewards/chosen": -2.930124282836914, + "rewards/margins": 1.0036227703094482, + "rewards/rejected": -3.9337470531463623, + "step": 3340 + }, + { + "epoch": 0.88, + "grad_norm": 8.5625, + "learning_rate": 2.2799507522944048e-07, + "logits/chosen": -1.361971139907837, + "logits/rejected": -1.2745027542114258, + "logps/chosen": -532.9896240234375, + "logps/rejected": -649.6702270507812, + "loss": 0.4514, + "rewards/accuracies": 0.7437499761581421, + "rewards/chosen": -2.671921730041504, + "rewards/margins": 1.2491633892059326, + "rewards/rejected": -3.9210853576660156, + "step": 3350 + }, + { + "epoch": 0.88, + "grad_norm": 11.375, + "learning_rate": 2.1855865323510056e-07, + "logits/chosen": -1.424619436264038, + "logits/rejected": -1.2369648218154907, + "logps/chosen": -545.8442993164062, + "logps/rejected": -683.6163330078125, + "loss": 0.4292, + "rewards/accuracies": 0.8062499761581421, + "rewards/chosen": -2.731658458709717, + "rewards/margins": 1.4368107318878174, + "rewards/rejected": -4.168468952178955, + "step": 3360 + }, + { + "epoch": 0.88, + "grad_norm": 8.3125, + "learning_rate": 2.0931273093666575e-07, + "logits/chosen": -1.3846908807754517, + "logits/rejected": -1.2318280935287476, + "logps/chosen": -527.0106201171875, + "logps/rejected": -627.6875, + "loss": 0.4511, + "rewards/accuracies": 0.84375, + "rewards/chosen": -2.868238925933838, + "rewards/margins": 1.2093108892440796, + "rewards/rejected": -4.077549934387207, + "step": 3370 + }, + { + "epoch": 0.88, + "grad_norm": 12.0, + "learning_rate": 2.002580803659873e-07, + "logits/chosen": -1.3799206018447876, + "logits/rejected": -1.2638591527938843, + "logps/chosen": -536.2264404296875, + "logps/rejected": -633.6256713867188, + "loss": 0.4582, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -2.816713333129883, + "rewards/margins": 1.155531406402588, + "rewards/rejected": -3.97224497795105, + "step": 3380 + }, + { + "epoch": 0.89, + "grad_norm": 6.125, + "learning_rate": 1.913954575837826e-07, + "logits/chosen": -1.455733299255371, + "logits/rejected": -1.1953377723693848, + "logps/chosen": -554.5827026367188, + "logps/rejected": -613.561767578125, + "loss": 0.4729, + "rewards/accuracies": 0.793749988079071, + "rewards/chosen": -2.8246824741363525, + "rewards/margins": 1.0867326259613037, + "rewards/rejected": -3.9114151000976562, + "step": 3390 + }, + { + "epoch": 0.89, + "grad_norm": 9.1875, + "learning_rate": 1.827256026165028e-07, + "logits/chosen": -1.46336829662323, + "logits/rejected": -1.2622567415237427, + "logps/chosen": -575.8062744140625, + "logps/rejected": -644.166748046875, + "loss": 0.4526, + "rewards/accuracies": 0.793749988079071, + "rewards/chosen": -2.6073572635650635, + "rewards/margins": 1.2542911767959595, + "rewards/rejected": -3.8616480827331543, + "step": 3400 + }, + { + "epoch": 0.89, + "eval_logits/chosen": -1.3318783044815063, + "eval_logits/rejected": -1.205055594444275, + "eval_logps/chosen": -543.2083129882812, + "eval_logps/rejected": -635.9655151367188, + "eval_loss": 0.4895820915699005, + "eval_rewards/accuracies": 0.7444999814033508, + "eval_rewards/chosen": -2.785562753677368, + "eval_rewards/margins": 1.1280065774917603, + "eval_rewards/rejected": -3.913569450378418, + "eval_runtime": 384.9489, + "eval_samples_per_second": 5.195, + "eval_steps_per_second": 0.649, + "step": 3400 + }, + { + "epoch": 0.89, + "grad_norm": 11.4375, + "learning_rate": 1.7424923939454274e-07, + "logits/chosen": -1.4153268337249756, + "logits/rejected": -1.235887885093689, + "logps/chosen": -558.7437744140625, + "logps/rejected": -640.8536376953125, + "loss": 0.4144, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": -2.77394437789917, + "rewards/margins": 1.254831075668335, + "rewards/rejected": -4.028775215148926, + "step": 3410 + }, + { + "epoch": 0.9, + "grad_norm": 19.25, + "learning_rate": 1.6596707569179304e-07, + "logits/chosen": -1.4984407424926758, + "logits/rejected": -1.3432929515838623, + "logps/chosen": -559.1913452148438, + "logps/rejected": -636.6985473632812, + "loss": 0.491, + "rewards/accuracies": 0.768750011920929, + "rewards/chosen": -2.7800402641296387, + "rewards/margins": 1.1263272762298584, + "rewards/rejected": -3.906367540359497, + "step": 3420 + }, + { + "epoch": 0.9, + "grad_norm": 8.75, + "learning_rate": 1.578798030665385e-07, + "logits/chosen": -1.4436315298080444, + "logits/rejected": -1.2577316761016846, + "logps/chosen": -546.3856811523438, + "logps/rejected": -664.1823120117188, + "loss": 0.4391, + "rewards/accuracies": 0.768750011920929, + "rewards/chosen": -2.7168757915496826, + "rewards/margins": 1.348455786705017, + "rewards/rejected": -4.065331935882568, + "step": 3430 + }, + { + "epoch": 0.9, + "grad_norm": 8.875, + "learning_rate": 1.499880968037165e-07, + "logits/chosen": -1.4255344867706299, + "logits/rejected": -1.2893887758255005, + "logps/chosen": -532.6036376953125, + "logps/rejected": -603.4581298828125, + "loss": 0.5173, + "rewards/accuracies": 0.768750011920929, + "rewards/chosen": -2.7699015140533447, + "rewards/margins": 1.071218729019165, + "rewards/rejected": -3.8411202430725098, + "step": 3440 + }, + { + "epoch": 0.9, + "grad_norm": 12.4375, + "learning_rate": 1.4229261585852805e-07, + "logits/chosen": -1.4486864805221558, + "logits/rejected": -1.366389513015747, + "logps/chosen": -536.9041748046875, + "logps/rejected": -628.5606689453125, + "loss": 0.4501, + "rewards/accuracies": 0.762499988079071, + "rewards/chosen": -2.6847262382507324, + "rewards/margins": 1.1747690439224243, + "rewards/rejected": -3.859494686126709, + "step": 3450 + }, + { + "epoch": 0.91, + "grad_norm": 11.5, + "learning_rate": 1.3479400280141886e-07, + "logits/chosen": -1.3643203973770142, + "logits/rejected": -1.319645643234253, + "logps/chosen": -524.4163818359375, + "logps/rejected": -644.7371826171875, + "loss": 0.4688, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -2.781571865081787, + "rewards/margins": 1.223384976387024, + "rewards/rejected": -4.00495719909668, + "step": 3460 + }, + { + "epoch": 0.91, + "grad_norm": 7.25, + "learning_rate": 1.2749288376442044e-07, + "logits/chosen": -1.4585063457489014, + "logits/rejected": -1.2669061422348022, + "logps/chosen": -567.2780151367188, + "logps/rejected": -623.2936401367188, + "loss": 0.4761, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -2.70743727684021, + "rewards/margins": 1.135317325592041, + "rewards/rejected": -3.842755079269409, + "step": 3470 + }, + { + "epoch": 0.91, + "grad_norm": 9.1875, + "learning_rate": 1.203898683888713e-07, + "logits/chosen": -1.4662349224090576, + "logits/rejected": -1.3190191984176636, + "logps/chosen": -531.9495849609375, + "logps/rejected": -622.1035766601562, + "loss": 0.5705, + "rewards/accuracies": 0.6625000238418579, + "rewards/chosen": -2.879521369934082, + "rewards/margins": 0.9325839281082153, + "rewards/rejected": -3.812105178833008, + "step": 3480 + }, + { + "epoch": 0.91, + "grad_norm": 8.875, + "learning_rate": 1.1348554977451132e-07, + "logits/chosen": -1.4893817901611328, + "logits/rejected": -1.3383376598358154, + "logps/chosen": -555.6965942382812, + "logps/rejected": -626.2882690429688, + "loss": 0.5165, + "rewards/accuracies": 0.71875, + "rewards/chosen": -2.737074613571167, + "rewards/margins": 1.034942865371704, + "rewards/rejected": -3.77201771736145, + "step": 3490 + }, + { + "epoch": 0.92, + "grad_norm": 7.09375, + "learning_rate": 1.0678050442995802e-07, + "logits/chosen": -1.444490671157837, + "logits/rejected": -1.2492029666900635, + "logps/chosen": -563.559326171875, + "logps/rejected": -623.2242431640625, + "loss": 0.5432, + "rewards/accuracies": 0.7437499761581421, + "rewards/chosen": -2.8276607990264893, + "rewards/margins": 1.0474214553833008, + "rewards/rejected": -3.875082015991211, + "step": 3500 + }, + { + "epoch": 0.92, + "eval_logits/chosen": -1.331380009651184, + "eval_logits/rejected": -1.2045425176620483, + "eval_logps/chosen": -543.0230712890625, + "eval_logps/rejected": -635.9038696289062, + "eval_loss": 0.4895781874656677, + "eval_rewards/accuracies": 0.7440000176429749, + "eval_rewards/chosen": -2.783710479736328, + "eval_rewards/margins": 1.1292426586151123, + "eval_rewards/rejected": -3.9129531383514404, + "eval_runtime": 385.1815, + "eval_samples_per_second": 5.192, + "eval_steps_per_second": 0.649, + "step": 3500 + }, + { + "epoch": 0.92, + "grad_norm": 8.0625, + "learning_rate": 1.0027529222456755e-07, + "logits/chosen": -1.423830509185791, + "logits/rejected": -1.2484782934188843, + "logps/chosen": -528.2448120117188, + "logps/rejected": -626.2152099609375, + "loss": 0.4541, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -2.7407870292663574, + "rewards/margins": 1.155261754989624, + "rewards/rejected": -3.8960487842559814, + "step": 3510 + }, + { + "epoch": 0.92, + "grad_norm": 10.1875, + "learning_rate": 9.397045634168766e-08, + "logits/chosen": -1.4465529918670654, + "logits/rejected": -1.3796955347061157, + "logps/chosen": -536.9094848632812, + "logps/rejected": -664.8760375976562, + "loss": 0.4559, + "rewards/accuracies": 0.762499988079071, + "rewards/chosen": -2.6832659244537354, + "rewards/margins": 1.277266502380371, + "rewards/rejected": -3.9605324268341064, + "step": 3520 + }, + { + "epoch": 0.92, + "grad_norm": 10.125, + "learning_rate": 8.78665232332998e-08, + "logits/chosen": -1.3762016296386719, + "logits/rejected": -1.3040244579315186, + "logps/chosen": -512.94140625, + "logps/rejected": -617.2539672851562, + "loss": 0.4859, + "rewards/accuracies": 0.71875, + "rewards/chosen": -2.8223133087158203, + "rewards/margins": 1.0507687330245972, + "rewards/rejected": -3.873081922531128, + "step": 3530 + }, + { + "epoch": 0.93, + "grad_norm": 9.5, + "learning_rate": 8.196400257606208e-08, + "logits/chosen": -1.4817397594451904, + "logits/rejected": -1.334993839263916, + "logps/chosen": -555.3631591796875, + "logps/rejected": -686.2461547851562, + "loss": 0.4321, + "rewards/accuracies": 0.793749988079071, + "rewards/chosen": -2.72278094291687, + "rewards/margins": 1.3615996837615967, + "rewards/rejected": -4.084380626678467, + "step": 3540 + }, + { + "epoch": 0.93, + "grad_norm": 8.875, + "learning_rate": 7.626338722875076e-08, + "logits/chosen": -1.423555612564087, + "logits/rejected": -1.3563083410263062, + "logps/chosen": -526.125244140625, + "logps/rejected": -638.4963989257812, + "loss": 0.4891, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -2.720315933227539, + "rewards/margins": 1.1140156984329224, + "rewards/rejected": -3.83433198928833, + "step": 3550 + }, + { + "epoch": 0.93, + "grad_norm": 6.125, + "learning_rate": 7.076515319110688e-08, + "logits/chosen": -1.42192804813385, + "logits/rejected": -1.3248012065887451, + "logps/chosen": -528.8367919921875, + "logps/rejected": -604.7586669921875, + "loss": 0.5312, + "rewards/accuracies": 0.737500011920929, + "rewards/chosen": -2.725490093231201, + "rewards/margins": 1.1749989986419678, + "rewards/rejected": -3.900489091873169, + "step": 3560 + }, + { + "epoch": 0.93, + "grad_norm": 9.6875, + "learning_rate": 6.54697595640899e-08, + "logits/chosen": -1.4399796724319458, + "logits/rejected": -1.3037444353103638, + "logps/chosen": -573.2326049804688, + "logps/rejected": -660.8978271484375, + "loss": 0.4778, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -2.8101038932800293, + "rewards/margins": 1.1283040046691895, + "rewards/rejected": -3.938408374786377, + "step": 3570 + }, + { + "epoch": 0.94, + "grad_norm": 8.625, + "learning_rate": 6.037764851154426e-08, + "logits/chosen": -1.425642490386963, + "logits/rejected": -1.3774255514144897, + "logps/chosen": -533.1048583984375, + "logps/rejected": -653.04736328125, + "loss": 0.4959, + "rewards/accuracies": 0.706250011920929, + "rewards/chosen": -2.6904561519622803, + "rewards/margins": 1.1518871784210205, + "rewards/rejected": -3.842343807220459, + "step": 3580 + }, + { + "epoch": 0.94, + "grad_norm": 7.5625, + "learning_rate": 5.548924522327748e-08, + "logits/chosen": -1.4221152067184448, + "logits/rejected": -1.2739677429199219, + "logps/chosen": -533.532958984375, + "logps/rejected": -629.9300537109375, + "loss": 0.4811, + "rewards/accuracies": 0.737500011920929, + "rewards/chosen": -2.7177557945251465, + "rewards/margins": 1.110734224319458, + "rewards/rejected": -3.8284904956817627, + "step": 3590 + }, + { + "epoch": 0.94, + "grad_norm": 10.375, + "learning_rate": 5.0804957879556915e-08, + "logits/chosen": -1.3408573865890503, + "logits/rejected": -1.2565336227416992, + "logps/chosen": -498.77581787109375, + "logps/rejected": -613.9055786132812, + "loss": 0.4617, + "rewards/accuracies": 0.7437499761581421, + "rewards/chosen": -2.732095241546631, + "rewards/margins": 1.105553150177002, + "rewards/rejected": -3.837648391723633, + "step": 3600 + }, + { + "epoch": 0.94, + "eval_logits/chosen": -1.337357521057129, + "eval_logits/rejected": -1.210402011871338, + "eval_logps/chosen": -543.2186279296875, + "eval_logps/rejected": -636.1134643554688, + "eval_loss": 0.4894636869430542, + "eval_rewards/accuracies": 0.7434999942779541, + "eval_rewards/chosen": -2.785665988922119, + "eval_rewards/margins": 1.1293821334838867, + "eval_rewards/rejected": -3.915048122406006, + "eval_runtime": 384.9877, + "eval_samples_per_second": 5.195, + "eval_steps_per_second": 0.649, + "step": 3600 + }, + { + "epoch": 0.94, + "grad_norm": 12.8125, + "learning_rate": 4.632517761702815e-08, + "logits/chosen": -1.3654999732971191, + "logits/rejected": -1.2209546566009521, + "logps/chosen": -514.5064697265625, + "logps/rejected": -634.9259033203125, + "loss": 0.451, + "rewards/accuracies": 0.7562500238418579, + "rewards/chosen": -2.7906320095062256, + "rewards/margins": 1.32748281955719, + "rewards/rejected": -4.118114948272705, + "step": 3610 + }, + { + "epoch": 0.95, + "grad_norm": 11.9375, + "learning_rate": 4.205027849605359e-08, + "logits/chosen": -1.3972408771514893, + "logits/rejected": -1.2909516096115112, + "logps/chosen": -535.4818115234375, + "logps/rejected": -608.0510864257812, + "loss": 0.5396, + "rewards/accuracies": 0.7437499761581421, + "rewards/chosen": -2.875584363937378, + "rewards/margins": 1.0263030529022217, + "rewards/rejected": -3.9018874168395996, + "step": 3620 + }, + { + "epoch": 0.95, + "grad_norm": 9.875, + "learning_rate": 3.798061746947995e-08, + "logits/chosen": -1.5177159309387207, + "logits/rejected": -1.354299783706665, + "logps/chosen": -539.2265014648438, + "logps/rejected": -615.7210693359375, + "loss": 0.4806, + "rewards/accuracies": 0.6875, + "rewards/chosen": -2.750765800476074, + "rewards/margins": 1.1568615436553955, + "rewards/rejected": -3.907627820968628, + "step": 3630 + }, + { + "epoch": 0.95, + "grad_norm": 10.5, + "learning_rate": 3.411653435283158e-08, + "logits/chosen": -1.4375641345977783, + "logits/rejected": -1.2251198291778564, + "logps/chosen": -544.5794677734375, + "logps/rejected": -597.5071411132812, + "loss": 0.4841, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -2.713228464126587, + "rewards/margins": 1.0865360498428345, + "rewards/rejected": -3.799764633178711, + "step": 3640 + }, + { + "epoch": 0.96, + "grad_norm": 13.125, + "learning_rate": 3.04583517959367e-08, + "logits/chosen": -1.4777367115020752, + "logits/rejected": -1.3228670358657837, + "logps/chosen": -516.0816650390625, + "logps/rejected": -592.5147094726562, + "loss": 0.4782, + "rewards/accuracies": 0.762499988079071, + "rewards/chosen": -2.6396498680114746, + "rewards/margins": 1.0887558460235596, + "rewards/rejected": -3.728405714035034, + "step": 3650 + }, + { + "epoch": 0.96, + "grad_norm": 15.25, + "learning_rate": 2.7006375255985984e-08, + "logits/chosen": -1.4119120836257935, + "logits/rejected": -1.3788807392120361, + "logps/chosen": -553.791015625, + "logps/rejected": -641.79443359375, + "loss": 0.5775, + "rewards/accuracies": 0.6812499761581421, + "rewards/chosen": -2.8898892402648926, + "rewards/margins": 0.8745861053466797, + "rewards/rejected": -3.7644753456115723, + "step": 3660 + }, + { + "epoch": 0.96, + "grad_norm": 10.0, + "learning_rate": 2.3760892972027328e-08, + "logits/chosen": -1.5255684852600098, + "logits/rejected": -1.3464335203170776, + "logps/chosen": -559.984375, + "logps/rejected": -636.6268310546875, + "loss": 0.5395, + "rewards/accuracies": 0.7437499761581421, + "rewards/chosen": -2.874812602996826, + "rewards/margins": 1.1065670251846313, + "rewards/rejected": -3.981379270553589, + "step": 3670 + }, + { + "epoch": 0.96, + "grad_norm": 12.8125, + "learning_rate": 2.072217594089765e-08, + "logits/chosen": -1.3876298666000366, + "logits/rejected": -1.365395188331604, + "logps/chosen": -542.5714721679688, + "logps/rejected": -655.452880859375, + "loss": 0.4209, + "rewards/accuracies": 0.7875000238418579, + "rewards/chosen": -2.8103041648864746, + "rewards/margins": 1.2513482570648193, + "rewards/rejected": -4.061652660369873, + "step": 3680 + }, + { + "epoch": 0.97, + "grad_norm": 8.375, + "learning_rate": 1.789047789459375e-08, + "logits/chosen": -1.488208532333374, + "logits/rejected": -1.2888844013214111, + "logps/chosen": -593.5070190429688, + "logps/rejected": -661.3151245117188, + "loss": 0.5217, + "rewards/accuracies": 0.7875000238418579, + "rewards/chosen": -2.765303373336792, + "rewards/margins": 1.168646216392517, + "rewards/rejected": -3.9339499473571777, + "step": 3690 + }, + { + "epoch": 0.97, + "grad_norm": 7.65625, + "learning_rate": 1.5266035279088708e-08, + "logits/chosen": -1.322689414024353, + "logits/rejected": -1.1827501058578491, + "logps/chosen": -589.371826171875, + "logps/rejected": -676.3383178710938, + "loss": 0.4797, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -2.927542209625244, + "rewards/margins": 1.131638765335083, + "rewards/rejected": -4.059180736541748, + "step": 3700 + }, + { + "epoch": 0.97, + "eval_logits/chosen": -1.3342877626419067, + "eval_logits/rejected": -1.2075273990631104, + "eval_logps/chosen": -543.0763549804688, + "eval_logps/rejected": -635.919189453125, + "eval_loss": 0.4895748794078827, + "eval_rewards/accuracies": 0.7434999942779541, + "eval_rewards/chosen": -2.784243583679199, + "eval_rewards/margins": 1.1288617849349976, + "eval_rewards/rejected": -3.9131054878234863, + "eval_runtime": 385.4925, + "eval_samples_per_second": 5.188, + "eval_steps_per_second": 0.649, + "step": 3700 + }, + { + "epoch": 0.97, + "grad_norm": 16.0, + "learning_rate": 1.2849067234584623e-08, + "logits/chosen": -1.3140819072723389, + "logits/rejected": -1.2478830814361572, + "logps/chosen": -514.704833984375, + "logps/rejected": -628.93017578125, + "loss": 0.4821, + "rewards/accuracies": 0.7437499761581421, + "rewards/chosen": -2.7834410667419434, + "rewards/margins": 1.2183729410171509, + "rewards/rejected": -4.001813888549805, + "step": 3710 + }, + { + "epoch": 0.97, + "grad_norm": 15.1875, + "learning_rate": 1.0639775577218625e-08, + "logits/chosen": -1.3101227283477783, + "logits/rejected": -1.1396509408950806, + "logps/chosen": -528.97265625, + "logps/rejected": -609.4863891601562, + "loss": 0.5179, + "rewards/accuracies": 0.71875, + "rewards/chosen": -2.7978572845458984, + "rewards/margins": 1.1644933223724365, + "rewards/rejected": -3.962350368499756, + "step": 3720 + }, + { + "epoch": 0.98, + "grad_norm": 11.5625, + "learning_rate": 8.638344782207486e-09, + "logits/chosen": -1.3434375524520874, + "logits/rejected": -1.2269071340560913, + "logps/chosen": -513.5137939453125, + "logps/rejected": -601.6809692382812, + "loss": 0.4808, + "rewards/accuracies": 0.78125, + "rewards/chosen": -2.6822597980499268, + "rewards/margins": 1.1096186637878418, + "rewards/rejected": -3.7918784618377686, + "step": 3730 + }, + { + "epoch": 0.98, + "grad_norm": 9.875, + "learning_rate": 6.84494196844715e-09, + "logits/chosen": -1.3948211669921875, + "logits/rejected": -1.2849534749984741, + "logps/chosen": -548.9085083007812, + "logps/rejected": -666.4553833007812, + "loss": 0.4667, + "rewards/accuracies": 0.762499988079071, + "rewards/chosen": -2.7680201530456543, + "rewards/margins": 1.2913219928741455, + "rewards/rejected": -4.059341907501221, + "step": 3740 + }, + { + "epoch": 0.98, + "grad_norm": 8.875, + "learning_rate": 5.259716884556121e-09, + "logits/chosen": -1.4579029083251953, + "logits/rejected": -1.3110452890396118, + "logps/chosen": -538.3919067382812, + "logps/rejected": -639.5615234375, + "loss": 0.4595, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -2.739250659942627, + "rewards/margins": 1.1599111557006836, + "rewards/rejected": -3.8991622924804688, + "step": 3750 + }, + { + "epoch": 0.98, + "grad_norm": 8.8125, + "learning_rate": 3.882801896372967e-09, + "logits/chosen": -1.4429559707641602, + "logits/rejected": -1.3740085363388062, + "logps/chosen": -534.4335327148438, + "logps/rejected": -618.3828735351562, + "loss": 0.4868, + "rewards/accuracies": 0.762499988079071, + "rewards/chosen": -2.6907172203063965, + "rewards/margins": 1.1578295230865479, + "rewards/rejected": -3.8485465049743652, + "step": 3760 + }, + { + "epoch": 0.99, + "grad_norm": 8.625, + "learning_rate": 2.7143119759026614e-09, + "logits/chosen": -1.4640624523162842, + "logits/rejected": -1.2803726196289062, + "logps/chosen": -557.8724975585938, + "logps/rejected": -646.456298828125, + "loss": 0.4267, + "rewards/accuracies": 0.7875000238418579, + "rewards/chosen": -2.7360005378723145, + "rewards/margins": 1.1408417224884033, + "rewards/rejected": -3.8768420219421387, + "step": 3770 + }, + { + "epoch": 0.99, + "grad_norm": 11.125, + "learning_rate": 1.754344691717591e-09, + "logits/chosen": -1.3610948324203491, + "logits/rejected": -1.3128563165664673, + "logps/chosen": -533.3128662109375, + "logps/rejected": -645.0467529296875, + "loss": 0.5359, + "rewards/accuracies": 0.7124999761581421, + "rewards/chosen": -2.835211992263794, + "rewards/margins": 0.896623432636261, + "rewards/rejected": -3.731835126876831, + "step": 3780 + }, + { + "epoch": 0.99, + "grad_norm": 9.9375, + "learning_rate": 1.0029802008096335e-09, + "logits/chosen": -1.3813669681549072, + "logits/rejected": -1.2357242107391357, + "logps/chosen": -551.9505615234375, + "logps/rejected": -645.6619262695312, + "loss": 0.4777, + "rewards/accuracies": 0.75, + "rewards/chosen": -2.7765860557556152, + "rewards/margins": 1.1649653911590576, + "rewards/rejected": -3.941551685333252, + "step": 3790 + }, + { + "epoch": 0.99, + "grad_norm": 11.0, + "learning_rate": 4.602812418974534e-10, + "logits/chosen": -1.48513925075531, + "logits/rejected": -1.3569138050079346, + "logps/chosen": -561.6650390625, + "logps/rejected": -650.4069213867188, + "loss": 0.5092, + "rewards/accuracies": 0.71875, + "rewards/chosen": -2.7982420921325684, + "rewards/margins": 1.1328747272491455, + "rewards/rejected": -3.931116819381714, + "step": 3800 + }, + { + "epoch": 0.99, + "eval_logits/chosen": -1.33402681350708, + "eval_logits/rejected": -1.207356572151184, + "eval_logps/chosen": -543.0327758789062, + "eval_logps/rejected": -635.90625, + "eval_loss": 0.48944273591041565, + "eval_rewards/accuracies": 0.7444999814033508, + "eval_rewards/chosen": -2.7838070392608643, + "eval_rewards/margins": 1.1291695833206177, + "eval_rewards/rejected": -3.9129767417907715, + "eval_runtime": 385.4448, + "eval_samples_per_second": 5.189, + "eval_steps_per_second": 0.649, + "step": 3800 + }, + { + "epoch": 1.0, + "grad_norm": 11.25, + "learning_rate": 1.2629313018819312e-10, + "logits/chosen": -1.4069092273712158, + "logits/rejected": -1.28346848487854, + "logps/chosen": -526.5909423828125, + "logps/rejected": -611.5968017578125, + "loss": 0.5062, + "rewards/accuracies": 0.75, + "rewards/chosen": -2.7411856651306152, + "rewards/margins": 1.006593942642212, + "rewards/rejected": -3.7477798461914062, + "step": 3810 + }, + { + "epoch": 1.0, + "grad_norm": 20.25, + "learning_rate": 1.0437535929996855e-12, + "logits/chosen": -1.399791955947876, + "logits/rejected": -1.2320655584335327, + "logps/chosen": -567.6913452148438, + "logps/rejected": -660.8294067382812, + "loss": 0.4642, + "rewards/accuracies": 0.7562500238418579, + "rewards/chosen": -2.8092477321624756, + "rewards/margins": 1.3779563903808594, + "rewards/rejected": -4.187203884124756, + "step": 3820 + }, + { + "epoch": 1.0, + "step": 3821, + "total_flos": 0.0, + "train_loss": 0.521260229900413, + "train_runtime": 42446.6624, + "train_samples_per_second": 1.44, + "train_steps_per_second": 0.09 + } + ], + "logging_steps": 10, + "max_steps": 3821, + "num_input_tokens_seen": 0, + "num_train_epochs": 1, + "save_steps": 100, + "total_flos": 0.0, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +}