| { | |
| "best_metric": null, | |
| "best_model_checkpoint": null, | |
| "epoch": 1.0, | |
| "eval_steps": 500, | |
| "global_step": 637, | |
| "is_hyper_param_search": false, | |
| "is_local_process_zero": true, | |
| "is_world_process_zero": true, | |
| "log_history": [ | |
| { | |
| "epoch": 0.006279434850863423, | |
| "grad_norm": 10.468690716679266, | |
| "learning_rate": 3.125e-08, | |
| "logits/chosen": -2.8471853733062744, | |
| "logits/rejected": -2.8164334297180176, | |
| "logps/chosen": -315.592529296875, | |
| "logps/rejected": -230.47390747070312, | |
| "loss": 0.6931, | |
| "rewards/accuracies": 0.0, | |
| "rewards/chosen": 0.0, | |
| "rewards/margins": 0.0, | |
| "rewards/rejected": 0.0, | |
| "step": 1 | |
| }, | |
| { | |
| "epoch": 0.06279434850863422, | |
| "grad_norm": 9.878509369466363, | |
| "learning_rate": 3.1249999999999997e-07, | |
| "logits/chosen": -2.8302619457244873, | |
| "logits/rejected": -2.8141226768493652, | |
| "logps/chosen": -265.6414794921875, | |
| "logps/rejected": -265.56695556640625, | |
| "loss": 0.6932, | |
| "rewards/accuracies": 0.4548611044883728, | |
| "rewards/chosen": 0.0005584080354310572, | |
| "rewards/margins": 0.0001245423045475036, | |
| "rewards/rejected": 0.00043386564357206225, | |
| "step": 10 | |
| }, | |
| { | |
| "epoch": 0.12558869701726844, | |
| "grad_norm": 10.278326132890863, | |
| "learning_rate": 4.990353313429303e-07, | |
| "logits/chosen": -2.8090524673461914, | |
| "logits/rejected": -2.788933277130127, | |
| "logps/chosen": -253.39077758789062, | |
| "logps/rejected": -246.2702178955078, | |
| "loss": 0.6882, | |
| "rewards/accuracies": 0.6875, | |
| "rewards/chosen": 0.019792597740888596, | |
| "rewards/margins": 0.011014475487172604, | |
| "rewards/rejected": 0.00877812597900629, | |
| "step": 20 | |
| }, | |
| { | |
| "epoch": 0.18838304552590268, | |
| "grad_norm": 11.048616082787978, | |
| "learning_rate": 4.882681251368548e-07, | |
| "logits/chosen": -2.763706684112549, | |
| "logits/rejected": -2.75482439994812, | |
| "logps/chosen": -270.31256103515625, | |
| "logps/rejected": -236.15945434570312, | |
| "loss": 0.6709, | |
| "rewards/accuracies": 0.6875, | |
| "rewards/chosen": 0.038934122771024704, | |
| "rewards/margins": 0.04564380645751953, | |
| "rewards/rejected": -0.006709681358188391, | |
| "step": 30 | |
| }, | |
| { | |
| "epoch": 0.25117739403453687, | |
| "grad_norm": 11.185166495621557, | |
| "learning_rate": 4.6604720940421207e-07, | |
| "logits/chosen": -2.7219185829162598, | |
| "logits/rejected": -2.701842784881592, | |
| "logps/chosen": -254.3495330810547, | |
| "logps/rejected": -254.6168212890625, | |
| "loss": 0.6605, | |
| "rewards/accuracies": 0.6781250238418579, | |
| "rewards/chosen": -0.03600288927555084, | |
| "rewards/margins": 0.07350684702396393, | |
| "rewards/rejected": -0.10950972884893417, | |
| "step": 40 | |
| }, | |
| { | |
| "epoch": 0.3139717425431711, | |
| "grad_norm": 13.063361771402542, | |
| "learning_rate": 4.3344075855595097e-07, | |
| "logits/chosen": -2.6782164573669434, | |
| "logits/rejected": -2.6529879570007324, | |
| "logps/chosen": -274.06939697265625, | |
| "logps/rejected": -282.3935546875, | |
| "loss": 0.634, | |
| "rewards/accuracies": 0.721875011920929, | |
| "rewards/chosen": -0.07310701161623001, | |
| "rewards/margins": 0.18823260068893433, | |
| "rewards/rejected": -0.26133960485458374, | |
| "step": 50 | |
| }, | |
| { | |
| "epoch": 0.37676609105180536, | |
| "grad_norm": 15.023213763564414, | |
| "learning_rate": 3.920161866827889e-07, | |
| "logits/chosen": -2.7063775062561035, | |
| "logits/rejected": -2.677776336669922, | |
| "logps/chosen": -301.80023193359375, | |
| "logps/rejected": -276.10565185546875, | |
| "loss": 0.6256, | |
| "rewards/accuracies": 0.6968749761581421, | |
| "rewards/chosen": -0.1930098533630371, | |
| "rewards/margins": 0.21952715516090393, | |
| "rewards/rejected": -0.41253700852394104, | |
| "step": 60 | |
| }, | |
| { | |
| "epoch": 0.43956043956043955, | |
| "grad_norm": 15.201166100431667, | |
| "learning_rate": 3.4376480090239047e-07, | |
| "logits/chosen": -2.7157435417175293, | |
| "logits/rejected": -2.6909706592559814, | |
| "logps/chosen": -312.8190612792969, | |
| "logps/rejected": -312.00616455078125, | |
| "loss": 0.6094, | |
| "rewards/accuracies": 0.671875, | |
| "rewards/chosen": -0.30874696373939514, | |
| "rewards/margins": 0.24815471470355988, | |
| "rewards/rejected": -0.556901752948761, | |
| "step": 70 | |
| }, | |
| { | |
| "epoch": 0.5023547880690737, | |
| "grad_norm": 22.235789809346436, | |
| "learning_rate": 2.910060778827554e-07, | |
| "logits/chosen": -2.711451292037964, | |
| "logits/rejected": -2.6855673789978027, | |
| "logps/chosen": -325.4831237792969, | |
| "logps/rejected": -324.86187744140625, | |
| "loss": 0.5897, | |
| "rewards/accuracies": 0.703125, | |
| "rewards/chosen": -0.31690555810928345, | |
| "rewards/margins": 0.35460132360458374, | |
| "rewards/rejected": -0.6715068817138672, | |
| "step": 80 | |
| }, | |
| { | |
| "epoch": 0.565149136577708, | |
| "grad_norm": 18.534325691706606, | |
| "learning_rate": 2.3627616503391812e-07, | |
| "logits/chosen": -2.72774338722229, | |
| "logits/rejected": -2.7118916511535645, | |
| "logps/chosen": -298.873779296875, | |
| "logps/rejected": -330.8174133300781, | |
| "loss": 0.5713, | |
| "rewards/accuracies": 0.71875, | |
| "rewards/chosen": -0.43646472692489624, | |
| "rewards/margins": 0.34521809220314026, | |
| "rewards/rejected": -0.7816828489303589, | |
| "step": 90 | |
| }, | |
| { | |
| "epoch": 0.6279434850863422, | |
| "grad_norm": 19.673309131968132, | |
| "learning_rate": 1.8220596619089573e-07, | |
| "logits/chosen": -2.6677684783935547, | |
| "logits/rejected": -2.6389312744140625, | |
| "logps/chosen": -322.52081298828125, | |
| "logps/rejected": -328.7558288574219, | |
| "loss": 0.5875, | |
| "rewards/accuracies": 0.703125, | |
| "rewards/chosen": -0.37618938088417053, | |
| "rewards/margins": 0.3784511685371399, | |
| "rewards/rejected": -0.7546405792236328, | |
| "step": 100 | |
| }, | |
| { | |
| "epoch": 0.1726844583987441, | |
| "grad_norm": 31.95579631076908, | |
| "learning_rate": 4.920911573406924e-07, | |
| "logits/chosen": -2.737246513366699, | |
| "logits/rejected": -2.694401979446411, | |
| "logps/chosen": -324.00128173828125, | |
| "logps/rejected": -355.02178955078125, | |
| "loss": 0.4353, | |
| "rewards/accuracies": 0.887499988079071, | |
| "rewards/chosen": -0.3587312698364258, | |
| "rewards/margins": 0.7477051615715027, | |
| "rewards/rejected": -1.1064363718032837, | |
| "step": 110 | |
| }, | |
| { | |
| "epoch": 0.18838304552590268, | |
| "grad_norm": 35.71890678681308, | |
| "learning_rate": 4.883087164434672e-07, | |
| "logits/chosen": -2.626753568649292, | |
| "logits/rejected": -2.6289310455322266, | |
| "logps/chosen": -319.3797302246094, | |
| "logps/rejected": -375.67864990234375, | |
| "loss": 0.4172, | |
| "rewards/accuracies": 0.8374999761581421, | |
| "rewards/chosen": -0.4024926722049713, | |
| "rewards/margins": 0.8855185508728027, | |
| "rewards/rejected": -1.2880111932754517, | |
| "step": 120 | |
| }, | |
| { | |
| "epoch": 0.20408163265306123, | |
| "grad_norm": 44.90656496644609, | |
| "learning_rate": 4.838100964592904e-07, | |
| "logits/chosen": -2.5899486541748047, | |
| "logits/rejected": -2.5745112895965576, | |
| "logps/chosen": -272.218994140625, | |
| "logps/rejected": -347.6167297363281, | |
| "loss": 0.4832, | |
| "rewards/accuracies": 0.7875000238418579, | |
| "rewards/chosen": -0.4248787760734558, | |
| "rewards/margins": 0.8124195337295532, | |
| "rewards/rejected": -1.2372982501983643, | |
| "step": 130 | |
| }, | |
| { | |
| "epoch": 0.21978021978021978, | |
| "grad_norm": 76.16900405992267, | |
| "learning_rate": 4.786088169001671e-07, | |
| "logits/chosen": -2.5720114707946777, | |
| "logits/rejected": -2.5637192726135254, | |
| "logps/chosen": -289.96759033203125, | |
| "logps/rejected": -435.85308837890625, | |
| "loss": 0.3981, | |
| "rewards/accuracies": 0.824999988079071, | |
| "rewards/chosen": -0.3887278437614441, | |
| "rewards/margins": 0.9565574526786804, | |
| "rewards/rejected": -1.3452852964401245, | |
| "step": 140 | |
| }, | |
| { | |
| "epoch": 0.23547880690737832, | |
| "grad_norm": 70.20692587613685, | |
| "learning_rate": 4.727205089511466e-07, | |
| "logits/chosen": -2.5425446033477783, | |
| "logits/rejected": -2.525315523147583, | |
| "logps/chosen": -313.8908996582031, | |
| "logps/rejected": -377.4201354980469, | |
| "loss": 0.4486, | |
| "rewards/accuracies": 0.8374999761581421, | |
| "rewards/chosen": -0.7373483180999756, | |
| "rewards/margins": 0.8309048414230347, | |
| "rewards/rejected": -1.5682532787322998, | |
| "step": 150 | |
| }, | |
| { | |
| "epoch": 0.25117739403453687, | |
| "grad_norm": 49.06765782836144, | |
| "learning_rate": 4.661628684945851e-07, | |
| "logits/chosen": -2.573646068572998, | |
| "logits/rejected": -2.5104339122772217, | |
| "logps/chosen": -370.089599609375, | |
| "logps/rejected": -390.0255126953125, | |
| "loss": 0.4023, | |
| "rewards/accuracies": 0.8500000238418579, | |
| "rewards/chosen": -0.8807455897331238, | |
| "rewards/margins": 0.7309385538101196, | |
| "rewards/rejected": -1.6116840839385986, | |
| "step": 160 | |
| }, | |
| { | |
| "epoch": 0.2668759811616955, | |
| "grad_norm": 54.18143326196735, | |
| "learning_rate": 4.5895560292945996e-07, | |
| "logits/chosen": -2.5094046592712402, | |
| "logits/rejected": -2.519862413406372, | |
| "logps/chosen": -328.95098876953125, | |
| "logps/rejected": -409.6293029785156, | |
| "loss": 0.4229, | |
| "rewards/accuracies": 0.887499988079071, | |
| "rewards/chosen": -0.8252763748168945, | |
| "rewards/margins": 0.8684083223342896, | |
| "rewards/rejected": -1.6936845779418945, | |
| "step": 170 | |
| }, | |
| { | |
| "epoch": 0.282574568288854, | |
| "grad_norm": 47.86539813882727, | |
| "learning_rate": 4.5112037194555876e-07, | |
| "logits/chosen": -2.4061732292175293, | |
| "logits/rejected": -2.336141586303711, | |
| "logps/chosen": -340.967529296875, | |
| "logps/rejected": -416.18438720703125, | |
| "loss": 0.4129, | |
| "rewards/accuracies": 0.862500011920929, | |
| "rewards/chosen": -0.9469585418701172, | |
| "rewards/margins": 1.003477692604065, | |
| "rewards/rejected": -1.9504363536834717, | |
| "step": 180 | |
| }, | |
| { | |
| "epoch": 0.29827315541601257, | |
| "grad_norm": 47.40203923720411, | |
| "learning_rate": 4.426807224305315e-07, | |
| "logits/chosen": -2.434738874435425, | |
| "logits/rejected": -2.4182467460632324, | |
| "logps/chosen": -390.0809631347656, | |
| "logps/rejected": -541.5980224609375, | |
| "loss": 0.39, | |
| "rewards/accuracies": 0.925000011920929, | |
| "rewards/chosen": -0.907140851020813, | |
| "rewards/margins": 1.477752447128296, | |
| "rewards/rejected": -2.3848931789398193, | |
| "step": 190 | |
| }, | |
| { | |
| "epoch": 0.3139717425431711, | |
| "grad_norm": 59.480022233916614, | |
| "learning_rate": 4.3366201770542687e-07, | |
| "logits/chosen": -2.3925621509552, | |
| "logits/rejected": -2.3594958782196045, | |
| "logps/chosen": -399.2648010253906, | |
| "logps/rejected": -485.82061767578125, | |
| "loss": 0.3722, | |
| "rewards/accuracies": 0.8500000238418579, | |
| "rewards/chosen": -1.242919683456421, | |
| "rewards/margins": 1.0100038051605225, | |
| "rewards/rejected": -2.2529234886169434, | |
| "step": 200 | |
| }, | |
| { | |
| "epoch": 0.32967032967032966, | |
| "grad_norm": 54.533637965345, | |
| "learning_rate": 4.2409136130137845e-07, | |
| "logits/chosen": -2.391516923904419, | |
| "logits/rejected": -2.3367998600006104, | |
| "logps/chosen": -410.27752685546875, | |
| "logps/rejected": -488.3935546875, | |
| "loss": 0.3723, | |
| "rewards/accuracies": 0.824999988079071, | |
| "rewards/chosen": -1.2089805603027344, | |
| "rewards/margins": 1.3162455558776855, | |
| "rewards/rejected": -2.52522611618042, | |
| "step": 210 | |
| }, | |
| { | |
| "epoch": 0.3453689167974882, | |
| "grad_norm": 134.3137263377747, | |
| "learning_rate": 4.1399751550651084e-07, | |
| "logits/chosen": -2.2474474906921387, | |
| "logits/rejected": -2.273486852645874, | |
| "logps/chosen": -422.23419189453125, | |
| "logps/rejected": -493.7196350097656, | |
| "loss": 0.4143, | |
| "rewards/accuracies": 0.7875000238418579, | |
| "rewards/chosen": -1.3034000396728516, | |
| "rewards/margins": 1.1734570264816284, | |
| "rewards/rejected": -2.4768571853637695, | |
| "step": 220 | |
| }, | |
| { | |
| "epoch": 0.36106750392464676, | |
| "grad_norm": 51.06322946002842, | |
| "learning_rate": 4.034108149278543e-07, | |
| "logits/chosen": -2.246279716491699, | |
| "logits/rejected": -2.218812942504883, | |
| "logps/chosen": -396.13433837890625, | |
| "logps/rejected": -494.9988708496094, | |
| "loss": 0.3583, | |
| "rewards/accuracies": 0.8999999761581421, | |
| "rewards/chosen": -1.4342329502105713, | |
| "rewards/margins": 1.3676201105117798, | |
| "rewards/rejected": -2.8018529415130615, | |
| "step": 230 | |
| }, | |
| { | |
| "epoch": 0.37676609105180536, | |
| "grad_norm": 53.106943652386406, | |
| "learning_rate": 3.923630753280357e-07, | |
| "logits/chosen": -2.296950578689575, | |
| "logits/rejected": -2.193643808364868, | |
| "logps/chosen": -446.6080017089844, | |
| "logps/rejected": -504.3912658691406, | |
| "loss": 0.439, | |
| "rewards/accuracies": 0.7875000238418579, | |
| "rewards/chosen": -1.5059573650360107, | |
| "rewards/margins": 1.111060380935669, | |
| "rewards/rejected": -2.617017984390259, | |
| "step": 240 | |
| }, | |
| { | |
| "epoch": 0.3924646781789639, | |
| "grad_norm": 31.823002355070333, | |
| "learning_rate": 3.8088749801071496e-07, | |
| "logits/chosen": -2.179251194000244, | |
| "logits/rejected": -2.139425277709961, | |
| "logps/chosen": -373.7418518066406, | |
| "logps/rejected": -501.843505859375, | |
| "loss": 0.369, | |
| "rewards/accuracies": 0.8500000238418579, | |
| "rewards/chosen": -1.2382627725601196, | |
| "rewards/margins": 1.2532708644866943, | |
| "rewards/rejected": -2.4915337562561035, | |
| "step": 250 | |
| }, | |
| { | |
| "epoch": 0.40816326530612246, | |
| "grad_norm": 57.958160333752495, | |
| "learning_rate": 3.6901857004211443e-07, | |
| "logits/chosen": -2.2685585021972656, | |
| "logits/rejected": -2.1836700439453125, | |
| "logps/chosen": -438.3352966308594, | |
| "logps/rejected": -474.6566467285156, | |
| "loss": 0.3729, | |
| "rewards/accuracies": 0.8999999761581421, | |
| "rewards/chosen": -1.387054443359375, | |
| "rewards/margins": 1.218056559562683, | |
| "rewards/rejected": -2.6051113605499268, | |
| "step": 260 | |
| }, | |
| { | |
| "epoch": 0.423861852433281, | |
| "grad_norm": 78.46346160026032, | |
| "learning_rate": 3.5679196060850034e-07, | |
| "logits/chosen": -2.194472312927246, | |
| "logits/rejected": -2.1565122604370117, | |
| "logps/chosen": -470.94024658203125, | |
| "logps/rejected": -557.4560546875, | |
| "loss": 0.3988, | |
| "rewards/accuracies": 0.800000011920929, | |
| "rewards/chosen": -1.681726098060608, | |
| "rewards/margins": 1.0995349884033203, | |
| "rewards/rejected": -2.7812609672546387, | |
| "step": 270 | |
| }, | |
| { | |
| "epoch": 0.43956043956043955, | |
| "grad_norm": 73.42817989092451, | |
| "learning_rate": 3.4424441382108826e-07, | |
| "logits/chosen": -2.0928378105163574, | |
| "logits/rejected": -2.080376148223877, | |
| "logps/chosen": -437.2099609375, | |
| "logps/rejected": -575.6405029296875, | |
| "loss": 0.3881, | |
| "rewards/accuracies": 0.7749999761581421, | |
| "rewards/chosen": -1.617456078529358, | |
| "rewards/margins": 1.3479641675949097, | |
| "rewards/rejected": -2.9654202461242676, | |
| "step": 280 | |
| }, | |
| { | |
| "epoch": 0.4552590266875981, | |
| "grad_norm": 62.6558696654114, | |
| "learning_rate": 3.314136382905234e-07, | |
| "logits/chosen": -2.0131213665008545, | |
| "logits/rejected": -1.9851818084716797, | |
| "logps/chosen": -512.6057739257812, | |
| "logps/rejected": -646.832275390625, | |
| "loss": 0.4216, | |
| "rewards/accuracies": 0.7875000238418579, | |
| "rewards/chosen": -2.3018593788146973, | |
| "rewards/margins": 1.346274971961975, | |
| "rewards/rejected": -3.648134708404541, | |
| "step": 290 | |
| }, | |
| { | |
| "epoch": 0.47095761381475665, | |
| "grad_norm": 78.12598055564801, | |
| "learning_rate": 3.1833819380279023e-07, | |
| "logits/chosen": -2.0949795246124268, | |
| "logits/rejected": -1.9874067306518555, | |
| "logps/chosen": -488.44384765625, | |
| "logps/rejected": -571.2259521484375, | |
| "loss": 0.386, | |
| "rewards/accuracies": 0.8125, | |
| "rewards/chosen": -2.06516695022583, | |
| "rewards/margins": 1.3252384662628174, | |
| "rewards/rejected": -3.3904056549072266, | |
| "step": 300 | |
| }, | |
| { | |
| "epoch": 0.48665620094191525, | |
| "grad_norm": 48.25836081307725, | |
| "learning_rate": 3.0505737543712275e-07, | |
| "logits/chosen": -2.1759631633758545, | |
| "logits/rejected": -2.095853805541992, | |
| "logps/chosen": -437.9419860839844, | |
| "logps/rejected": -518.4359130859375, | |
| "loss": 0.3903, | |
| "rewards/accuracies": 0.9125000238418579, | |
| "rewards/chosen": -1.5794050693511963, | |
| "rewards/margins": 1.1850736141204834, | |
| "rewards/rejected": -2.7644786834716797, | |
| "step": 310 | |
| }, | |
| { | |
| "epoch": 0.5023547880690737, | |
| "grad_norm": 60.78980701157502, | |
| "learning_rate": 2.9161109547416667e-07, | |
| "logits/chosen": -2.1254727840423584, | |
| "logits/rejected": -2.0983071327209473, | |
| "logps/chosen": -487.24383544921875, | |
| "logps/rejected": -561.2527465820312, | |
| "loss": 0.3585, | |
| "rewards/accuracies": 0.862500011920929, | |
| "rewards/chosen": -1.5642211437225342, | |
| "rewards/margins": 1.3017809391021729, | |
| "rewards/rejected": -2.866001844406128, | |
| "step": 320 | |
| }, | |
| { | |
| "epoch": 0.5180533751962323, | |
| "grad_norm": 59.58938057948085, | |
| "learning_rate": 2.780397634492949e-07, | |
| "logits/chosen": -2.1095099449157715, | |
| "logits/rejected": -2.0623815059661865, | |
| "logps/chosen": -420.601318359375, | |
| "logps/rejected": -561.5743408203125, | |
| "loss": 0.4054, | |
| "rewards/accuracies": 0.8500000238418579, | |
| "rewards/chosen": -1.7670698165893555, | |
| "rewards/margins": 1.3623578548431396, | |
| "rewards/rejected": -3.129427671432495, | |
| "step": 330 | |
| }, | |
| { | |
| "epoch": 0.533751962323391, | |
| "grad_norm": 59.60421047453409, | |
| "learning_rate": 2.6438416471154273e-07, | |
| "logits/chosen": -2.105985641479492, | |
| "logits/rejected": -2.1100611686706543, | |
| "logps/chosen": -455.4019470214844, | |
| "logps/rejected": -597.48583984375, | |
| "loss": 0.3559, | |
| "rewards/accuracies": 0.7875000238418579, | |
| "rewards/chosen": -1.826162338256836, | |
| "rewards/margins": 1.1689088344573975, | |
| "rewards/rejected": -2.9950711727142334, | |
| "step": 340 | |
| }, | |
| { | |
| "epoch": 0.5494505494505495, | |
| "grad_norm": 64.80810897640455, | |
| "learning_rate": 2.5068533785312666e-07, | |
| "logits/chosen": -2.1501102447509766, | |
| "logits/rejected": -2.098646640777588, | |
| "logps/chosen": -398.44293212890625, | |
| "logps/rejected": -512.3727416992188, | |
| "loss": 0.3817, | |
| "rewards/accuracies": 0.8374999761581421, | |
| "rewards/chosen": -1.6598011255264282, | |
| "rewards/margins": 1.3402740955352783, | |
| "rewards/rejected": -3.000075101852417, | |
| "step": 350 | |
| }, | |
| { | |
| "epoch": 0.565149136577708, | |
| "grad_norm": 66.30622337743506, | |
| "learning_rate": 2.3698445137790258e-07, | |
| "logits/chosen": -2.183788776397705, | |
| "logits/rejected": -2.1399548053741455, | |
| "logps/chosen": -450.28790283203125, | |
| "logps/rejected": -570.9381103515625, | |
| "loss": 0.3441, | |
| "rewards/accuracies": 0.8999999761581421, | |
| "rewards/chosen": -1.7852160930633545, | |
| "rewards/margins": 1.4079563617706299, | |
| "rewards/rejected": -3.1931726932525635, | |
| "step": 360 | |
| }, | |
| { | |
| "epoch": 0.5808477237048666, | |
| "grad_norm": 74.385047879014, | |
| "learning_rate": 2.2332267997940513e-07, | |
| "logits/chosen": -1.9558976888656616, | |
| "logits/rejected": -1.8889505863189697, | |
| "logps/chosen": -472.29046630859375, | |
| "logps/rejected": -564.1910400390625, | |
| "loss": 0.4144, | |
| "rewards/accuracies": 0.8125, | |
| "rewards/chosen": -2.128002405166626, | |
| "rewards/margins": 1.2228187322616577, | |
| "rewards/rejected": -3.3508212566375732, | |
| "step": 370 | |
| }, | |
| { | |
| "epoch": 0.5965463108320251, | |
| "grad_norm": 98.38326705187197, | |
| "learning_rate": 2.0974108080028692e-07, | |
| "logits/chosen": -2.1048827171325684, | |
| "logits/rejected": -2.014101266860962, | |
| "logps/chosen": -472.6429138183594, | |
| "logps/rejected": -552.962646484375, | |
| "loss": 0.362, | |
| "rewards/accuracies": 0.887499988079071, | |
| "rewards/chosen": -1.7709888219833374, | |
| "rewards/margins": 1.5231949090957642, | |
| "rewards/rejected": -3.2941837310791016, | |
| "step": 380 | |
| }, | |
| { | |
| "epoch": 0.6122448979591837, | |
| "grad_norm": 28.798007236746653, | |
| "learning_rate": 1.962804700450265e-07, | |
| "logits/chosen": -2.229259490966797, | |
| "logits/rejected": -2.136399269104004, | |
| "logps/chosen": -404.88970947265625, | |
| "logps/rejected": -605.4417724609375, | |
| "loss": 0.2913, | |
| "rewards/accuracies": 0.9125000238418579, | |
| "rewards/chosen": -1.31179940700531, | |
| "rewards/margins": 2.022050142288208, | |
| "rewards/rejected": -3.3338496685028076, | |
| "step": 390 | |
| }, | |
| { | |
| "epoch": 0.6279434850863422, | |
| "grad_norm": 47.93461485983235, | |
| "learning_rate": 1.8298130031671972e-07, | |
| "logits/chosen": -2.066481351852417, | |
| "logits/rejected": -2.0052218437194824, | |
| "logps/chosen": -461.60711669921875, | |
| "logps/rejected": -593.0902099609375, | |
| "loss": 0.3182, | |
| "rewards/accuracies": 0.887499988079071, | |
| "rewards/chosen": -1.50743567943573, | |
| "rewards/margins": 1.5388964414596558, | |
| "rewards/rejected": -3.0463321208953857, | |
| "step": 400 | |
| }, | |
| { | |
| "epoch": 0.6436420722135008, | |
| "grad_norm": 81.64181086704542, | |
| "learning_rate": 1.6988353904658492e-07, | |
| "logits/chosen": -2.079695224761963, | |
| "logits/rejected": -2.0234522819519043, | |
| "logps/chosen": -435.9869079589844, | |
| "logps/rejected": -548.7467041015625, | |
| "loss": 0.5391, | |
| "rewards/accuracies": 0.7124999761581421, | |
| "rewards/chosen": -1.8887078762054443, | |
| "rewards/margins": 1.0673691034317017, | |
| "rewards/rejected": -2.9560768604278564, | |
| "step": 410 | |
| }, | |
| { | |
| "epoch": 0.6593406593406593, | |
| "grad_norm": 99.80504939653643, | |
| "learning_rate": 1.570265483815364e-07, | |
| "logits/chosen": -2.146688222885132, | |
| "logits/rejected": -2.1056787967681885, | |
| "logps/chosen": -441.99456787109375, | |
| "logps/rejected": -486.8045959472656, | |
| "loss": 0.6064, | |
| "rewards/accuracies": 0.625, | |
| "rewards/chosen": -1.862978219985962, | |
| "rewards/margins": 0.5048335790634155, | |
| "rewards/rejected": -2.367811679840088, | |
| "step": 420 | |
| }, | |
| { | |
| "epoch": 0.6750392464678179, | |
| "grad_norm": 85.01436657258617, | |
| "learning_rate": 1.444489668907914e-07, | |
| "logits/chosen": -2.0182952880859375, | |
| "logits/rejected": -2.0226969718933105, | |
| "logps/chosen": -346.7263488769531, | |
| "logps/rejected": -505.09619140625, | |
| "loss": 0.5198, | |
| "rewards/accuracies": 0.800000011920929, | |
| "rewards/chosen": -1.6009681224822998, | |
| "rewards/margins": 1.235584020614624, | |
| "rewards/rejected": -2.836552143096924, | |
| "step": 430 | |
| }, | |
| { | |
| "epoch": 0.6907378335949764, | |
| "grad_norm": 75.62069078299841, | |
| "learning_rate": 1.3218859344701632e-07, | |
| "logits/chosen": -2.2084121704101562, | |
| "logits/rejected": -2.1915016174316406, | |
| "logps/chosen": -412.80224609375, | |
| "logps/rejected": -515.23095703125, | |
| "loss": 0.5238, | |
| "rewards/accuracies": 0.7875000238418579, | |
| "rewards/chosen": -1.4952690601348877, | |
| "rewards/margins": 1.0581085681915283, | |
| "rewards/rejected": -2.553377628326416, | |
| "step": 440 | |
| }, | |
| { | |
| "epoch": 0.706436420722135, | |
| "grad_norm": 150.50609049436147, | |
| "learning_rate": 1.202822736309758e-07, | |
| "logits/chosen": -2.1147706508636475, | |
| "logits/rejected": -2.0880684852600098, | |
| "logps/chosen": -442.6726989746094, | |
| "logps/rejected": -475.3531799316406, | |
| "loss": 0.5647, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": -1.6886441707611084, | |
| "rewards/margins": 0.8048279881477356, | |
| "rewards/rejected": -2.4934723377227783, | |
| "step": 450 | |
| }, | |
| { | |
| "epoch": 0.7221350078492935, | |
| "grad_norm": 76.56164006248804, | |
| "learning_rate": 1.0876578900107053e-07, | |
| "logits/chosen": -2.2221930027008057, | |
| "logits/rejected": -2.1485908031463623, | |
| "logps/chosen": -464.043701171875, | |
| "logps/rejected": -508.6534118652344, | |
| "loss": 0.522, | |
| "rewards/accuracies": 0.6875, | |
| "rewards/chosen": -1.763662338256836, | |
| "rewards/margins": 0.7875553965568542, | |
| "rewards/rejected": -2.551217555999756, | |
| "step": 460 | |
| }, | |
| { | |
| "epoch": 0.7378335949764521, | |
| "grad_norm": 74.89323927902005, | |
| "learning_rate": 9.767374956053584e-08, | |
| "logits/chosen": -2.205277919769287, | |
| "logits/rejected": -2.18218994140625, | |
| "logps/chosen": -412.97686767578125, | |
| "logps/rejected": -543.1309204101562, | |
| "loss": 0.5596, | |
| "rewards/accuracies": 0.7749999761581421, | |
| "rewards/chosen": -1.553432822227478, | |
| "rewards/margins": 0.9423764944076538, | |
| "rewards/rejected": -2.495809316635132, | |
| "step": 470 | |
| }, | |
| { | |
| "epoch": 0.7535321821036107, | |
| "grad_norm": 87.37555939932336, | |
| "learning_rate": 8.70394897454659e-08, | |
| "logits/chosen": -2.2252628803253174, | |
| "logits/rejected": -2.1848928928375244, | |
| "logps/chosen": -382.27752685546875, | |
| "logps/rejected": -457.525390625, | |
| "loss": 0.547, | |
| "rewards/accuracies": 0.612500011920929, | |
| "rewards/chosen": -1.6248447895050049, | |
| "rewards/margins": 0.4326212406158447, | |
| "rewards/rejected": -2.0574657917022705, | |
| "step": 480 | |
| }, | |
| { | |
| "epoch": 0.7692307692307693, | |
| "grad_norm": 93.79826007492892, | |
| "learning_rate": 7.689496824624525e-08, | |
| "logits/chosen": -2.2677321434020996, | |
| "logits/rejected": -2.212900161743164, | |
| "logps/chosen": -403.1687316894531, | |
| "logps/rejected": -456.62615966796875, | |
| "loss": 0.5459, | |
| "rewards/accuracies": 0.7124999761581421, | |
| "rewards/chosen": -1.4313546419143677, | |
| "rewards/margins": 0.6928239464759827, | |
| "rewards/rejected": -2.124178647994995, | |
| "step": 490 | |
| }, | |
| { | |
| "epoch": 0.7849293563579278, | |
| "grad_norm": 56.75850068630019, | |
| "learning_rate": 6.727067196345099e-08, | |
| "logits/chosen": -2.3276655673980713, | |
| "logits/rejected": -2.2189202308654785, | |
| "logps/chosen": -438.74267578125, | |
| "logps/rejected": -478.62298583984375, | |
| "loss": 0.5453, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": -1.3515713214874268, | |
| "rewards/margins": 1.0120915174484253, | |
| "rewards/rejected": -2.3636624813079834, | |
| "step": 500 | |
| }, | |
| { | |
| "epoch": 0.8006279434850864, | |
| "grad_norm": 97.05649014340479, | |
| "learning_rate": 5.8195524386862374e-08, | |
| "logits/chosen": -2.311316967010498, | |
| "logits/rejected": -2.242011547088623, | |
| "logps/chosen": -362.28399658203125, | |
| "logps/rejected": -465.5743103027344, | |
| "loss": 0.4852, | |
| "rewards/accuracies": 0.800000011920929, | |
| "rewards/chosen": -1.409887433052063, | |
| "rewards/margins": 0.9168035387992859, | |
| "rewards/rejected": -2.326690912246704, | |
| "step": 510 | |
| }, | |
| { | |
| "epoch": 0.8163265306122449, | |
| "grad_norm": 127.5887173764836, | |
| "learning_rate": 4.969679867292276e-08, | |
| "logits/chosen": -2.2007040977478027, | |
| "logits/rejected": -2.138988971710205, | |
| "logps/chosen": -413.65447998046875, | |
| "logps/rejected": -489.22540283203125, | |
| "loss": 0.4985, | |
| "rewards/accuracies": 0.762499988079071, | |
| "rewards/chosen": -1.3157858848571777, | |
| "rewards/margins": 1.0105135440826416, | |
| "rewards/rejected": -2.3262994289398193, | |
| "step": 520 | |
| }, | |
| { | |
| "epoch": 0.8320251177394035, | |
| "grad_norm": 92.31568060854485, | |
| "learning_rate": 4.180003568187776e-08, | |
| "logits/chosen": -2.2751996517181396, | |
| "logits/rejected": -2.2204182147979736, | |
| "logps/chosen": -437.9576110839844, | |
| "logps/rejected": -478.0186462402344, | |
| "loss": 0.5193, | |
| "rewards/accuracies": 0.699999988079071, | |
| "rewards/chosen": -1.3832905292510986, | |
| "rewards/margins": 0.656178891658783, | |
| "rewards/rejected": -2.0394692420959473, | |
| "step": 530 | |
| }, | |
| { | |
| "epoch": 0.847723704866562, | |
| "grad_norm": 93.73800900430595, | |
| "learning_rate": 3.452896722091128e-08, | |
| "logits/chosen": -2.2862913608551025, | |
| "logits/rejected": -2.1965749263763428, | |
| "logps/chosen": -418.68218994140625, | |
| "logps/rejected": -488.07208251953125, | |
| "loss": 0.549, | |
| "rewards/accuracies": 0.762499988079071, | |
| "rewards/chosen": -1.4643903970718384, | |
| "rewards/margins": 0.8227276802062988, | |
| "rewards/rejected": -2.2871181964874268, | |
| "step": 540 | |
| }, | |
| { | |
| "epoch": 0.8634222919937206, | |
| "grad_norm": 74.68277239397396, | |
| "learning_rate": 2.7905444723949762e-08, | |
| "logits/chosen": -2.2326064109802246, | |
| "logits/rejected": -2.1876773834228516, | |
| "logps/chosen": -434.93377685546875, | |
| "logps/rejected": -486.3379821777344, | |
| "loss": 0.5464, | |
| "rewards/accuracies": 0.7124999761581421, | |
| "rewards/chosen": -1.5688042640686035, | |
| "rewards/margins": 0.6943384408950806, | |
| "rewards/rejected": -2.2631428241729736, | |
| "step": 550 | |
| }, | |
| { | |
| "epoch": 0.8791208791208791, | |
| "grad_norm": 98.95496114421825, | |
| "learning_rate": 2.194937358247506e-08, | |
| "logits/chosen": -2.314251661300659, | |
| "logits/rejected": -2.282222032546997, | |
| "logps/chosen": -444.4891662597656, | |
| "logps/rejected": -482.2521057128906, | |
| "loss": 0.5058, | |
| "rewards/accuracies": 0.7875000238418579, | |
| "rewards/chosen": -1.4890687465667725, | |
| "rewards/margins": 0.7982088327407837, | |
| "rewards/rejected": -2.2872776985168457, | |
| "step": 560 | |
| }, | |
| { | |
| "epoch": 0.8948194662480377, | |
| "grad_norm": 93.93456697380334, | |
| "learning_rate": 1.6678653324693787e-08, | |
| "logits/chosen": -2.271334171295166, | |
| "logits/rejected": -2.243417263031006, | |
| "logps/chosen": -379.41131591796875, | |
| "logps/rejected": -474.8644104003906, | |
| "loss": 0.5554, | |
| "rewards/accuracies": 0.737500011920929, | |
| "rewards/chosen": -1.2463254928588867, | |
| "rewards/margins": 1.0109448432922363, | |
| "rewards/rejected": -2.257270336151123, | |
| "step": 570 | |
| }, | |
| { | |
| "epoch": 0.9105180533751962, | |
| "grad_norm": 63.908735732483215, | |
| "learning_rate": 1.2109123822844653e-08, | |
| "logits/chosen": -2.2195041179656982, | |
| "logits/rejected": -2.152104139328003, | |
| "logps/chosen": -424.376220703125, | |
| "logps/rejected": -493.041748046875, | |
| "loss": 0.5407, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": -1.484989047050476, | |
| "rewards/margins": 0.8357736468315125, | |
| "rewards/rejected": -2.3207626342773438, | |
| "step": 580 | |
| }, | |
| { | |
| "epoch": 0.9262166405023547, | |
| "grad_norm": 77.29245256601979, | |
| "learning_rate": 8.254517690300944e-09, | |
| "logits/chosen": -2.1921186447143555, | |
| "logits/rejected": -2.160266876220703, | |
| "logps/chosen": -358.38665771484375, | |
| "logps/rejected": -419.24053955078125, | |
| "loss": 0.5065, | |
| "rewards/accuracies": 0.699999988079071, | |
| "rewards/chosen": -1.4891427755355835, | |
| "rewards/margins": 0.6564360857009888, | |
| "rewards/rejected": -2.1455788612365723, | |
| "step": 590 | |
| }, | |
| { | |
| "epoch": 0.9419152276295133, | |
| "grad_norm": 85.49976100967837, | |
| "learning_rate": 5.126419011529992e-09, | |
| "logits/chosen": -2.247999668121338, | |
| "logits/rejected": -2.1932740211486816, | |
| "logps/chosen": -437.494384765625, | |
| "logps/rejected": -485.1595764160156, | |
| "loss": 0.4877, | |
| "rewards/accuracies": 0.737500011920929, | |
| "rewards/chosen": -1.302431583404541, | |
| "rewards/margins": 0.9456756711006165, | |
| "rewards/rejected": -2.2481071949005127, | |
| "step": 600 | |
| }, | |
| { | |
| "epoch": 0.957613814756672, | |
| "grad_norm": 101.21792438469406, | |
| "learning_rate": 2.734228528934679e-09, | |
| "logits/chosen": -2.12449312210083, | |
| "logits/rejected": -2.1045031547546387, | |
| "logps/chosen": -403.0607604980469, | |
| "logps/rejected": -488.42059326171875, | |
| "loss": 0.5817, | |
| "rewards/accuracies": 0.6875, | |
| "rewards/chosen": -1.3698210716247559, | |
| "rewards/margins": 0.6692644953727722, | |
| "rewards/rejected": -2.0390853881835938, | |
| "step": 610 | |
| }, | |
| { | |
| "epoch": 0.9733124018838305, | |
| "grad_norm": 62.863474044060304, | |
| "learning_rate": 1.0851353912008642e-09, | |
| "logits/chosen": -2.2791266441345215, | |
| "logits/rejected": -2.18548583984375, | |
| "logps/chosen": -420.2642517089844, | |
| "logps/rejected": -431.43804931640625, | |
| "loss": 0.5458, | |
| "rewards/accuracies": 0.737500011920929, | |
| "rewards/chosen": -1.357425332069397, | |
| "rewards/margins": 0.5371707677841187, | |
| "rewards/rejected": -1.8945963382720947, | |
| "step": 620 | |
| }, | |
| { | |
| "epoch": 0.989010989010989, | |
| "grad_norm": 82.02664725646991, | |
| "learning_rate": 1.840955480532924e-10, | |
| "logits/chosen": -2.251556873321533, | |
| "logits/rejected": -2.2051682472229004, | |
| "logps/chosen": -372.6395568847656, | |
| "logps/rejected": -425.1973571777344, | |
| "loss": 0.5177, | |
| "rewards/accuracies": 0.762499988079071, | |
| "rewards/chosen": -1.1917184591293335, | |
| "rewards/margins": 0.899652361869812, | |
| "rewards/rejected": -2.0913710594177246, | |
| "step": 630 | |
| }, | |
| { | |
| "epoch": 1.0, | |
| "step": 637, | |
| "total_flos": 0.0, | |
| "train_loss": 0.030315694479590496, | |
| "train_runtime": 321.4302, | |
| "train_samples_per_second": 47.547, | |
| "train_steps_per_second": 1.982 | |
| } | |
| ], | |
| "logging_steps": 10, | |
| "max_steps": 637, | |
| "num_input_tokens_seen": 0, | |
| "num_train_epochs": 1, | |
| "save_steps": 100, | |
| "stateful_callbacks": { | |
| "TrainerControl": { | |
| "args": { | |
| "should_epoch_stop": false, | |
| "should_evaluate": false, | |
| "should_log": false, | |
| "should_save": true, | |
| "should_training_stop": false | |
| }, | |
| "attributes": {} | |
| } | |
| }, | |
| "total_flos": 0.0, | |
| "train_batch_size": 8, | |
| "trial_name": null, | |
| "trial_params": null | |
| } | |