| { | |
| "best_metric": null, | |
| "best_model_checkpoint": null, | |
| "epoch": 0.9991503823279524, | |
| "eval_steps": 500, | |
| "global_step": 147, | |
| "is_hyper_param_search": false, | |
| "is_local_process_zero": true, | |
| "is_world_process_zero": true, | |
| "log_history": [ | |
| { | |
| "epoch": 0.01, | |
| "grad_norm": 48.577925295489194, | |
| "learning_rate": 3.3333333333333334e-08, | |
| "logps/chosen": -67.63638305664062, | |
| "logps/rejected": -55.72713851928711, | |
| "loss": 0.6931, | |
| "losses/dpo": 0.6931471824645996, | |
| "losses/sft": 0.34192484617233276, | |
| "losses/total": 0.6931471824645996, | |
| "ref_logps/chosen": -67.63638305664062, | |
| "ref_logps/rejected": -55.72713851928711, | |
| "rewards/accuracies": 0.0, | |
| "rewards/chosen": 0.0, | |
| "rewards/margins": 0.0, | |
| "rewards/rejected": 0.0, | |
| "step": 1 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "grad_norm": 31.47832995229055, | |
| "learning_rate": 6.666666666666667e-08, | |
| "logps/chosen": -68.23696899414062, | |
| "logps/rejected": -57.0877799987793, | |
| "loss": 0.6931, | |
| "losses/dpo": 0.6931471824645996, | |
| "losses/sft": 0.35053497552871704, | |
| "losses/total": 0.6931471824645996, | |
| "ref_logps/chosen": -68.23696899414062, | |
| "ref_logps/rejected": -57.0877799987793, | |
| "rewards/accuracies": 0.0, | |
| "rewards/chosen": 0.0, | |
| "rewards/margins": 0.0, | |
| "rewards/rejected": 0.0, | |
| "step": 2 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "grad_norm": 50.46042718530731, | |
| "learning_rate": 1e-07, | |
| "logps/chosen": -63.95356750488281, | |
| "logps/rejected": -56.19622802734375, | |
| "loss": 0.6913, | |
| "losses/dpo": 0.7034662961959839, | |
| "losses/sft": 0.34674179553985596, | |
| "losses/total": 0.7034662961959839, | |
| "ref_logps/chosen": -64.00633239746094, | |
| "ref_logps/rejected": -56.198883056640625, | |
| "rewards/accuracies": 0.5, | |
| "rewards/chosen": 0.005276843905448914, | |
| "rewards/margins": 0.005011209286749363, | |
| "rewards/rejected": 0.0002656341530382633, | |
| "step": 3 | |
| }, | |
| { | |
| "epoch": 0.03, | |
| "grad_norm": 32.81062658932443, | |
| "learning_rate": 1.3333333333333334e-07, | |
| "logps/chosen": -61.69102096557617, | |
| "logps/rejected": -54.53285217285156, | |
| "loss": 0.6963, | |
| "losses/dpo": 0.6837673783302307, | |
| "losses/sft": 0.3563593626022339, | |
| "losses/total": 0.6837673783302307, | |
| "ref_logps/chosen": -61.65943145751953, | |
| "ref_logps/rejected": -54.55246353149414, | |
| "rewards/accuracies": 0.390625, | |
| "rewards/chosen": -0.003159371204674244, | |
| "rewards/margins": -0.005120169371366501, | |
| "rewards/rejected": 0.0019607983995229006, | |
| "step": 4 | |
| }, | |
| { | |
| "epoch": 0.03, | |
| "grad_norm": 33.80575482734628, | |
| "learning_rate": 1.6666666666666665e-07, | |
| "logps/chosen": -66.74890899658203, | |
| "logps/rejected": -54.70701599121094, | |
| "loss": 0.6925, | |
| "losses/dpo": 0.6997189521789551, | |
| "losses/sft": 0.3123997151851654, | |
| "losses/total": 0.6997189521789551, | |
| "ref_logps/chosen": -66.69550323486328, | |
| "ref_logps/rejected": -54.629825592041016, | |
| "rewards/accuracies": 0.53125, | |
| "rewards/chosen": -0.005340744741261005, | |
| "rewards/margins": 0.0023781214840710163, | |
| "rewards/rejected": -0.00771886482834816, | |
| "step": 5 | |
| }, | |
| { | |
| "epoch": 0.04, | |
| "grad_norm": 39.07641400536226, | |
| "learning_rate": 2e-07, | |
| "logps/chosen": -68.85008239746094, | |
| "logps/rejected": -58.72641372680664, | |
| "loss": 0.6966, | |
| "losses/dpo": 0.6637319922447205, | |
| "losses/sft": 0.3361469805240631, | |
| "losses/total": 0.6637319922447205, | |
| "ref_logps/chosen": -68.81605529785156, | |
| "ref_logps/rejected": -58.74713134765625, | |
| "rewards/accuracies": 0.453125, | |
| "rewards/chosen": -0.003402549307793379, | |
| "rewards/margins": -0.005474450532346964, | |
| "rewards/rejected": 0.0020719014573842287, | |
| "step": 6 | |
| }, | |
| { | |
| "epoch": 0.05, | |
| "grad_norm": 34.000425983037914, | |
| "learning_rate": 2.3333333333333333e-07, | |
| "logps/chosen": -61.8620491027832, | |
| "logps/rejected": -51.837215423583984, | |
| "loss": 0.694, | |
| "losses/dpo": 0.6815477013587952, | |
| "losses/sft": 0.34513574838638306, | |
| "losses/total": 0.6815477013587952, | |
| "ref_logps/chosen": -61.80104446411133, | |
| "ref_logps/rejected": -51.780853271484375, | |
| "rewards/accuracies": 0.515625, | |
| "rewards/chosen": -0.006101294886320829, | |
| "rewards/margins": -0.0004650282207876444, | |
| "rewards/rejected": -0.005636268295347691, | |
| "step": 7 | |
| }, | |
| { | |
| "epoch": 0.05, | |
| "grad_norm": 39.18548739683539, | |
| "learning_rate": 2.6666666666666667e-07, | |
| "logps/chosen": -65.5419921875, | |
| "logps/rejected": -53.712867736816406, | |
| "loss": 0.6864, | |
| "losses/dpo": 0.6901150345802307, | |
| "losses/sft": 0.3755946159362793, | |
| "losses/total": 0.6901150345802307, | |
| "ref_logps/chosen": -65.61614227294922, | |
| "ref_logps/rejected": -53.64268493652344, | |
| "rewards/accuracies": 0.65625, | |
| "rewards/chosen": 0.007414784282445908, | |
| "rewards/margins": 0.014433156698942184, | |
| "rewards/rejected": -0.007018372416496277, | |
| "step": 8 | |
| }, | |
| { | |
| "epoch": 0.06, | |
| "grad_norm": 34.05787792436153, | |
| "learning_rate": 3e-07, | |
| "logps/chosen": -66.96398162841797, | |
| "logps/rejected": -52.923221588134766, | |
| "loss": 0.7011, | |
| "losses/dpo": 0.6919645071029663, | |
| "losses/sft": 0.34879329800605774, | |
| "losses/total": 0.6919645071029663, | |
| "ref_logps/chosen": -66.94356536865234, | |
| "ref_logps/rejected": -53.04799270629883, | |
| "rewards/accuracies": 0.4375, | |
| "rewards/chosen": -0.0020427405834198, | |
| "rewards/margins": -0.014519838616251945, | |
| "rewards/rejected": 0.012477097101509571, | |
| "step": 9 | |
| }, | |
| { | |
| "epoch": 0.07, | |
| "grad_norm": 32.92454994636796, | |
| "learning_rate": 3.333333333333333e-07, | |
| "logps/chosen": -68.3951416015625, | |
| "logps/rejected": -53.54661178588867, | |
| "loss": 0.6903, | |
| "losses/dpo": 0.6679682731628418, | |
| "losses/sft": 0.3381645083427429, | |
| "losses/total": 0.6679682731628418, | |
| "ref_logps/chosen": -68.44535827636719, | |
| "ref_logps/rejected": -53.52064895629883, | |
| "rewards/accuracies": 0.515625, | |
| "rewards/chosen": 0.005021924152970314, | |
| "rewards/margins": 0.007618834264576435, | |
| "rewards/rejected": -0.002596910111606121, | |
| "step": 10 | |
| }, | |
| { | |
| "epoch": 0.07, | |
| "grad_norm": 28.62792358507012, | |
| "learning_rate": 3.666666666666666e-07, | |
| "logps/chosen": -58.64153289794922, | |
| "logps/rejected": -49.53267288208008, | |
| "loss": 0.6999, | |
| "losses/dpo": 0.6928383111953735, | |
| "losses/sft": 0.34639859199523926, | |
| "losses/total": 0.6928383111953735, | |
| "ref_logps/chosen": -58.6944465637207, | |
| "ref_logps/rejected": -49.710044860839844, | |
| "rewards/accuracies": 0.375, | |
| "rewards/chosen": 0.005291242152452469, | |
| "rewards/margins": -0.012445551343262196, | |
| "rewards/rejected": 0.01773679442703724, | |
| "step": 11 | |
| }, | |
| { | |
| "epoch": 0.08, | |
| "grad_norm": 34.95324550960433, | |
| "learning_rate": 4e-07, | |
| "logps/chosen": -63.45814514160156, | |
| "logps/rejected": -51.141937255859375, | |
| "loss": 0.6926, | |
| "losses/dpo": 0.6734712719917297, | |
| "losses/sft": 0.38542595505714417, | |
| "losses/total": 0.6734712719917297, | |
| "ref_logps/chosen": -63.53767395019531, | |
| "ref_logps/rejected": -51.197601318359375, | |
| "rewards/accuracies": 0.5, | |
| "rewards/chosen": 0.007953131571412086, | |
| "rewards/margins": 0.0023870535660535097, | |
| "rewards/rejected": 0.00556607823818922, | |
| "step": 12 | |
| }, | |
| { | |
| "epoch": 0.09, | |
| "grad_norm": 40.49931666997992, | |
| "learning_rate": 4.3333333333333335e-07, | |
| "logps/chosen": -72.46746826171875, | |
| "logps/rejected": -58.34828186035156, | |
| "loss": 0.6943, | |
| "losses/dpo": 0.6939187049865723, | |
| "losses/sft": 0.3672393262386322, | |
| "losses/total": 0.6939187049865723, | |
| "ref_logps/chosen": -72.57909393310547, | |
| "ref_logps/rejected": -58.46982955932617, | |
| "rewards/accuracies": 0.421875, | |
| "rewards/chosen": 0.011162188835442066, | |
| "rewards/margins": -0.000992664834484458, | |
| "rewards/rejected": 0.012154852971434593, | |
| "step": 13 | |
| }, | |
| { | |
| "epoch": 0.1, | |
| "grad_norm": 35.43728228028493, | |
| "learning_rate": 4.6666666666666666e-07, | |
| "logps/chosen": -60.627281188964844, | |
| "logps/rejected": -52.48558807373047, | |
| "loss": 0.6921, | |
| "losses/dpo": 0.6789003014564514, | |
| "losses/sft": 0.3384055495262146, | |
| "losses/total": 0.6789003014564514, | |
| "ref_logps/chosen": -60.64947509765625, | |
| "ref_logps/rejected": -52.47667694091797, | |
| "rewards/accuracies": 0.484375, | |
| "rewards/chosen": 0.0022186655551195145, | |
| "rewards/margins": 0.003109627403318882, | |
| "rewards/rejected": -0.0008909617317840457, | |
| "step": 14 | |
| }, | |
| { | |
| "epoch": 0.1, | |
| "grad_norm": 37.2571656085439, | |
| "learning_rate": 5e-07, | |
| "logps/chosen": -64.5811538696289, | |
| "logps/rejected": -56.402488708496094, | |
| "loss": 0.6889, | |
| "losses/dpo": 0.6773965358734131, | |
| "losses/sft": 0.32888567447662354, | |
| "losses/total": 0.6773965358734131, | |
| "ref_logps/chosen": -64.77919006347656, | |
| "ref_logps/rejected": -56.505523681640625, | |
| "rewards/accuracies": 0.578125, | |
| "rewards/chosen": 0.019803302362561226, | |
| "rewards/margins": 0.009500503540039062, | |
| "rewards/rejected": 0.010302798822522163, | |
| "step": 15 | |
| }, | |
| { | |
| "epoch": 0.11, | |
| "grad_norm": 37.083526947632535, | |
| "learning_rate": 4.999291986732822e-07, | |
| "logps/chosen": -67.24109649658203, | |
| "logps/rejected": -54.730838775634766, | |
| "loss": 0.6882, | |
| "losses/dpo": 0.6811863780021667, | |
| "losses/sft": 0.32620126008987427, | |
| "losses/total": 0.6811863780021667, | |
| "ref_logps/chosen": -67.33450317382812, | |
| "ref_logps/rejected": -54.712337493896484, | |
| "rewards/accuracies": 0.546875, | |
| "rewards/chosen": 0.009341094642877579, | |
| "rewards/margins": 0.011191939935088158, | |
| "rewards/rejected": -0.0018508443608880043, | |
| "step": 16 | |
| }, | |
| { | |
| "epoch": 0.12, | |
| "grad_norm": 31.937569243375922, | |
| "learning_rate": 4.99716834795752e-07, | |
| "logps/chosen": -63.45280456542969, | |
| "logps/rejected": -52.51921463012695, | |
| "loss": 0.6823, | |
| "losses/dpo": 0.678799569606781, | |
| "losses/sft": 0.3536127507686615, | |
| "losses/total": 0.678799569606781, | |
| "ref_logps/chosen": -63.669334411621094, | |
| "ref_logps/rejected": -52.50813674926758, | |
| "rewards/accuracies": 0.65625, | |
| "rewards/chosen": 0.021653007715940475, | |
| "rewards/margins": 0.022760074585676193, | |
| "rewards/rejected": -0.0011070647742599249, | |
| "step": 17 | |
| }, | |
| { | |
| "epoch": 0.12, | |
| "grad_norm": 40.33340375657743, | |
| "learning_rate": 4.993630286525634e-07, | |
| "logps/chosen": -67.25548553466797, | |
| "logps/rejected": -56.92215347290039, | |
| "loss": 0.6874, | |
| "losses/dpo": 0.6505956053733826, | |
| "losses/sft": 0.3500673770904541, | |
| "losses/total": 0.6505956053733826, | |
| "ref_logps/chosen": -67.41217041015625, | |
| "ref_logps/rejected": -56.942283630371094, | |
| "rewards/accuracies": 0.578125, | |
| "rewards/chosen": 0.01566872000694275, | |
| "rewards/margins": 0.013655520044267178, | |
| "rewards/rejected": 0.0020132004283368587, | |
| "step": 18 | |
| }, | |
| { | |
| "epoch": 0.13, | |
| "grad_norm": 38.0338752718148, | |
| "learning_rate": 4.988679806432711e-07, | |
| "logps/chosen": -68.8526611328125, | |
| "logps/rejected": -57.075347900390625, | |
| "loss": 0.6723, | |
| "losses/dpo": 0.6660802960395813, | |
| "losses/sft": 0.34118345379829407, | |
| "losses/total": 0.6660802960395813, | |
| "ref_logps/chosen": -69.11880493164062, | |
| "ref_logps/rejected": -56.906070709228516, | |
| "rewards/accuracies": 0.640625, | |
| "rewards/chosen": 0.026613827794790268, | |
| "rewards/margins": 0.04354187101125717, | |
| "rewards/rejected": -0.016928039491176605, | |
| "step": 19 | |
| }, | |
| { | |
| "epoch": 0.14, | |
| "grad_norm": 35.61816166775489, | |
| "learning_rate": 4.982319711683221e-07, | |
| "logps/chosen": -72.06625366210938, | |
| "logps/rejected": -59.52400207519531, | |
| "loss": 0.6721, | |
| "losses/dpo": 0.6615892052650452, | |
| "losses/sft": 0.34661561250686646, | |
| "losses/total": 0.6615892052650452, | |
| "ref_logps/chosen": -72.32228088378906, | |
| "ref_logps/rejected": -59.32331848144531, | |
| "rewards/accuracies": 0.65625, | |
| "rewards/chosen": 0.02560277283191681, | |
| "rewards/margins": 0.04567151144146919, | |
| "rewards/rejected": -0.02006874606013298, | |
| "step": 20 | |
| }, | |
| { | |
| "epoch": 0.14, | |
| "grad_norm": 35.39098813520557, | |
| "learning_rate": 4.974553604702332e-07, | |
| "logps/chosen": -69.02694702148438, | |
| "logps/rejected": -56.37590789794922, | |
| "loss": 0.6788, | |
| "losses/dpo": 0.6626015901565552, | |
| "losses/sft": 0.3223264813423157, | |
| "losses/total": 0.6626015901565552, | |
| "ref_logps/chosen": -69.218994140625, | |
| "ref_logps/rejected": -56.257652282714844, | |
| "rewards/accuracies": 0.625, | |
| "rewards/chosen": 0.01920466497540474, | |
| "rewards/margins": 0.031030891463160515, | |
| "rewards/rejected": -0.011826226487755775, | |
| "step": 21 | |
| }, | |
| { | |
| "epoch": 0.15, | |
| "grad_norm": 33.9907845041531, | |
| "learning_rate": 4.965385884295466e-07, | |
| "logps/chosen": -58.412986755371094, | |
| "logps/rejected": -53.20189666748047, | |
| "loss": 0.6719, | |
| "losses/dpo": 0.6767945289611816, | |
| "losses/sft": 0.3387165367603302, | |
| "losses/total": 0.6767945289611816, | |
| "ref_logps/chosen": -58.631317138671875, | |
| "ref_logps/rejected": -52.960777282714844, | |
| "rewards/accuracies": 0.640625, | |
| "rewards/chosen": 0.021833162754774094, | |
| "rewards/margins": 0.045944713056087494, | |
| "rewards/rejected": -0.024111557751893997, | |
| "step": 22 | |
| }, | |
| { | |
| "epoch": 0.16, | |
| "grad_norm": 33.80480054954672, | |
| "learning_rate": 4.954821743156767e-07, | |
| "logps/chosen": -67.24698638916016, | |
| "logps/rejected": -56.11846160888672, | |
| "loss": 0.6741, | |
| "losses/dpo": 0.6613680124282837, | |
| "losses/sft": 0.34880080819129944, | |
| "losses/total": 0.6613680124282837, | |
| "ref_logps/chosen": -67.44197082519531, | |
| "ref_logps/rejected": -55.9046630859375, | |
| "rewards/accuracies": 0.609375, | |
| "rewards/chosen": 0.019499044865369797, | |
| "rewards/margins": 0.040879033505916595, | |
| "rewards/rejected": -0.0213799886405468, | |
| "step": 23 | |
| }, | |
| { | |
| "epoch": 0.16, | |
| "grad_norm": 33.73597666461483, | |
| "learning_rate": 4.942867164927899e-07, | |
| "logps/chosen": -62.87995910644531, | |
| "logps/rejected": -55.92186737060547, | |
| "loss": 0.6554, | |
| "losses/dpo": 0.6621065139770508, | |
| "losses/sft": 0.350619375705719, | |
| "losses/total": 0.6621065139770508, | |
| "ref_logps/chosen": -63.19189453125, | |
| "ref_logps/rejected": -55.41385269165039, | |
| "rewards/accuracies": 0.6875, | |
| "rewards/chosen": 0.031193653121590614, | |
| "rewards/margins": 0.08199521154165268, | |
| "rewards/rejected": -0.05080154910683632, | |
| "step": 24 | |
| }, | |
| { | |
| "epoch": 0.17, | |
| "grad_norm": 31.252590331214783, | |
| "learning_rate": 4.929528920808854e-07, | |
| "logps/chosen": -70.07772064208984, | |
| "logps/rejected": -58.75320816040039, | |
| "loss": 0.6566, | |
| "losses/dpo": 0.646221935749054, | |
| "losses/sft": 0.3677119016647339, | |
| "losses/total": 0.646221935749054, | |
| "ref_logps/chosen": -70.34449005126953, | |
| "ref_logps/rejected": -58.24370574951172, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": 0.026676896959543228, | |
| "rewards/margins": 0.07762698084115982, | |
| "rewards/rejected": -0.05095008388161659, | |
| "step": 25 | |
| }, | |
| { | |
| "epoch": 0.18, | |
| "grad_norm": 31.628164542125273, | |
| "learning_rate": 4.91481456572267e-07, | |
| "logps/chosen": -63.15786361694336, | |
| "logps/rejected": -51.286197662353516, | |
| "loss": 0.6378, | |
| "losses/dpo": 0.6170041561126709, | |
| "losses/sft": 0.36194464564323425, | |
| "losses/total": 0.6170041561126709, | |
| "ref_logps/chosen": -63.57653045654297, | |
| "ref_logps/rejected": -50.502952575683594, | |
| "rewards/accuracies": 0.796875, | |
| "rewards/chosen": 0.04186610132455826, | |
| "rewards/margins": 0.12019069492816925, | |
| "rewards/rejected": -0.07832459360361099, | |
| "step": 26 | |
| }, | |
| { | |
| "epoch": 0.18, | |
| "grad_norm": 29.472445320476258, | |
| "learning_rate": 4.898732434036243e-07, | |
| "logps/chosen": -60.576107025146484, | |
| "logps/rejected": -51.86144256591797, | |
| "loss": 0.6304, | |
| "losses/dpo": 0.6577066779136658, | |
| "losses/sft": 0.31096988916397095, | |
| "losses/total": 0.6577066779136658, | |
| "ref_logps/chosen": -61.04180145263672, | |
| "ref_logps/rejected": -50.955745697021484, | |
| "rewards/accuracies": 0.828125, | |
| "rewards/chosen": 0.04656900465488434, | |
| "rewards/margins": 0.13713900744915009, | |
| "rewards/rejected": -0.09057000279426575, | |
| "step": 27 | |
| }, | |
| { | |
| "epoch": 0.19, | |
| "grad_norm": 34.12892875093247, | |
| "learning_rate": 4.881291634839651e-07, | |
| "logps/chosen": -71.571044921875, | |
| "logps/rejected": -59.838462829589844, | |
| "loss": 0.6348, | |
| "losses/dpo": 0.6427409052848816, | |
| "losses/sft": 0.370533287525177, | |
| "losses/total": 0.6427409052848816, | |
| "ref_logps/chosen": -71.80653381347656, | |
| "ref_logps/rejected": -58.735103607177734, | |
| "rewards/accuracies": 0.640625, | |
| "rewards/chosen": 0.023549124598503113, | |
| "rewards/margins": 0.1338854283094406, | |
| "rewards/rejected": -0.1103362962603569, | |
| "step": 28 | |
| }, | |
| { | |
| "epoch": 0.2, | |
| "grad_norm": 32.265156441586456, | |
| "learning_rate": 4.86250204678667e-07, | |
| "logps/chosen": -65.21573638916016, | |
| "logps/rejected": -56.004364013671875, | |
| "loss": 0.6235, | |
| "losses/dpo": 0.6029931902885437, | |
| "losses/sft": 0.3526380658149719, | |
| "losses/total": 0.6029931902885437, | |
| "ref_logps/chosen": -65.50960540771484, | |
| "ref_logps/rejected": -54.726261138916016, | |
| "rewards/accuracies": 0.8125, | |
| "rewards/chosen": 0.029387805610895157, | |
| "rewards/margins": 0.15719717741012573, | |
| "rewards/rejected": -0.12780936062335968, | |
| "step": 29 | |
| }, | |
| { | |
| "epoch": 0.2, | |
| "grad_norm": 37.3823888174941, | |
| "learning_rate": 4.842374312499405e-07, | |
| "logps/chosen": -73.87957763671875, | |
| "logps/rejected": -60.16520309448242, | |
| "loss": 0.5862, | |
| "losses/dpo": 0.5859757661819458, | |
| "losses/sft": 0.34089845418930054, | |
| "losses/total": 0.5859757661819458, | |
| "ref_logps/chosen": -74.58474731445312, | |
| "ref_logps/rejected": -58.38753890991211, | |
| "rewards/accuracies": 0.796875, | |
| "rewards/chosen": 0.0705181285738945, | |
| "rewards/margins": 0.24828468263149261, | |
| "rewards/rejected": -0.1777665764093399, | |
| "step": 30 | |
| }, | |
| { | |
| "epoch": 0.21, | |
| "grad_norm": 32.144180117174194, | |
| "learning_rate": 4.820919832540181e-07, | |
| "logps/chosen": -65.9237060546875, | |
| "logps/rejected": -57.733497619628906, | |
| "loss": 0.6071, | |
| "losses/dpo": 0.6222472190856934, | |
| "losses/sft": 0.33616167306900024, | |
| "losses/total": 0.6222472190856934, | |
| "ref_logps/chosen": -65.80138397216797, | |
| "ref_logps/rejected": -55.510379791259766, | |
| "rewards/accuracies": 0.71875, | |
| "rewards/chosen": -0.012232396751642227, | |
| "rewards/margins": 0.21007922291755676, | |
| "rewards/rejected": -0.22231163084506989, | |
| "step": 31 | |
| }, | |
| { | |
| "epoch": 0.22, | |
| "grad_norm": 35.501453167718694, | |
| "learning_rate": 4.798150758954164e-07, | |
| "logps/chosen": -68.75838470458984, | |
| "logps/rejected": -60.68508529663086, | |
| "loss": 0.5798, | |
| "losses/dpo": 0.5868023037910461, | |
| "losses/sft": 0.3538406789302826, | |
| "losses/total": 0.5868023037910461, | |
| "ref_logps/chosen": -68.71626281738281, | |
| "ref_logps/rejected": -57.89375305175781, | |
| "rewards/accuracies": 0.765625, | |
| "rewards/chosen": -0.004212833940982819, | |
| "rewards/margins": 0.2749200761318207, | |
| "rewards/rejected": -0.2791329324245453, | |
| "step": 32 | |
| }, | |
| { | |
| "epoch": 0.22, | |
| "grad_norm": 32.699996982131026, | |
| "learning_rate": 4.774079988386296e-07, | |
| "logps/chosen": -70.96333312988281, | |
| "logps/rejected": -61.67507553100586, | |
| "loss": 0.5944, | |
| "losses/dpo": 0.5362982749938965, | |
| "losses/sft": 0.37232011556625366, | |
| "losses/total": 0.5362982749938965, | |
| "ref_logps/chosen": -70.30509948730469, | |
| "ref_logps/rejected": -58.60476303100586, | |
| "rewards/accuracies": 0.765625, | |
| "rewards/chosen": -0.06582287698984146, | |
| "rewards/margins": 0.24120771884918213, | |
| "rewards/rejected": -0.3070306181907654, | |
| "step": 33 | |
| }, | |
| { | |
| "epoch": 0.23, | |
| "grad_norm": 37.488816567711076, | |
| "learning_rate": 4.748721154776508e-07, | |
| "logps/chosen": -65.87406921386719, | |
| "logps/rejected": -59.7636604309082, | |
| "loss": 0.5652, | |
| "losses/dpo": 0.500527024269104, | |
| "losses/sft": 0.3647816479206085, | |
| "losses/total": 0.500527024269104, | |
| "ref_logps/chosen": -65.68878173828125, | |
| "ref_logps/rejected": -56.097267150878906, | |
| "rewards/accuracies": 0.734375, | |
| "rewards/chosen": -0.018529564142227173, | |
| "rewards/margins": 0.3481099605560303, | |
| "rewards/rejected": -0.36663955450057983, | |
| "step": 34 | |
| }, | |
| { | |
| "epoch": 0.24, | |
| "grad_norm": 33.919463466183984, | |
| "learning_rate": 4.7220886216373085e-07, | |
| "logps/chosen": -67.95443725585938, | |
| "logps/rejected": -64.25914001464844, | |
| "loss": 0.5351, | |
| "losses/dpo": 0.5877989530563354, | |
| "losses/sft": 0.3324495255947113, | |
| "losses/total": 0.5877989530563354, | |
| "ref_logps/chosen": -67.34272766113281, | |
| "ref_logps/rejected": -59.69349670410156, | |
| "rewards/accuracies": 0.875, | |
| "rewards/chosen": -0.06117025017738342, | |
| "rewards/margins": 0.3953939974308014, | |
| "rewards/rejected": -0.4565642774105072, | |
| "step": 35 | |
| }, | |
| { | |
| "epoch": 0.24, | |
| "grad_norm": 36.85611869447123, | |
| "learning_rate": 4.694197473918139e-07, | |
| "logps/chosen": -73.41097259521484, | |
| "logps/rejected": -63.35462188720703, | |
| "loss": 0.5958, | |
| "losses/dpo": 0.5920328497886658, | |
| "losses/sft": 0.34827345609664917, | |
| "losses/total": 0.5920328497886658, | |
| "ref_logps/chosen": -72.06607055664062, | |
| "ref_logps/rejected": -59.26370620727539, | |
| "rewards/accuracies": 0.671875, | |
| "rewards/chosen": -0.1344902515411377, | |
| "rewards/margins": 0.2746003270149231, | |
| "rewards/rejected": -0.4090905785560608, | |
| "step": 36 | |
| }, | |
| { | |
| "epoch": 0.25, | |
| "grad_norm": 29.396807219224034, | |
| "learning_rate": 4.6650635094610966e-07, | |
| "logps/chosen": -69.74252319335938, | |
| "logps/rejected": -59.216861724853516, | |
| "loss": 0.511, | |
| "losses/dpo": 0.4569849967956543, | |
| "losses/sft": 0.330870658159256, | |
| "losses/total": 0.4569849967956543, | |
| "ref_logps/chosen": -68.98128509521484, | |
| "ref_logps/rejected": -53.205997467041016, | |
| "rewards/accuracies": 0.84375, | |
| "rewards/chosen": -0.07612372934818268, | |
| "rewards/margins": 0.5249630212783813, | |
| "rewards/rejected": -0.6010866761207581, | |
| "step": 37 | |
| }, | |
| { | |
| "epoch": 0.26, | |
| "grad_norm": 32.158672339711444, | |
| "learning_rate": 4.63470323005287e-07, | |
| "logps/chosen": -74.17395782470703, | |
| "logps/rejected": -65.09367370605469, | |
| "loss": 0.4616, | |
| "losses/dpo": 0.4395897388458252, | |
| "losses/sft": 0.3282167613506317, | |
| "losses/total": 0.4395897388458252, | |
| "ref_logps/chosen": -72.66117858886719, | |
| "ref_logps/rejected": -56.74953079223633, | |
| "rewards/accuracies": 0.859375, | |
| "rewards/chosen": -0.15127676725387573, | |
| "rewards/margins": 0.6831383109092712, | |
| "rewards/rejected": -0.8344150185585022, | |
| "step": 38 | |
| }, | |
| { | |
| "epoch": 0.27, | |
| "grad_norm": 30.39214935377121, | |
| "learning_rate": 4.603133832077953e-07, | |
| "logps/chosen": -72.17027282714844, | |
| "logps/rejected": -66.1070785522461, | |
| "loss": 0.5353, | |
| "losses/dpo": 0.47761982679367065, | |
| "losses/sft": 0.38627445697784424, | |
| "losses/total": 0.47761982679367065, | |
| "ref_logps/chosen": -69.07978820800781, | |
| "ref_logps/rejected": -58.215328216552734, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": -0.3090481758117676, | |
| "rewards/margins": 0.4801270365715027, | |
| "rewards/rejected": -0.7891752123832703, | |
| "step": 39 | |
| }, | |
| { | |
| "epoch": 0.27, | |
| "grad_norm": 29.86640568721065, | |
| "learning_rate": 4.5703731967784265e-07, | |
| "logps/chosen": -61.24689483642578, | |
| "logps/rejected": -59.919918060302734, | |
| "loss": 0.5273, | |
| "losses/dpo": 0.7497649192810059, | |
| "losses/sft": 0.35406211018562317, | |
| "losses/total": 0.7497649192810059, | |
| "ref_logps/chosen": -57.38946533203125, | |
| "ref_logps/rejected": -50.80248260498047, | |
| "rewards/accuracies": 0.734375, | |
| "rewards/chosen": -0.3857432007789612, | |
| "rewards/margins": 0.5260001420974731, | |
| "rewards/rejected": -0.9117434024810791, | |
| "step": 40 | |
| }, | |
| { | |
| "epoch": 0.28, | |
| "grad_norm": 29.500472780356045, | |
| "learning_rate": 4.5364398801258394e-07, | |
| "logps/chosen": -63.82752990722656, | |
| "logps/rejected": -60.21675491333008, | |
| "loss": 0.5103, | |
| "losses/dpo": 0.5127725601196289, | |
| "losses/sft": 0.3456054925918579, | |
| "losses/total": 0.5127725601196289, | |
| "ref_logps/chosen": -60.51768493652344, | |
| "ref_logps/rejected": -50.69208526611328, | |
| "rewards/accuracies": 0.84375, | |
| "rewards/chosen": -0.3309844732284546, | |
| "rewards/margins": 0.6214821934700012, | |
| "rewards/rejected": -0.952466607093811, | |
| "step": 41 | |
| }, | |
| { | |
| "epoch": 0.29, | |
| "grad_norm": 28.303717346192418, | |
| "learning_rate": 4.501353102310901e-07, | |
| "logps/chosen": -64.48641967773438, | |
| "logps/rejected": -60.165767669677734, | |
| "loss": 0.5197, | |
| "losses/dpo": 0.4626217782497406, | |
| "losses/sft": 0.3867911994457245, | |
| "losses/total": 0.4626217782497406, | |
| "ref_logps/chosen": -60.505496978759766, | |
| "ref_logps/rejected": -50.563045501708984, | |
| "rewards/accuracies": 0.765625, | |
| "rewards/chosen": -0.3980926275253296, | |
| "rewards/margins": 0.562179684638977, | |
| "rewards/rejected": -0.9602723121643066, | |
| "step": 42 | |
| }, | |
| { | |
| "epoch": 0.29, | |
| "grad_norm": 26.546916183591545, | |
| "learning_rate": 4.4651327368569684e-07, | |
| "logps/chosen": -74.27953338623047, | |
| "logps/rejected": -66.18184661865234, | |
| "loss": 0.478, | |
| "losses/dpo": 0.5494570136070251, | |
| "losses/sft": 0.3961746394634247, | |
| "losses/total": 0.5494570136070251, | |
| "ref_logps/chosen": -68.87181854248047, | |
| "ref_logps/rejected": -54.055946350097656, | |
| "rewards/accuracies": 0.84375, | |
| "rewards/chosen": -0.5407715439796448, | |
| "rewards/margins": 0.671818733215332, | |
| "rewards/rejected": -1.212590217590332, | |
| "step": 43 | |
| }, | |
| { | |
| "epoch": 0.3, | |
| "grad_norm": 26.197389360540555, | |
| "learning_rate": 4.4277992993634686e-07, | |
| "logps/chosen": -79.60818481445312, | |
| "logps/rejected": -79.91624450683594, | |
| "loss": 0.4052, | |
| "losses/dpo": 0.4185926914215088, | |
| "losses/sft": 0.3548857569694519, | |
| "losses/total": 0.4185926914215088, | |
| "ref_logps/chosen": -72.74970245361328, | |
| "ref_logps/rejected": -62.91980743408203, | |
| "rewards/accuracies": 0.84375, | |
| "rewards/chosen": -0.6858479976654053, | |
| "rewards/margins": 1.0137956142425537, | |
| "rewards/rejected": -1.699643611907959, | |
| "step": 44 | |
| }, | |
| { | |
| "epoch": 0.31, | |
| "grad_norm": 27.708445312624427, | |
| "learning_rate": 4.3893739358856455e-07, | |
| "logps/chosen": -79.15251159667969, | |
| "logps/rejected": -72.04096221923828, | |
| "loss": 0.4729, | |
| "losses/dpo": 0.4918556213378906, | |
| "losses/sft": 0.37935709953308105, | |
| "losses/total": 0.4918556213378906, | |
| "ref_logps/chosen": -72.23287963867188, | |
| "ref_logps/rejected": -56.32197570800781, | |
| "rewards/accuracies": 0.71875, | |
| "rewards/chosen": -0.6919631958007812, | |
| "rewards/margins": 0.8799355626106262, | |
| "rewards/rejected": -1.5718988180160522, | |
| "step": 45 | |
| }, | |
| { | |
| "epoch": 0.31, | |
| "grad_norm": 30.962017558121154, | |
| "learning_rate": 4.34987841095721e-07, | |
| "logps/chosen": -75.75450134277344, | |
| "logps/rejected": -75.48016357421875, | |
| "loss": 0.4757, | |
| "losses/dpo": 0.47784677147865295, | |
| "losses/sft": 0.37939363718032837, | |
| "losses/total": 0.47784677147865295, | |
| "ref_logps/chosen": -67.21387481689453, | |
| "ref_logps/rejected": -58.56239318847656, | |
| "rewards/accuracies": 0.84375, | |
| "rewards/chosen": -0.8540633320808411, | |
| "rewards/margins": 0.8377144932746887, | |
| "rewards/rejected": -1.6917778253555298, | |
| "step": 46 | |
| }, | |
| { | |
| "epoch": 0.32, | |
| "grad_norm": 27.327055413855135, | |
| "learning_rate": 4.309335095262675e-07, | |
| "logps/chosen": -84.39332580566406, | |
| "logps/rejected": -78.34506225585938, | |
| "loss": 0.4763, | |
| "losses/dpo": 0.3084137439727783, | |
| "losses/sft": 0.3708834648132324, | |
| "losses/total": 0.3084137439727783, | |
| "ref_logps/chosen": -74.23382568359375, | |
| "ref_logps/rejected": -59.50791931152344, | |
| "rewards/accuracies": 0.671875, | |
| "rewards/chosen": -1.0159505605697632, | |
| "rewards/margins": 0.8677631616592407, | |
| "rewards/rejected": -1.8837138414382935, | |
| "step": 47 | |
| }, | |
| { | |
| "epoch": 0.33, | |
| "grad_norm": 37.60224677954145, | |
| "learning_rate": 4.2677669529663686e-07, | |
| "logps/chosen": -85.45326232910156, | |
| "logps/rejected": -80.37303161621094, | |
| "loss": 0.498, | |
| "losses/dpo": 0.5172345638275146, | |
| "losses/sft": 0.46193623542785645, | |
| "losses/total": 0.5172345638275146, | |
| "ref_logps/chosen": -76.32438659667969, | |
| "ref_logps/rejected": -63.45566940307617, | |
| "rewards/accuracies": 0.734375, | |
| "rewards/chosen": -0.9128875732421875, | |
| "rewards/margins": 0.7788479328155518, | |
| "rewards/rejected": -1.6917355060577393, | |
| "step": 48 | |
| }, | |
| { | |
| "epoch": 0.33, | |
| "grad_norm": 26.012407404622028, | |
| "learning_rate": 4.22519752870528e-07, | |
| "logps/chosen": -86.257080078125, | |
| "logps/rejected": -79.96172332763672, | |
| "loss": 0.4015, | |
| "losses/dpo": 0.3850885331630707, | |
| "losses/sft": 0.4463062882423401, | |
| "losses/total": 0.3850885331630707, | |
| "ref_logps/chosen": -77.25395965576172, | |
| "ref_logps/rejected": -60.78544998168945, | |
| "rewards/accuracies": 0.890625, | |
| "rewards/chosen": -0.9003111124038696, | |
| "rewards/margins": 1.0173161029815674, | |
| "rewards/rejected": -1.9176270961761475, | |
| "step": 49 | |
| }, | |
| { | |
| "epoch": 0.34, | |
| "grad_norm": 29.5339729693226, | |
| "learning_rate": 4.1816509342531317e-07, | |
| "logps/chosen": -72.80648803710938, | |
| "logps/rejected": -73.68412780761719, | |
| "loss": 0.4609, | |
| "losses/dpo": 0.5073346495628357, | |
| "losses/sft": 0.3690868318080902, | |
| "losses/total": 0.5073346495628357, | |
| "ref_logps/chosen": -61.75213623046875, | |
| "ref_logps/rejected": -53.534000396728516, | |
| "rewards/accuracies": 0.765625, | |
| "rewards/chosen": -1.1054348945617676, | |
| "rewards/margins": 0.9095778465270996, | |
| "rewards/rejected": -2.015012741088867, | |
| "step": 50 | |
| }, | |
| { | |
| "epoch": 0.35, | |
| "grad_norm": 24.67832174233356, | |
| "learning_rate": 4.137151834863213e-07, | |
| "logps/chosen": -78.76303100585938, | |
| "logps/rejected": -81.20369720458984, | |
| "loss": 0.3581, | |
| "losses/dpo": 0.31153589487075806, | |
| "losses/sft": 0.34659337997436523, | |
| "losses/total": 0.31153589487075806, | |
| "ref_logps/chosen": -67.64138793945312, | |
| "ref_logps/rejected": -56.615203857421875, | |
| "rewards/accuracies": 0.84375, | |
| "rewards/chosen": -1.1121646165847778, | |
| "rewards/margins": 1.3466840982437134, | |
| "rewards/rejected": -2.458848714828491, | |
| "step": 51 | |
| }, | |
| { | |
| "epoch": 0.35, | |
| "grad_norm": 25.652161578301136, | |
| "learning_rate": 4.0917254352977206e-07, | |
| "logps/chosen": -80.27860260009766, | |
| "logps/rejected": -78.06929016113281, | |
| "loss": 0.4625, | |
| "losses/dpo": 0.5235142707824707, | |
| "losses/sft": 0.3862667977809906, | |
| "losses/total": 0.5235142707824707, | |
| "ref_logps/chosen": -65.9322280883789, | |
| "ref_logps/rejected": -54.390098571777344, | |
| "rewards/accuracies": 0.8125, | |
| "rewards/chosen": -1.4346369504928589, | |
| "rewards/margins": 0.9332831501960754, | |
| "rewards/rejected": -2.367920160293579, | |
| "step": 52 | |
| }, | |
| { | |
| "epoch": 0.36, | |
| "grad_norm": 31.613726978757875, | |
| "learning_rate": 4.045397465551513e-07, | |
| "logps/chosen": -84.57078552246094, | |
| "logps/rejected": -76.68920135498047, | |
| "loss": 0.5117, | |
| "losses/dpo": 0.4083520770072937, | |
| "losses/sft": 0.32504379749298096, | |
| "losses/total": 0.4083520770072937, | |
| "ref_logps/chosen": -72.13128662109375, | |
| "ref_logps/rejected": -53.99201965332031, | |
| "rewards/accuracies": 0.78125, | |
| "rewards/chosen": -1.2439489364624023, | |
| "rewards/margins": 1.0257703065872192, | |
| "rewards/rejected": -2.269719123840332, | |
| "step": 53 | |
| }, | |
| { | |
| "epoch": 0.37, | |
| "grad_norm": 29.84992723438259, | |
| "learning_rate": 3.998194166278367e-07, | |
| "logps/chosen": -76.96097564697266, | |
| "logps/rejected": -74.58714294433594, | |
| "loss": 0.4777, | |
| "losses/dpo": 0.4091704785823822, | |
| "losses/sft": 0.4479122757911682, | |
| "losses/total": 0.4091704785823822, | |
| "ref_logps/chosen": -61.228057861328125, | |
| "ref_logps/rejected": -50.48232650756836, | |
| "rewards/accuracies": 0.71875, | |
| "rewards/chosen": -1.573291540145874, | |
| "rewards/margins": 0.8371899127960205, | |
| "rewards/rejected": -2.4104812145233154, | |
| "step": 54 | |
| }, | |
| { | |
| "epoch": 0.37, | |
| "grad_norm": 36.88672265696095, | |
| "learning_rate": 3.9501422739279953e-07, | |
| "logps/chosen": -81.66922760009766, | |
| "logps/rejected": -75.91566467285156, | |
| "loss": 0.6061, | |
| "losses/dpo": 0.5785080194473267, | |
| "losses/sft": 0.37718018889427185, | |
| "losses/total": 0.5785080194473267, | |
| "ref_logps/chosen": -67.54790496826172, | |
| "ref_logps/rejected": -57.624385833740234, | |
| "rewards/accuracies": 0.640625, | |
| "rewards/chosen": -1.4121325016021729, | |
| "rewards/margins": 0.4169955253601074, | |
| "rewards/rejected": -1.8291279077529907, | |
| "step": 55 | |
| }, | |
| { | |
| "epoch": 0.38, | |
| "grad_norm": 26.409540611725895, | |
| "learning_rate": 3.9012690056022345e-07, | |
| "logps/chosen": -88.31976318359375, | |
| "logps/rejected": -80.9065170288086, | |
| "loss": 0.3836, | |
| "losses/dpo": 0.4441046714782715, | |
| "losses/sft": 0.4260564148426056, | |
| "losses/total": 0.4441046714782715, | |
| "ref_logps/chosen": -74.53655242919922, | |
| "ref_logps/rejected": -56.45116424560547, | |
| "rewards/accuracies": 0.875, | |
| "rewards/chosen": -1.3783206939697266, | |
| "rewards/margins": 1.0672156810760498, | |
| "rewards/rejected": -2.4455361366271973, | |
| "step": 56 | |
| }, | |
| { | |
| "epoch": 0.39, | |
| "grad_norm": 26.34257247806165, | |
| "learning_rate": 3.851602043638994e-07, | |
| "logps/chosen": -80.94815826416016, | |
| "logps/rejected": -77.87812805175781, | |
| "loss": 0.4378, | |
| "losses/dpo": 0.38587698340415955, | |
| "losses/sft": 0.39670416712760925, | |
| "losses/total": 0.38587698340415955, | |
| "ref_logps/chosen": -64.29881286621094, | |
| "ref_logps/rejected": -51.452911376953125, | |
| "rewards/accuracies": 0.8125, | |
| "rewards/chosen": -1.6649348735809326, | |
| "rewards/margins": 0.9775880575180054, | |
| "rewards/rejected": -2.6425228118896484, | |
| "step": 57 | |
| }, | |
| { | |
| "epoch": 0.39, | |
| "grad_norm": 27.08511242764537, | |
| "learning_rate": 3.8011695199326805e-07, | |
| "logps/chosen": -80.61515808105469, | |
| "logps/rejected": -81.2110366821289, | |
| "loss": 0.4141, | |
| "losses/dpo": 0.3337501883506775, | |
| "losses/sft": 0.3477111756801605, | |
| "losses/total": 0.3337501883506775, | |
| "ref_logps/chosen": -66.6812744140625, | |
| "ref_logps/rejected": -56.64939880371094, | |
| "rewards/accuracies": 0.84375, | |
| "rewards/chosen": -1.393389344215393, | |
| "rewards/margins": 1.062773585319519, | |
| "rewards/rejected": -2.456162929534912, | |
| "step": 58 | |
| }, | |
| { | |
| "epoch": 0.4, | |
| "grad_norm": 29.443828368623393, | |
| "learning_rate": 3.75e-07, | |
| "logps/chosen": -79.15809631347656, | |
| "logps/rejected": -77.99955749511719, | |
| "loss": 0.4174, | |
| "losses/dpo": 0.2030327320098877, | |
| "losses/sft": 0.36094900965690613, | |
| "losses/total": 0.2030327320098877, | |
| "ref_logps/chosen": -64.46220397949219, | |
| "ref_logps/rejected": -51.404056549072266, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": -1.4695887565612793, | |
| "rewards/margins": 1.1899616718292236, | |
| "rewards/rejected": -2.659550428390503, | |
| "step": 59 | |
| }, | |
| { | |
| "epoch": 0.41, | |
| "grad_norm": 31.360451766133608, | |
| "learning_rate": 3.698122466800142e-07, | |
| "logps/chosen": -83.48946380615234, | |
| "logps/rejected": -80.81828308105469, | |
| "loss": 0.5084, | |
| "losses/dpo": 0.6314846873283386, | |
| "losses/sft": 0.4423762559890747, | |
| "losses/total": 0.6314846873283386, | |
| "ref_logps/chosen": -69.77751159667969, | |
| "ref_logps/rejected": -59.079166412353516, | |
| "rewards/accuracies": 0.734375, | |
| "rewards/chosen": -1.3711949586868286, | |
| "rewards/margins": 0.802715539932251, | |
| "rewards/rejected": -2.173910617828369, | |
| "step": 60 | |
| }, | |
| { | |
| "epoch": 0.41, | |
| "grad_norm": 32.77515833972762, | |
| "learning_rate": 3.645566304318526e-07, | |
| "logps/chosen": -88.54704284667969, | |
| "logps/rejected": -83.86685180664062, | |
| "loss": 0.4477, | |
| "losses/dpo": 0.4280022382736206, | |
| "losses/sft": 0.3504616618156433, | |
| "losses/total": 0.4280022382736206, | |
| "ref_logps/chosen": -75.23544311523438, | |
| "ref_logps/rejected": -60.14040756225586, | |
| "rewards/accuracies": 0.84375, | |
| "rewards/chosen": -1.331160068511963, | |
| "rewards/margins": 1.04148530960083, | |
| "rewards/rejected": -2.372645378112793, | |
| "step": 61 | |
| }, | |
| { | |
| "epoch": 0.42, | |
| "grad_norm": 27.946308305497556, | |
| "learning_rate": 3.5923612809233984e-07, | |
| "logps/chosen": -78.10906982421875, | |
| "logps/rejected": -79.12262725830078, | |
| "loss": 0.4413, | |
| "losses/dpo": 0.4408959746360779, | |
| "losses/sft": 0.3510644733905792, | |
| "losses/total": 0.4408959746360779, | |
| "ref_logps/chosen": -64.80330657958984, | |
| "ref_logps/rejected": -56.04015350341797, | |
| "rewards/accuracies": 0.828125, | |
| "rewards/chosen": -1.3305763006210327, | |
| "rewards/margins": 0.9776719212532043, | |
| "rewards/rejected": -2.308248281478882, | |
| "step": 62 | |
| }, | |
| { | |
| "epoch": 0.43, | |
| "grad_norm": 41.17153515866478, | |
| "learning_rate": 3.5385375325047163e-07, | |
| "logps/chosen": -80.082275390625, | |
| "logps/rejected": -79.17640686035156, | |
| "loss": 0.4871, | |
| "losses/dpo": 0.45268240571022034, | |
| "losses/sft": 0.3942197263240814, | |
| "losses/total": 0.45268240571022034, | |
| "ref_logps/chosen": -65.72177124023438, | |
| "ref_logps/rejected": -55.820762634277344, | |
| "rewards/accuracies": 0.765625, | |
| "rewards/chosen": -1.4360499382019043, | |
| "rewards/margins": 0.8995141983032227, | |
| "rewards/rejected": -2.335564136505127, | |
| "step": 63 | |
| }, | |
| { | |
| "epoch": 0.44, | |
| "grad_norm": 26.987470660431804, | |
| "learning_rate": 3.4841255454048535e-07, | |
| "logps/chosen": -78.40692138671875, | |
| "logps/rejected": -81.3203125, | |
| "loss": 0.4319, | |
| "losses/dpo": 0.3416772782802582, | |
| "losses/sft": 0.351028174161911, | |
| "losses/total": 0.3416772782802582, | |
| "ref_logps/chosen": -66.26089477539062, | |
| "ref_logps/rejected": -58.60002136230469, | |
| "rewards/accuracies": 0.84375, | |
| "rewards/chosen": -1.214601755142212, | |
| "rewards/margins": 1.057426929473877, | |
| "rewards/rejected": -2.272028923034668, | |
| "step": 64 | |
| }, | |
| { | |
| "epoch": 0.44, | |
| "grad_norm": 33.31694146367439, | |
| "learning_rate": 3.4291561391508185e-07, | |
| "logps/chosen": -82.72046661376953, | |
| "logps/rejected": -85.01447296142578, | |
| "loss": 0.494, | |
| "losses/dpo": 0.4605589509010315, | |
| "losses/sft": 0.3883049190044403, | |
| "losses/total": 0.4605589509010315, | |
| "ref_logps/chosen": -67.7024917602539, | |
| "ref_logps/rejected": -59.51292419433594, | |
| "rewards/accuracies": 0.703125, | |
| "rewards/chosen": -1.5017977952957153, | |
| "rewards/margins": 1.0483572483062744, | |
| "rewards/rejected": -2.5501549243927, | |
| "step": 65 | |
| }, | |
| { | |
| "epoch": 0.45, | |
| "grad_norm": 30.20641981592105, | |
| "learning_rate": 3.373660448997746e-07, | |
| "logps/chosen": -78.78214263916016, | |
| "logps/rejected": -76.67687225341797, | |
| "loss": 0.461, | |
| "losses/dpo": 0.3649926483631134, | |
| "losses/sft": 0.3876701891422272, | |
| "losses/total": 0.3649926483631134, | |
| "ref_logps/chosen": -65.60338592529297, | |
| "ref_logps/rejected": -53.363441467285156, | |
| "rewards/accuracies": 0.765625, | |
| "rewards/chosen": -1.3178752660751343, | |
| "rewards/margins": 1.0134679079055786, | |
| "rewards/rejected": -2.331343173980713, | |
| "step": 66 | |
| }, | |
| { | |
| "epoch": 0.46, | |
| "grad_norm": 28.499338933696293, | |
| "learning_rate": 3.317669908293554e-07, | |
| "logps/chosen": -80.8163070678711, | |
| "logps/rejected": -78.53517150878906, | |
| "loss": 0.4977, | |
| "losses/dpo": 0.5431786775588989, | |
| "losses/sft": 0.3635212779045105, | |
| "losses/total": 0.5431786775588989, | |
| "ref_logps/chosen": -67.38165283203125, | |
| "ref_logps/rejected": -56.12194061279297, | |
| "rewards/accuracies": 0.765625, | |
| "rewards/chosen": -1.3434655666351318, | |
| "rewards/margins": 0.8978573679924011, | |
| "rewards/rejected": -2.2413227558135986, | |
| "step": 67 | |
| }, | |
| { | |
| "epoch": 0.46, | |
| "grad_norm": 30.8426766786845, | |
| "learning_rate": 3.261216230674768e-07, | |
| "logps/chosen": -80.0118408203125, | |
| "logps/rejected": -78.16852569580078, | |
| "loss": 0.4656, | |
| "losses/dpo": 0.22820590436458588, | |
| "losses/sft": 0.41188591718673706, | |
| "losses/total": 0.22820590436458588, | |
| "ref_logps/chosen": -67.98802185058594, | |
| "ref_logps/rejected": -55.998008728027344, | |
| "rewards/accuracies": 0.765625, | |
| "rewards/chosen": -1.2023829221725464, | |
| "rewards/margins": 1.0146691799163818, | |
| "rewards/rejected": -2.2170519828796387, | |
| "step": 68 | |
| }, | |
| { | |
| "epoch": 0.47, | |
| "grad_norm": 26.8942337262588, | |
| "learning_rate": 3.204331392103574e-07, | |
| "logps/chosen": -76.27379608154297, | |
| "logps/rejected": -72.5108413696289, | |
| "loss": 0.3977, | |
| "losses/dpo": 0.349028617143631, | |
| "losses/sft": 0.36189785599708557, | |
| "losses/total": 0.349028617143631, | |
| "ref_logps/chosen": -65.81362915039062, | |
| "ref_logps/rejected": -51.00428009033203, | |
| "rewards/accuracies": 0.890625, | |
| "rewards/chosen": -1.0460165739059448, | |
| "rewards/margins": 1.1046404838562012, | |
| "rewards/rejected": -2.1506569385528564, | |
| "step": 69 | |
| }, | |
| { | |
| "epoch": 0.48, | |
| "grad_norm": 26.047792222109905, | |
| "learning_rate": 3.147047612756302e-07, | |
| "logps/chosen": -86.95272827148438, | |
| "logps/rejected": -83.82672119140625, | |
| "loss": 0.3953, | |
| "losses/dpo": 0.46198731660842896, | |
| "losses/sft": 0.4407782554626465, | |
| "losses/total": 0.46198731660842896, | |
| "ref_logps/chosen": -75.7898178100586, | |
| "ref_logps/rejected": -60.74498748779297, | |
| "rewards/accuracies": 0.8125, | |
| "rewards/chosen": -1.1162903308868408, | |
| "rewards/margins": 1.1918830871582031, | |
| "rewards/rejected": -2.308173418045044, | |
| "step": 70 | |
| }, | |
| { | |
| "epoch": 0.48, | |
| "grad_norm": 24.670894739162968, | |
| "learning_rate": 3.0893973387735683e-07, | |
| "logps/chosen": -70.71044921875, | |
| "logps/rejected": -71.96138763427734, | |
| "loss": 0.3944, | |
| "losses/dpo": 0.29774680733680725, | |
| "losses/sft": 0.3790546953678131, | |
| "losses/total": 0.29774680733680725, | |
| "ref_logps/chosen": -61.44975280761719, | |
| "ref_logps/rejected": -51.039268493652344, | |
| "rewards/accuracies": 0.859375, | |
| "rewards/chosen": -0.9260695576667786, | |
| "rewards/margins": 1.166142225265503, | |
| "rewards/rejected": -2.092211961746216, | |
| "step": 71 | |
| }, | |
| { | |
| "epoch": 0.49, | |
| "grad_norm": 28.239155495003125, | |
| "learning_rate": 3.0314132238824415e-07, | |
| "logps/chosen": -81.26232147216797, | |
| "logps/rejected": -78.04046630859375, | |
| "loss": 0.3912, | |
| "losses/dpo": 0.5779224038124084, | |
| "losses/sft": 0.3678759038448334, | |
| "losses/total": 0.5779224038124084, | |
| "ref_logps/chosen": -72.41909790039062, | |
| "ref_logps/rejected": -57.58437728881836, | |
| "rewards/accuracies": 0.859375, | |
| "rewards/chosen": -0.8843216300010681, | |
| "rewards/margins": 1.1612873077392578, | |
| "rewards/rejected": -2.0456089973449707, | |
| "step": 72 | |
| }, | |
| { | |
| "epoch": 0.5, | |
| "grad_norm": 23.65680821014391, | |
| "learning_rate": 2.9731281109010253e-07, | |
| "logps/chosen": -78.29800415039062, | |
| "logps/rejected": -76.63243103027344, | |
| "loss": 0.3589, | |
| "losses/dpo": 0.39184096455574036, | |
| "losses/sft": 0.35749632120132446, | |
| "losses/total": 0.39184096455574036, | |
| "ref_logps/chosen": -70.57292175292969, | |
| "ref_logps/rejected": -56.80116653442383, | |
| "rewards/accuracies": 0.875, | |
| "rewards/chosen": -0.7725088000297546, | |
| "rewards/margins": 1.210618257522583, | |
| "rewards/rejected": -1.9831271171569824, | |
| "step": 73 | |
| }, | |
| { | |
| "epoch": 0.5, | |
| "grad_norm": 57.763902094880514, | |
| "learning_rate": 2.9145750131359355e-07, | |
| "logps/chosen": -71.69022369384766, | |
| "logps/rejected": -74.53846740722656, | |
| "loss": 0.4711, | |
| "losses/dpo": 0.3447693884372711, | |
| "losses/sft": 0.3251613676548004, | |
| "losses/total": 0.3447693884372711, | |
| "ref_logps/chosen": -61.327369689941406, | |
| "ref_logps/rejected": -53.874820709228516, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": -1.036285638809204, | |
| "rewards/margins": 1.0300801992416382, | |
| "rewards/rejected": -2.0663657188415527, | |
| "step": 74 | |
| }, | |
| { | |
| "epoch": 0.51, | |
| "grad_norm": 37.61505605094831, | |
| "learning_rate": 2.8557870956832133e-07, | |
| "logps/chosen": -78.79322052001953, | |
| "logps/rejected": -73.63280487060547, | |
| "loss": 0.4466, | |
| "losses/dpo": 0.4499085545539856, | |
| "losses/sft": 0.3389120101928711, | |
| "losses/total": 0.4499085545539856, | |
| "ref_logps/chosen": -70.82493591308594, | |
| "ref_logps/rejected": -56.25859832763672, | |
| "rewards/accuracies": 0.8125, | |
| "rewards/chosen": -0.7968282699584961, | |
| "rewards/margins": 0.9405927658081055, | |
| "rewards/rejected": -1.7374211549758911, | |
| "step": 75 | |
| }, | |
| { | |
| "epoch": 0.52, | |
| "grad_norm": 25.268092842517042, | |
| "learning_rate": 2.7967976566432627e-07, | |
| "logps/chosen": -79.96376037597656, | |
| "logps/rejected": -75.98302459716797, | |
| "loss": 0.3581, | |
| "losses/dpo": 0.3253936469554901, | |
| "losses/sft": 0.3978617191314697, | |
| "losses/total": 0.3253936469554901, | |
| "ref_logps/chosen": -70.92337036132812, | |
| "ref_logps/rejected": -53.73899841308594, | |
| "rewards/accuracies": 0.84375, | |
| "rewards/chosen": -0.9040391445159912, | |
| "rewards/margins": 1.320363163948059, | |
| "rewards/rejected": -2.2244021892547607, | |
| "step": 76 | |
| }, | |
| { | |
| "epoch": 0.52, | |
| "grad_norm": 24.79898935884772, | |
| "learning_rate": 2.737640108260456e-07, | |
| "logps/chosen": -74.60429382324219, | |
| "logps/rejected": -74.33484649658203, | |
| "loss": 0.3639, | |
| "losses/dpo": 0.19668929278850555, | |
| "losses/sft": 0.34319061040878296, | |
| "losses/total": 0.19668929278850555, | |
| "ref_logps/chosen": -67.25056457519531, | |
| "ref_logps/rejected": -54.8765869140625, | |
| "rewards/accuracies": 0.921875, | |
| "rewards/chosen": -0.7353739142417908, | |
| "rewards/margins": 1.2104520797729492, | |
| "rewards/rejected": -1.9458260536193848, | |
| "step": 77 | |
| }, | |
| { | |
| "epoch": 0.53, | |
| "grad_norm": 32.80096075705349, | |
| "learning_rate": 2.6783479579980807e-07, | |
| "logps/chosen": -73.52947998046875, | |
| "logps/rejected": -75.42818450927734, | |
| "loss": 0.4477, | |
| "losses/dpo": 0.4471086859703064, | |
| "losses/sft": 0.3155350685119629, | |
| "losses/total": 0.4471086859703064, | |
| "ref_logps/chosen": -64.02719116210938, | |
| "ref_logps/rejected": -56.57443618774414, | |
| "rewards/accuracies": 0.828125, | |
| "rewards/chosen": -0.9502288103103638, | |
| "rewards/margins": 0.9351459741592407, | |
| "rewards/rejected": -1.885374903678894, | |
| "step": 78 | |
| }, | |
| { | |
| "epoch": 0.54, | |
| "grad_norm": 30.01631729835111, | |
| "learning_rate": 2.618954789559356e-07, | |
| "logps/chosen": -78.71662902832031, | |
| "logps/rejected": -70.70683288574219, | |
| "loss": 0.4583, | |
| "losses/dpo": 0.4225349426269531, | |
| "losses/sft": 0.37885919213294983, | |
| "losses/total": 0.4225349426269531, | |
| "ref_logps/chosen": -70.06804656982422, | |
| "ref_logps/rejected": -53.02828598022461, | |
| "rewards/accuracies": 0.78125, | |
| "rewards/chosen": -0.8648585677146912, | |
| "rewards/margins": 0.9029962420463562, | |
| "rewards/rejected": -1.7678548097610474, | |
| "step": 79 | |
| }, | |
| { | |
| "epoch": 0.54, | |
| "grad_norm": 27.778753710279016, | |
| "learning_rate": 2.5594942438652685e-07, | |
| "logps/chosen": -74.57829284667969, | |
| "logps/rejected": -75.41488647460938, | |
| "loss": 0.4252, | |
| "losses/dpo": 0.4811527132987976, | |
| "losses/sft": 0.3381648361682892, | |
| "losses/total": 0.4811527132987976, | |
| "ref_logps/chosen": -65.43878173828125, | |
| "ref_logps/rejected": -56.21765899658203, | |
| "rewards/accuracies": 0.78125, | |
| "rewards/chosen": -0.9139504432678223, | |
| "rewards/margins": 1.005772352218628, | |
| "rewards/rejected": -1.9197226762771606, | |
| "step": 80 | |
| }, | |
| { | |
| "epoch": 0.55, | |
| "grad_norm": 27.418542939543443, | |
| "learning_rate": 2.5e-07, | |
| "logps/chosen": -84.89808654785156, | |
| "logps/rejected": -77.45452117919922, | |
| "loss": 0.457, | |
| "losses/dpo": 0.4078216552734375, | |
| "losses/sft": 0.3400785028934479, | |
| "losses/total": 0.4078216552734375, | |
| "ref_logps/chosen": -78.10498046875, | |
| "ref_logps/rejected": -61.716976165771484, | |
| "rewards/accuracies": 0.78125, | |
| "rewards/chosen": -0.6793106198310852, | |
| "rewards/margins": 0.8944438695907593, | |
| "rewards/rejected": -1.5737544298171997, | |
| "step": 81 | |
| }, | |
| { | |
| "epoch": 0.56, | |
| "grad_norm": 29.24936699437702, | |
| "learning_rate": 2.4405057561347313e-07, | |
| "logps/chosen": -71.50326538085938, | |
| "logps/rejected": -74.53978729248047, | |
| "loss": 0.496, | |
| "losses/dpo": 0.3848089277744293, | |
| "losses/sft": 0.3731555938720703, | |
| "losses/total": 0.3848089277744293, | |
| "ref_logps/chosen": -60.67548370361328, | |
| "ref_logps/rejected": -55.07884216308594, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": -1.0827784538269043, | |
| "rewards/margins": 0.8633174896240234, | |
| "rewards/rejected": -1.9460958242416382, | |
| "step": 82 | |
| }, | |
| { | |
| "epoch": 0.56, | |
| "grad_norm": 48.23269343517185, | |
| "learning_rate": 2.381045210440644e-07, | |
| "logps/chosen": -78.87052917480469, | |
| "logps/rejected": -73.31388092041016, | |
| "loss": 0.4528, | |
| "losses/dpo": 0.43188297748565674, | |
| "losses/sft": 0.36860212683677673, | |
| "losses/total": 0.43188297748565674, | |
| "ref_logps/chosen": -70.29788970947266, | |
| "ref_logps/rejected": -54.075828552246094, | |
| "rewards/accuracies": 0.796875, | |
| "rewards/chosen": -0.8572635650634766, | |
| "rewards/margins": 1.0665409564971924, | |
| "rewards/rejected": -1.923804521560669, | |
| "step": 83 | |
| }, | |
| { | |
| "epoch": 0.57, | |
| "grad_norm": 36.11750167575466, | |
| "learning_rate": 2.321652042001919e-07, | |
| "logps/chosen": -75.86955261230469, | |
| "logps/rejected": -77.73924255371094, | |
| "loss": 0.5125, | |
| "losses/dpo": 0.49389997124671936, | |
| "losses/sft": 0.4043150544166565, | |
| "losses/total": 0.49389997124671936, | |
| "ref_logps/chosen": -67.49108123779297, | |
| "ref_logps/rejected": -60.198638916015625, | |
| "rewards/accuracies": 0.734375, | |
| "rewards/chosen": -0.8378474712371826, | |
| "rewards/margins": 0.9162120819091797, | |
| "rewards/rejected": -1.7540594339370728, | |
| "step": 84 | |
| }, | |
| { | |
| "epoch": 0.58, | |
| "grad_norm": 44.25422567833337, | |
| "learning_rate": 2.2623598917395436e-07, | |
| "logps/chosen": -78.2550048828125, | |
| "logps/rejected": -69.63221740722656, | |
| "loss": 0.4812, | |
| "losses/dpo": 0.24741792678833008, | |
| "losses/sft": 0.38957494497299194, | |
| "losses/total": 0.24741792678833008, | |
| "ref_logps/chosen": -69.76954650878906, | |
| "ref_logps/rejected": -52.76110076904297, | |
| "rewards/accuracies": 0.765625, | |
| "rewards/chosen": -0.8485451936721802, | |
| "rewards/margins": 0.8385663628578186, | |
| "rewards/rejected": -1.6871116161346436, | |
| "step": 85 | |
| }, | |
| { | |
| "epoch": 0.58, | |
| "grad_norm": 29.36328595493774, | |
| "learning_rate": 2.2032023433567376e-07, | |
| "logps/chosen": -72.15760040283203, | |
| "logps/rejected": -76.89938354492188, | |
| "loss": 0.3683, | |
| "losses/dpo": 0.3277527093887329, | |
| "losses/sft": 0.34014564752578735, | |
| "losses/total": 0.3277527093887329, | |
| "ref_logps/chosen": -63.951847076416016, | |
| "ref_logps/rejected": -56.66627883911133, | |
| "rewards/accuracies": 0.875, | |
| "rewards/chosen": -0.8205756545066833, | |
| "rewards/margins": 1.2027339935302734, | |
| "rewards/rejected": -2.0233094692230225, | |
| "step": 86 | |
| }, | |
| { | |
| "epoch": 0.59, | |
| "grad_norm": 24.453783435169065, | |
| "learning_rate": 2.1442129043167873e-07, | |
| "logps/chosen": -75.3466796875, | |
| "logps/rejected": -76.63810729980469, | |
| "loss": 0.352, | |
| "losses/dpo": 0.4673713743686676, | |
| "losses/sft": 0.3854469656944275, | |
| "losses/total": 0.4673713743686676, | |
| "ref_logps/chosen": -68.66751861572266, | |
| "ref_logps/rejected": -56.724952697753906, | |
| "rewards/accuracies": 0.84375, | |
| "rewards/chosen": -0.6679168939590454, | |
| "rewards/margins": 1.3233978748321533, | |
| "rewards/rejected": -1.9913148880004883, | |
| "step": 87 | |
| }, | |
| { | |
| "epoch": 0.6, | |
| "grad_norm": 28.45352429062705, | |
| "learning_rate": 2.085424986864065e-07, | |
| "logps/chosen": -80.1172103881836, | |
| "logps/rejected": -72.56062316894531, | |
| "loss": 0.4422, | |
| "losses/dpo": 0.3042376637458801, | |
| "losses/sft": 0.28263095021247864, | |
| "losses/total": 0.3042376637458801, | |
| "ref_logps/chosen": -74.51995849609375, | |
| "ref_logps/rejected": -56.53531265258789, | |
| "rewards/accuracies": 0.8125, | |
| "rewards/chosen": -0.5597256422042847, | |
| "rewards/margins": 1.0428049564361572, | |
| "rewards/rejected": -1.602530598640442, | |
| "step": 88 | |
| }, | |
| { | |
| "epoch": 0.6, | |
| "grad_norm": 29.271005488942123, | |
| "learning_rate": 2.0268718890989752e-07, | |
| "logps/chosen": -71.47293853759766, | |
| "logps/rejected": -76.03471374511719, | |
| "loss": 0.4194, | |
| "losses/dpo": 0.4656490683555603, | |
| "losses/sft": 0.3225964307785034, | |
| "losses/total": 0.4656490683555603, | |
| "ref_logps/chosen": -63.167877197265625, | |
| "ref_logps/rejected": -56.3370361328125, | |
| "rewards/accuracies": 0.859375, | |
| "rewards/chosen": -0.8305058479309082, | |
| "rewards/margins": 1.1392616033554077, | |
| "rewards/rejected": -1.9697675704956055, | |
| "step": 89 | |
| }, | |
| { | |
| "epoch": 0.61, | |
| "grad_norm": 35.681125753631115, | |
| "learning_rate": 1.968586776117558e-07, | |
| "logps/chosen": -79.76475524902344, | |
| "logps/rejected": -73.98800659179688, | |
| "loss": 0.4659, | |
| "losses/dpo": 0.35364383459091187, | |
| "losses/sft": 0.3435375690460205, | |
| "losses/total": 0.35364383459091187, | |
| "ref_logps/chosen": -72.63055419921875, | |
| "ref_logps/rejected": -56.81049346923828, | |
| "rewards/accuracies": 0.734375, | |
| "rewards/chosen": -0.7134197950363159, | |
| "rewards/margins": 1.004331350326538, | |
| "rewards/rejected": -1.7177510261535645, | |
| "step": 90 | |
| }, | |
| { | |
| "epoch": 0.62, | |
| "grad_norm": 36.49093259706875, | |
| "learning_rate": 1.9106026612264315e-07, | |
| "logps/chosen": -78.425537109375, | |
| "logps/rejected": -74.1988525390625, | |
| "loss": 0.3985, | |
| "losses/dpo": 0.19245147705078125, | |
| "losses/sft": 0.39025193452835083, | |
| "losses/total": 0.19245147705078125, | |
| "ref_logps/chosen": -69.4635009765625, | |
| "ref_logps/rejected": -53.60189437866211, | |
| "rewards/accuracies": 0.8125, | |
| "rewards/chosen": -0.8962031006813049, | |
| "rewards/margins": 1.1634931564331055, | |
| "rewards/rejected": -2.0596961975097656, | |
| "step": 91 | |
| }, | |
| { | |
| "epoch": 0.63, | |
| "grad_norm": 27.105142818170325, | |
| "learning_rate": 1.8529523872436977e-07, | |
| "logps/chosen": -75.47040557861328, | |
| "logps/rejected": -75.96112060546875, | |
| "loss": 0.4417, | |
| "losses/dpo": 0.3523982763290405, | |
| "losses/sft": 0.3631323575973511, | |
| "losses/total": 0.3523982763290405, | |
| "ref_logps/chosen": -67.11583709716797, | |
| "ref_logps/rejected": -58.03672790527344, | |
| "rewards/accuracies": 0.796875, | |
| "rewards/chosen": -0.8354578018188477, | |
| "rewards/margins": 0.9569823145866394, | |
| "rewards/rejected": -1.7924400568008423, | |
| "step": 92 | |
| }, | |
| { | |
| "epoch": 0.63, | |
| "grad_norm": 38.458971249921234, | |
| "learning_rate": 1.7956686078964255e-07, | |
| "logps/chosen": -72.20645141601562, | |
| "logps/rejected": -72.76055145263672, | |
| "loss": 0.4654, | |
| "losses/dpo": 0.48378482460975647, | |
| "losses/sft": 0.2976914644241333, | |
| "losses/total": 0.48378482460975647, | |
| "ref_logps/chosen": -64.7495346069336, | |
| "ref_logps/rejected": -56.04286193847656, | |
| "rewards/accuracies": 0.8125, | |
| "rewards/chosen": -0.7456916570663452, | |
| "rewards/margins": 0.9260772466659546, | |
| "rewards/rejected": -1.671769142150879, | |
| "step": 93 | |
| }, | |
| { | |
| "epoch": 0.64, | |
| "grad_norm": 21.64298997501433, | |
| "learning_rate": 1.7387837693252327e-07, | |
| "logps/chosen": -80.37052917480469, | |
| "logps/rejected": -79.18669891357422, | |
| "loss": 0.3604, | |
| "losses/dpo": 0.3508569896221161, | |
| "losses/sft": 0.39343804121017456, | |
| "losses/total": 0.3508569896221161, | |
| "ref_logps/chosen": -72.72434997558594, | |
| "ref_logps/rejected": -58.45079803466797, | |
| "rewards/accuracies": 0.875, | |
| "rewards/chosen": -0.764617919921875, | |
| "rewards/margins": 1.3089728355407715, | |
| "rewards/rejected": -2.0735907554626465, | |
| "step": 94 | |
| }, | |
| { | |
| "epoch": 0.65, | |
| "grad_norm": 28.31625318441289, | |
| "learning_rate": 1.6823300917064458e-07, | |
| "logps/chosen": -74.7480697631836, | |
| "logps/rejected": -71.78422546386719, | |
| "loss": 0.4383, | |
| "losses/dpo": 0.17416918277740479, | |
| "losses/sft": 0.3635005056858063, | |
| "losses/total": 0.17416918277740479, | |
| "ref_logps/chosen": -66.6783447265625, | |
| "ref_logps/rejected": -53.81765365600586, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": -0.8069729804992676, | |
| "rewards/margins": 0.9896839261054993, | |
| "rewards/rejected": -1.796656847000122, | |
| "step": 95 | |
| }, | |
| { | |
| "epoch": 0.65, | |
| "grad_norm": 31.43003387376896, | |
| "learning_rate": 1.6263395510022543e-07, | |
| "logps/chosen": -82.76495361328125, | |
| "logps/rejected": -77.74391174316406, | |
| "loss": 0.3885, | |
| "losses/dpo": 0.40873783826828003, | |
| "losses/sft": 0.3334735929965973, | |
| "losses/total": 0.40873783826828003, | |
| "ref_logps/chosen": -74.447998046875, | |
| "ref_logps/rejected": -57.47111511230469, | |
| "rewards/accuracies": 0.84375, | |
| "rewards/chosen": -0.8316947221755981, | |
| "rewards/margins": 1.1955851316452026, | |
| "rewards/rejected": -2.0272796154022217, | |
| "step": 96 | |
| }, | |
| { | |
| "epoch": 0.66, | |
| "grad_norm": 29.307347493310413, | |
| "learning_rate": 1.5708438608491815e-07, | |
| "logps/chosen": -73.7046890258789, | |
| "logps/rejected": -74.98145294189453, | |
| "loss": 0.4818, | |
| "losses/dpo": 0.3637702465057373, | |
| "losses/sft": 0.3260781168937683, | |
| "losses/total": 0.3637702465057373, | |
| "ref_logps/chosen": -67.44843292236328, | |
| "ref_logps/rejected": -59.09331130981445, | |
| "rewards/accuracies": 0.765625, | |
| "rewards/chosen": -0.625625491142273, | |
| "rewards/margins": 0.9631882905960083, | |
| "rewards/rejected": -1.5888137817382812, | |
| "step": 97 | |
| }, | |
| { | |
| "epoch": 0.67, | |
| "grad_norm": 51.123126271441144, | |
| "learning_rate": 1.5158744545951465e-07, | |
| "logps/chosen": -75.0676040649414, | |
| "logps/rejected": -72.10035705566406, | |
| "loss": 0.4318, | |
| "losses/dpo": 0.29737696051597595, | |
| "losses/sft": 0.3478523790836334, | |
| "losses/total": 0.29737696051597595, | |
| "ref_logps/chosen": -67.30294799804688, | |
| "ref_logps/rejected": -53.60346603393555, | |
| "rewards/accuracies": 0.8125, | |
| "rewards/chosen": -0.7764655947685242, | |
| "rewards/margins": 1.073223352432251, | |
| "rewards/rejected": -1.84968900680542, | |
| "step": 98 | |
| }, | |
| { | |
| "epoch": 0.67, | |
| "grad_norm": 26.667202752541407, | |
| "learning_rate": 1.461462467495284e-07, | |
| "logps/chosen": -75.2646484375, | |
| "logps/rejected": -74.6376953125, | |
| "loss": 0.3632, | |
| "losses/dpo": 0.4617415964603424, | |
| "losses/sft": 0.3319079577922821, | |
| "losses/total": 0.4617415964603424, | |
| "ref_logps/chosen": -67.51861572265625, | |
| "ref_logps/rejected": -55.421478271484375, | |
| "rewards/accuracies": 0.890625, | |
| "rewards/chosen": -0.7746031284332275, | |
| "rewards/margins": 1.1470186710357666, | |
| "rewards/rejected": -1.9216216802597046, | |
| "step": 99 | |
| }, | |
| { | |
| "epoch": 0.68, | |
| "grad_norm": 32.0757645021365, | |
| "learning_rate": 1.4076387190766014e-07, | |
| "logps/chosen": -67.63773345947266, | |
| "logps/rejected": -70.43733215332031, | |
| "loss": 0.4539, | |
| "losses/dpo": 0.4691275358200073, | |
| "losses/sft": 0.3476483225822449, | |
| "losses/total": 0.4691275358200073, | |
| "ref_logps/chosen": -57.368900299072266, | |
| "ref_logps/rejected": -50.622802734375, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": -1.0268828868865967, | |
| "rewards/margins": 0.9545705914497375, | |
| "rewards/rejected": -1.9814532995224, | |
| "step": 100 | |
| }, | |
| { | |
| "epoch": 0.69, | |
| "grad_norm": 26.664220237307592, | |
| "learning_rate": 1.354433695681474e-07, | |
| "logps/chosen": -80.13877868652344, | |
| "logps/rejected": -81.61981201171875, | |
| "loss": 0.4337, | |
| "losses/dpo": 0.4848173260688782, | |
| "losses/sft": 0.36751094460487366, | |
| "losses/total": 0.4848173260688782, | |
| "ref_logps/chosen": -72.32598876953125, | |
| "ref_logps/rejected": -63.25422668457031, | |
| "rewards/accuracies": 0.765625, | |
| "rewards/chosen": -0.7812784910202026, | |
| "rewards/margins": 1.0552804470062256, | |
| "rewards/rejected": -1.8365588188171387, | |
| "step": 101 | |
| }, | |
| { | |
| "epoch": 0.69, | |
| "grad_norm": 121.65683713646945, | |
| "learning_rate": 1.301877533199859e-07, | |
| "logps/chosen": -72.0447006225586, | |
| "logps/rejected": -72.38668823242188, | |
| "loss": 0.4732, | |
| "losses/dpo": 0.3546653389930725, | |
| "losses/sft": 0.34984883666038513, | |
| "losses/total": 0.3546653389930725, | |
| "ref_logps/chosen": -63.54951095581055, | |
| "ref_logps/rejected": -53.592952728271484, | |
| "rewards/accuracies": 0.734375, | |
| "rewards/chosen": -0.8495190739631653, | |
| "rewards/margins": 1.0298542976379395, | |
| "rewards/rejected": -1.87937331199646, | |
| "step": 102 | |
| }, | |
| { | |
| "epoch": 0.7, | |
| "grad_norm": 32.59677272775599, | |
| "learning_rate": 1.2500000000000005e-07, | |
| "logps/chosen": -66.81201934814453, | |
| "logps/rejected": -67.73169708251953, | |
| "loss": 0.4377, | |
| "losses/dpo": 0.4282957911491394, | |
| "losses/sft": 0.3223702311515808, | |
| "losses/total": 0.4282957911491394, | |
| "ref_logps/chosen": -59.40269088745117, | |
| "ref_logps/rejected": -50.77289581298828, | |
| "rewards/accuracies": 0.84375, | |
| "rewards/chosen": -0.7409323453903198, | |
| "rewards/margins": 0.9549475908279419, | |
| "rewards/rejected": -1.6958800554275513, | |
| "step": 103 | |
| }, | |
| { | |
| "epoch": 0.71, | |
| "grad_norm": 24.94130531155071, | |
| "learning_rate": 1.1988304800673196e-07, | |
| "logps/chosen": -69.9290542602539, | |
| "logps/rejected": -70.86064147949219, | |
| "loss": 0.4243, | |
| "losses/dpo": 0.4698595404624939, | |
| "losses/sft": 0.37894606590270996, | |
| "losses/total": 0.4698595404624939, | |
| "ref_logps/chosen": -62.15961456298828, | |
| "ref_logps/rejected": -51.80973815917969, | |
| "rewards/accuracies": 0.78125, | |
| "rewards/chosen": -0.7769437432289124, | |
| "rewards/margins": 1.1281468868255615, | |
| "rewards/rejected": -1.9050908088684082, | |
| "step": 104 | |
| }, | |
| { | |
| "epoch": 0.71, | |
| "grad_norm": 29.429373971218872, | |
| "learning_rate": 1.1483979563610069e-07, | |
| "logps/chosen": -78.253173828125, | |
| "logps/rejected": -75.22711181640625, | |
| "loss": 0.4335, | |
| "losses/dpo": 0.2583315968513489, | |
| "losses/sft": 0.35668855905532837, | |
| "losses/total": 0.2583315968513489, | |
| "ref_logps/chosen": -68.73954010009766, | |
| "ref_logps/rejected": -54.715335845947266, | |
| "rewards/accuracies": 0.765625, | |
| "rewards/chosen": -0.9513629674911499, | |
| "rewards/margins": 1.0998142957687378, | |
| "rewards/rejected": -2.0511772632598877, | |
| "step": 105 | |
| }, | |
| { | |
| "epoch": 0.72, | |
| "grad_norm": 27.54291684365646, | |
| "learning_rate": 1.0987309943977646e-07, | |
| "logps/chosen": -79.83787536621094, | |
| "logps/rejected": -78.37857055664062, | |
| "loss": 0.3904, | |
| "losses/dpo": 0.4721170663833618, | |
| "losses/sft": 0.3833865225315094, | |
| "losses/total": 0.4721170663833618, | |
| "ref_logps/chosen": -71.21440887451172, | |
| "ref_logps/rejected": -57.57903289794922, | |
| "rewards/accuracies": 0.8125, | |
| "rewards/chosen": -0.8623473048210144, | |
| "rewards/margins": 1.2176061868667603, | |
| "rewards/rejected": -2.07995343208313, | |
| "step": 106 | |
| }, | |
| { | |
| "epoch": 0.73, | |
| "grad_norm": 28.492473761434397, | |
| "learning_rate": 1.0498577260720048e-07, | |
| "logps/chosen": -78.24122619628906, | |
| "logps/rejected": -77.19696044921875, | |
| "loss": 0.4256, | |
| "losses/dpo": 0.5148700475692749, | |
| "losses/sft": 0.3965683579444885, | |
| "losses/total": 0.5148700475692749, | |
| "ref_logps/chosen": -72.94232177734375, | |
| "ref_logps/rejected": -62.735538482666016, | |
| "rewards/accuracies": 0.8125, | |
| "rewards/chosen": -0.5298899412155151, | |
| "rewards/margins": 0.9162522554397583, | |
| "rewards/rejected": -1.4461421966552734, | |
| "step": 107 | |
| }, | |
| { | |
| "epoch": 0.73, | |
| "grad_norm": 20.982493953158375, | |
| "learning_rate": 1.0018058337216326e-07, | |
| "logps/chosen": -76.80966186523438, | |
| "logps/rejected": -72.25653839111328, | |
| "loss": 0.3378, | |
| "losses/dpo": 0.3177239000797272, | |
| "losses/sft": 0.333109587430954, | |
| "losses/total": 0.3177239000797272, | |
| "ref_logps/chosen": -69.32987976074219, | |
| "ref_logps/rejected": -51.46967697143555, | |
| "rewards/accuracies": 0.890625, | |
| "rewards/chosen": -0.7479775547981262, | |
| "rewards/margins": 1.3307081460952759, | |
| "rewards/rejected": -2.078685760498047, | |
| "step": 108 | |
| }, | |
| { | |
| "epoch": 0.74, | |
| "grad_norm": 64.19848206989985, | |
| "learning_rate": 9.546025344484868e-08, | |
| "logps/chosen": -80.88861083984375, | |
| "logps/rejected": -78.88575744628906, | |
| "loss": 0.4023, | |
| "losses/dpo": 0.39368999004364014, | |
| "losses/sft": 0.34988299012184143, | |
| "losses/total": 0.39368999004364014, | |
| "ref_logps/chosen": -73.32632446289062, | |
| "ref_logps/rejected": -61.007301330566406, | |
| "rewards/accuracies": 0.890625, | |
| "rewards/chosen": -0.7562285661697388, | |
| "rewards/margins": 1.0316171646118164, | |
| "rewards/rejected": -1.7878458499908447, | |
| "step": 109 | |
| }, | |
| { | |
| "epoch": 0.75, | |
| "grad_norm": 23.38411036651133, | |
| "learning_rate": 9.082745647022797e-08, | |
| "logps/chosen": -81.5500717163086, | |
| "logps/rejected": -78.32415771484375, | |
| "loss": 0.3158, | |
| "losses/dpo": 0.32109877467155457, | |
| "losses/sft": 0.4066527783870697, | |
| "losses/total": 0.32109877467155457, | |
| "ref_logps/chosen": -74.28262329101562, | |
| "ref_logps/rejected": -55.555335998535156, | |
| "rewards/accuracies": 0.875, | |
| "rewards/chosen": -0.7267451882362366, | |
| "rewards/margins": 1.5501360893249512, | |
| "rewards/rejected": -2.276881217956543, | |
| "step": 110 | |
| }, | |
| { | |
| "epoch": 0.75, | |
| "grad_norm": 40.260701911712964, | |
| "learning_rate": 8.628481651367875e-08, | |
| "logps/chosen": -74.5044174194336, | |
| "logps/rejected": -76.19696044921875, | |
| "loss": 0.3743, | |
| "losses/dpo": 0.31068116426467896, | |
| "losses/sft": 0.34140655398368835, | |
| "losses/total": 0.31068116426467896, | |
| "ref_logps/chosen": -64.63957214355469, | |
| "ref_logps/rejected": -53.67190170288086, | |
| "rewards/accuracies": 0.84375, | |
| "rewards/chosen": -0.986484169960022, | |
| "rewards/margins": 1.266021728515625, | |
| "rewards/rejected": -2.2525057792663574, | |
| "step": 111 | |
| }, | |
| { | |
| "epoch": 0.76, | |
| "grad_norm": 31.80013523303015, | |
| "learning_rate": 8.183490657468686e-08, | |
| "logps/chosen": -71.22296142578125, | |
| "logps/rejected": -71.00870513916016, | |
| "loss": 0.5294, | |
| "losses/dpo": 0.4347510039806366, | |
| "losses/sft": 0.36885449290275574, | |
| "losses/total": 0.4347510039806366, | |
| "ref_logps/chosen": -60.00767517089844, | |
| "ref_logps/rejected": -51.94877624511719, | |
| "rewards/accuracies": 0.71875, | |
| "rewards/chosen": -1.1215291023254395, | |
| "rewards/margins": 0.7844638228416443, | |
| "rewards/rejected": -1.905992865562439, | |
| "step": 112 | |
| }, | |
| { | |
| "epoch": 0.77, | |
| "grad_norm": 26.736632589832187, | |
| "learning_rate": 7.748024712947204e-08, | |
| "logps/chosen": -79.0911636352539, | |
| "logps/rejected": -78.3819580078125, | |
| "loss": 0.3736, | |
| "losses/dpo": 0.38549065589904785, | |
| "losses/sft": 0.36682289838790894, | |
| "losses/total": 0.38549065589904785, | |
| "ref_logps/chosen": -71.06074523925781, | |
| "ref_logps/rejected": -57.33882141113281, | |
| "rewards/accuracies": 0.765625, | |
| "rewards/chosen": -0.8030425906181335, | |
| "rewards/margins": 1.301270842552185, | |
| "rewards/rejected": -2.104313373565674, | |
| "step": 113 | |
| }, | |
| { | |
| "epoch": 0.77, | |
| "grad_norm": 27.914463891065783, | |
| "learning_rate": 7.322330470336313e-08, | |
| "logps/chosen": -79.21043395996094, | |
| "logps/rejected": -78.50171661376953, | |
| "loss": 0.4552, | |
| "losses/dpo": 0.4180038869380951, | |
| "losses/sft": 0.3502272665500641, | |
| "losses/total": 0.4180038869380951, | |
| "ref_logps/chosen": -70.78549194335938, | |
| "ref_logps/rejected": -59.83561706542969, | |
| "rewards/accuracies": 0.78125, | |
| "rewards/chosen": -0.8424935340881348, | |
| "rewards/margins": 1.024115800857544, | |
| "rewards/rejected": -1.8666093349456787, | |
| "step": 114 | |
| }, | |
| { | |
| "epoch": 0.78, | |
| "grad_norm": 36.14683391187098, | |
| "learning_rate": 6.906649047373245e-08, | |
| "logps/chosen": -66.70146179199219, | |
| "logps/rejected": -67.81217193603516, | |
| "loss": 0.3941, | |
| "losses/dpo": 0.3595428466796875, | |
| "losses/sft": 0.4029202461242676, | |
| "losses/total": 0.3595428466796875, | |
| "ref_logps/chosen": -57.97339630126953, | |
| "ref_logps/rejected": -47.349143981933594, | |
| "rewards/accuracies": 0.765625, | |
| "rewards/chosen": -0.8728064298629761, | |
| "rewards/margins": 1.173496961593628, | |
| "rewards/rejected": -2.0463035106658936, | |
| "step": 115 | |
| }, | |
| { | |
| "epoch": 0.79, | |
| "grad_norm": 34.93282588073872, | |
| "learning_rate": 6.501215890427908e-08, | |
| "logps/chosen": -74.01496887207031, | |
| "logps/rejected": -71.6580810546875, | |
| "loss": 0.4737, | |
| "losses/dpo": 0.47708314657211304, | |
| "losses/sft": 0.3861905336380005, | |
| "losses/total": 0.47708314657211304, | |
| "ref_logps/chosen": -66.82437896728516, | |
| "ref_logps/rejected": -56.22287368774414, | |
| "rewards/accuracies": 0.765625, | |
| "rewards/chosen": -0.7190604209899902, | |
| "rewards/margins": 0.8244602680206299, | |
| "rewards/rejected": -1.5435206890106201, | |
| "step": 116 | |
| }, | |
| { | |
| "epoch": 0.8, | |
| "grad_norm": 33.61435457152851, | |
| "learning_rate": 6.106260641143546e-08, | |
| "logps/chosen": -79.60284423828125, | |
| "logps/rejected": -77.92535400390625, | |
| "loss": 0.4682, | |
| "losses/dpo": 0.290701687335968, | |
| "losses/sft": 0.3620028495788574, | |
| "losses/total": 0.290701687335968, | |
| "ref_logps/chosen": -70.14583587646484, | |
| "ref_logps/rejected": -59.40486526489258, | |
| "rewards/accuracies": 0.78125, | |
| "rewards/chosen": -0.9457005262374878, | |
| "rewards/margins": 0.9063488245010376, | |
| "rewards/rejected": -1.8520495891571045, | |
| "step": 117 | |
| }, | |
| { | |
| "epoch": 0.8, | |
| "grad_norm": 32.27039772238261, | |
| "learning_rate": 5.722007006365309e-08, | |
| "logps/chosen": -85.39405059814453, | |
| "logps/rejected": -77.39488220214844, | |
| "loss": 0.4029, | |
| "losses/dpo": 0.29392948746681213, | |
| "losses/sft": 0.4053553342819214, | |
| "losses/total": 0.29392948746681213, | |
| "ref_logps/chosen": -75.33457946777344, | |
| "ref_logps/rejected": -56.223270416259766, | |
| "rewards/accuracies": 0.84375, | |
| "rewards/chosen": -1.0059471130371094, | |
| "rewards/margins": 1.111214280128479, | |
| "rewards/rejected": -2.117161273956299, | |
| "step": 118 | |
| }, | |
| { | |
| "epoch": 0.81, | |
| "grad_norm": 30.691331571937976, | |
| "learning_rate": 5.3486726314303175e-08, | |
| "logps/chosen": -76.40483856201172, | |
| "logps/rejected": -76.4593734741211, | |
| "loss": 0.4526, | |
| "losses/dpo": 0.48409292101860046, | |
| "losses/sft": 0.3616543412208557, | |
| "losses/total": 0.48409292101860046, | |
| "ref_logps/chosen": -66.07306671142578, | |
| "ref_logps/rejected": -56.153717041015625, | |
| "rewards/accuracies": 0.828125, | |
| "rewards/chosen": -1.0331764221191406, | |
| "rewards/margins": 0.9973889589309692, | |
| "rewards/rejected": -2.0305655002593994, | |
| "step": 119 | |
| }, | |
| { | |
| "epoch": 0.82, | |
| "grad_norm": 29.441590057939685, | |
| "learning_rate": 4.986468976890992e-08, | |
| "logps/chosen": -75.17059326171875, | |
| "logps/rejected": -73.5225830078125, | |
| "loss": 0.4307, | |
| "losses/dpo": 0.33158645033836365, | |
| "losses/sft": 0.3496292531490326, | |
| "losses/total": 0.33158645033836365, | |
| "ref_logps/chosen": -68.26935577392578, | |
| "ref_logps/rejected": -55.77709197998047, | |
| "rewards/accuracies": 0.8125, | |
| "rewards/chosen": -0.69012451171875, | |
| "rewards/margins": 1.0844244956970215, | |
| "rewards/rejected": -1.774549126625061, | |
| "step": 120 | |
| }, | |
| { | |
| "epoch": 0.82, | |
| "grad_norm": 24.37008051594277, | |
| "learning_rate": 4.635601198741607e-08, | |
| "logps/chosen": -73.0021743774414, | |
| "logps/rejected": -74.11600494384766, | |
| "loss": 0.3949, | |
| "losses/dpo": 0.5414682030677795, | |
| "losses/sft": 0.37046751379966736, | |
| "losses/total": 0.5414682030677795, | |
| "ref_logps/chosen": -64.94493103027344, | |
| "ref_logps/rejected": -54.75990676879883, | |
| "rewards/accuracies": 0.8125, | |
| "rewards/chosen": -0.8057246804237366, | |
| "rewards/margins": 1.1298854351043701, | |
| "rewards/rejected": -1.935610294342041, | |
| "step": 121 | |
| }, | |
| { | |
| "epoch": 0.83, | |
| "grad_norm": 26.892127512970333, | |
| "learning_rate": 4.296268032215733e-08, | |
| "logps/chosen": -75.42649841308594, | |
| "logps/rejected": -76.27777099609375, | |
| "loss": 0.4499, | |
| "losses/dpo": 0.5582671761512756, | |
| "losses/sft": 0.3608691692352295, | |
| "losses/total": 0.5582671761512756, | |
| "ref_logps/chosen": -66.92321014404297, | |
| "ref_logps/rejected": -57.275753021240234, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": -0.8503296971321106, | |
| "rewards/margins": 1.0498719215393066, | |
| "rewards/rejected": -1.9002017974853516, | |
| "step": 122 | |
| }, | |
| { | |
| "epoch": 0.84, | |
| "grad_norm": 41.688513331786744, | |
| "learning_rate": 3.968661679220467e-08, | |
| "logps/chosen": -72.4287338256836, | |
| "logps/rejected": -75.88028717041016, | |
| "loss": 0.4095, | |
| "losses/dpo": 0.6144567131996155, | |
| "losses/sft": 0.3270949125289917, | |
| "losses/total": 0.6144567131996155, | |
| "ref_logps/chosen": -63.542545318603516, | |
| "ref_logps/rejected": -54.83192443847656, | |
| "rewards/accuracies": 0.84375, | |
| "rewards/chosen": -0.8886189460754395, | |
| "rewards/margins": 1.2162187099456787, | |
| "rewards/rejected": -2.104837656021118, | |
| "step": 123 | |
| }, | |
| { | |
| "epoch": 0.84, | |
| "grad_norm": 25.469129701195566, | |
| "learning_rate": 3.652967699471299e-08, | |
| "logps/chosen": -74.50592803955078, | |
| "logps/rejected": -72.23234558105469, | |
| "loss": 0.3866, | |
| "losses/dpo": 0.28718867897987366, | |
| "losses/sft": 0.37371402978897095, | |
| "losses/total": 0.28718867897987366, | |
| "ref_logps/chosen": -65.68087768554688, | |
| "ref_logps/rejected": -51.324337005615234, | |
| "rewards/accuracies": 0.828125, | |
| "rewards/chosen": -0.8825055360794067, | |
| "rewards/margins": 1.2082949876785278, | |
| "rewards/rejected": -2.0908007621765137, | |
| "step": 124 | |
| }, | |
| { | |
| "epoch": 0.85, | |
| "grad_norm": 35.98399239694971, | |
| "learning_rate": 3.349364905389032e-08, | |
| "logps/chosen": -79.32814025878906, | |
| "logps/rejected": -75.2787094116211, | |
| "loss": 0.3999, | |
| "losses/dpo": 0.5298404693603516, | |
| "losses/sft": 0.36322808265686035, | |
| "losses/total": 0.5298404693603516, | |
| "ref_logps/chosen": -72.24817657470703, | |
| "ref_logps/rejected": -56.569156646728516, | |
| "rewards/accuracies": 0.859375, | |
| "rewards/chosen": -0.7079958915710449, | |
| "rewards/margins": 1.1629595756530762, | |
| "rewards/rejected": -1.8709555864334106, | |
| "step": 125 | |
| }, | |
| { | |
| "epoch": 0.86, | |
| "grad_norm": 29.48115104276339, | |
| "learning_rate": 3.058025260818609e-08, | |
| "logps/chosen": -79.290283203125, | |
| "logps/rejected": -73.2989730834961, | |
| "loss": 0.4999, | |
| "losses/dpo": 0.6051620841026306, | |
| "losses/sft": 0.35250741243362427, | |
| "losses/total": 0.6051620841026306, | |
| "ref_logps/chosen": -70.70803833007812, | |
| "ref_logps/rejected": -55.791748046875, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": -0.8582238554954529, | |
| "rewards/margins": 0.8924983143806458, | |
| "rewards/rejected": -1.7507224082946777, | |
| "step": 126 | |
| }, | |
| { | |
| "epoch": 0.86, | |
| "grad_norm": 24.758070012254365, | |
| "learning_rate": 2.7791137836269158e-08, | |
| "logps/chosen": -78.37055969238281, | |
| "logps/rejected": -74.77693939208984, | |
| "loss": 0.3954, | |
| "losses/dpo": 0.3199983537197113, | |
| "losses/sft": 0.39879244565963745, | |
| "losses/total": 0.3199983537197113, | |
| "ref_logps/chosen": -69.10183715820312, | |
| "ref_logps/rejected": -54.88367462158203, | |
| "rewards/accuracies": 0.78125, | |
| "rewards/chosen": -0.9268715381622314, | |
| "rewards/margins": 1.062455177307129, | |
| "rewards/rejected": -1.98932683467865, | |
| "step": 127 | |
| }, | |
| { | |
| "epoch": 0.87, | |
| "grad_norm": 20.827088770452676, | |
| "learning_rate": 2.512788452234921e-08, | |
| "logps/chosen": -70.80789184570312, | |
| "logps/rejected": -75.08622741699219, | |
| "loss": 0.3325, | |
| "losses/dpo": 0.3453621566295624, | |
| "losses/sft": 0.3389894664287567, | |
| "losses/total": 0.3453621566295624, | |
| "ref_logps/chosen": -61.59326934814453, | |
| "ref_logps/rejected": -52.41365432739258, | |
| "rewards/accuracies": 0.859375, | |
| "rewards/chosen": -0.9214622974395752, | |
| "rewards/margins": 1.345794677734375, | |
| "rewards/rejected": -2.26725697517395, | |
| "step": 128 | |
| }, | |
| { | |
| "epoch": 0.88, | |
| "grad_norm": 31.45091455444117, | |
| "learning_rate": 2.259200116137039e-08, | |
| "logps/chosen": -76.39757537841797, | |
| "logps/rejected": -77.06990051269531, | |
| "loss": 0.4837, | |
| "losses/dpo": 0.6226012706756592, | |
| "losses/sft": 0.37185585498809814, | |
| "losses/total": 0.6226012706756592, | |
| "ref_logps/chosen": -66.43987274169922, | |
| "ref_logps/rejected": -58.27755355834961, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": -0.9957698583602905, | |
| "rewards/margins": 0.8834652900695801, | |
| "rewards/rejected": -1.8792351484298706, | |
| "step": 129 | |
| }, | |
| { | |
| "epoch": 0.88, | |
| "grad_norm": 25.29491524946383, | |
| "learning_rate": 2.0184924104583612e-08, | |
| "logps/chosen": -69.09542083740234, | |
| "logps/rejected": -73.32018280029297, | |
| "loss": 0.4023, | |
| "losses/dpo": 0.36410456895828247, | |
| "losses/sft": 0.33447620272636414, | |
| "losses/total": 0.36410456895828247, | |
| "ref_logps/chosen": -59.17249298095703, | |
| "ref_logps/rejected": -51.49285125732422, | |
| "rewards/accuracies": 0.828125, | |
| "rewards/chosen": -0.992293655872345, | |
| "rewards/margins": 1.190440058708191, | |
| "rewards/rejected": -2.1827335357666016, | |
| "step": 130 | |
| }, | |
| { | |
| "epoch": 0.89, | |
| "grad_norm": 27.582904411320246, | |
| "learning_rate": 1.7908016745981856e-08, | |
| "logps/chosen": -86.50100708007812, | |
| "logps/rejected": -82.30694580078125, | |
| "loss": 0.4445, | |
| "losses/dpo": 0.4145382344722748, | |
| "losses/sft": 0.346317321062088, | |
| "losses/total": 0.4145382344722748, | |
| "ref_logps/chosen": -77.55247497558594, | |
| "ref_logps/rejected": -61.19115447998047, | |
| "rewards/accuracies": 0.78125, | |
| "rewards/chosen": -0.8948546648025513, | |
| "rewards/margins": 1.2167249917984009, | |
| "rewards/rejected": -2.111579418182373, | |
| "step": 131 | |
| }, | |
| { | |
| "epoch": 0.9, | |
| "grad_norm": 28.77405060036572, | |
| "learning_rate": 1.57625687500596e-08, | |
| "logps/chosen": -75.88212585449219, | |
| "logps/rejected": -74.88499450683594, | |
| "loss": 0.5229, | |
| "losses/dpo": 0.4454580843448639, | |
| "losses/sft": 0.36321359872817993, | |
| "losses/total": 0.4454580843448639, | |
| "ref_logps/chosen": -67.06256866455078, | |
| "ref_logps/rejected": -57.12347412109375, | |
| "rewards/accuracies": 0.71875, | |
| "rewards/chosen": -0.8819558024406433, | |
| "rewards/margins": 0.8941954970359802, | |
| "rewards/rejected": -1.776151418685913, | |
| "step": 132 | |
| }, | |
| { | |
| "epoch": 0.9, | |
| "grad_norm": 27.7470456349191, | |
| "learning_rate": 1.3749795321332885e-08, | |
| "logps/chosen": -73.93744659423828, | |
| "logps/rejected": -76.04200744628906, | |
| "loss": 0.3594, | |
| "losses/dpo": 0.17011409997940063, | |
| "losses/sft": 0.40163373947143555, | |
| "losses/total": 0.17011409997940063, | |
| "ref_logps/chosen": -64.7332534790039, | |
| "ref_logps/rejected": -53.972320556640625, | |
| "rewards/accuracies": 0.9375, | |
| "rewards/chosen": -0.9204192757606506, | |
| "rewards/margins": 1.2865492105484009, | |
| "rewards/rejected": -2.206968307495117, | |
| "step": 133 | |
| }, | |
| { | |
| "epoch": 0.91, | |
| "grad_norm": 36.37444245491115, | |
| "learning_rate": 1.1870836516034876e-08, | |
| "logps/chosen": -82.49986267089844, | |
| "logps/rejected": -78.48277282714844, | |
| "loss": 0.481, | |
| "losses/dpo": 0.45800262689590454, | |
| "losses/sft": 0.4599795341491699, | |
| "losses/total": 0.45800262689590454, | |
| "ref_logps/chosen": -73.60972595214844, | |
| "ref_logps/rejected": -60.565696716308594, | |
| "rewards/accuracies": 0.828125, | |
| "rewards/chosen": -0.8890135288238525, | |
| "rewards/margins": 0.9026941657066345, | |
| "rewards/rejected": -1.7917077541351318, | |
| "step": 134 | |
| }, | |
| { | |
| "epoch": 0.92, | |
| "grad_norm": 85.09720359081324, | |
| "learning_rate": 1.0126756596375685e-08, | |
| "logps/chosen": -69.55728149414062, | |
| "logps/rejected": -67.4758071899414, | |
| "loss": 0.4461, | |
| "losses/dpo": 0.3670344650745392, | |
| "losses/sft": 0.37169790267944336, | |
| "losses/total": 0.3670344650745392, | |
| "ref_logps/chosen": -60.00959396362305, | |
| "ref_logps/rejected": -48.21699905395508, | |
| "rewards/accuracies": 0.796875, | |
| "rewards/chosen": -0.9547686576843262, | |
| "rewards/margins": 0.9711121916770935, | |
| "rewards/rejected": -1.9258809089660645, | |
| "step": 135 | |
| }, | |
| { | |
| "epoch": 0.92, | |
| "grad_norm": 25.805500137707952, | |
| "learning_rate": 8.518543427732949e-09, | |
| "logps/chosen": -77.5588607788086, | |
| "logps/rejected": -72.98957824707031, | |
| "loss": 0.3857, | |
| "losses/dpo": 0.34388694167137146, | |
| "losses/sft": 0.39463409781455994, | |
| "losses/total": 0.34388694167137146, | |
| "ref_logps/chosen": -71.34376525878906, | |
| "ref_logps/rejected": -55.052791595458984, | |
| "rewards/accuracies": 0.84375, | |
| "rewards/chosen": -0.6215101480484009, | |
| "rewards/margins": 1.1721692085266113, | |
| "rewards/rejected": -1.7936793565750122, | |
| "step": 136 | |
| }, | |
| { | |
| "epoch": 0.93, | |
| "grad_norm": 28.925653233926095, | |
| "learning_rate": 7.047107919114586e-09, | |
| "logps/chosen": -76.60894775390625, | |
| "logps/rejected": -67.70733642578125, | |
| "loss": 0.5413, | |
| "losses/dpo": 0.40706807374954224, | |
| "losses/sft": 0.38619473576545715, | |
| "losses/total": 0.40706807374954224, | |
| "ref_logps/chosen": -65.6761245727539, | |
| "ref_logps/rejected": -48.80441665649414, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": -1.0932811498641968, | |
| "rewards/margins": 0.7970114350318909, | |
| "rewards/rejected": -1.8902926445007324, | |
| "step": 137 | |
| }, | |
| { | |
| "epoch": 0.94, | |
| "grad_norm": 28.194863492832845, | |
| "learning_rate": 5.7132835072101486e-09, | |
| "logps/chosen": -73.64652252197266, | |
| "logps/rejected": -75.85942077636719, | |
| "loss": 0.3847, | |
| "losses/dpo": 0.4270258843898773, | |
| "losses/sft": 0.4282100200653076, | |
| "losses/total": 0.4270258843898773, | |
| "ref_logps/chosen": -65.63545227050781, | |
| "ref_logps/rejected": -55.20812225341797, | |
| "rewards/accuracies": 0.8125, | |
| "rewards/chosen": -0.8011073470115662, | |
| "rewards/margins": 1.2640223503112793, | |
| "rewards/rejected": -2.0651297569274902, | |
| "step": 138 | |
| }, | |
| { | |
| "epoch": 0.94, | |
| "grad_norm": 24.471573256004906, | |
| "learning_rate": 4.517825684323323e-09, | |
| "logps/chosen": -76.01177215576172, | |
| "logps/rejected": -72.83094787597656, | |
| "loss": 0.3493, | |
| "losses/dpo": 0.42532169818878174, | |
| "losses/sft": 0.38909339904785156, | |
| "losses/total": 0.42532169818878174, | |
| "ref_logps/chosen": -69.16812896728516, | |
| "ref_logps/rejected": -52.844810485839844, | |
| "rewards/accuracies": 0.90625, | |
| "rewards/chosen": -0.6843640208244324, | |
| "rewards/margins": 1.3142496347427368, | |
| "rewards/rejected": -1.9986138343811035, | |
| "step": 139 | |
| }, | |
| { | |
| "epoch": 0.95, | |
| "grad_norm": 28.104692013897658, | |
| "learning_rate": 3.4614115704533766e-09, | |
| "logps/chosen": -80.1229019165039, | |
| "logps/rejected": -77.62194061279297, | |
| "loss": 0.3651, | |
| "losses/dpo": 0.3558647334575653, | |
| "losses/sft": 0.3259223401546478, | |
| "losses/total": 0.3558647334575653, | |
| "ref_logps/chosen": -71.98576354980469, | |
| "ref_logps/rejected": -57.123634338378906, | |
| "rewards/accuracies": 0.84375, | |
| "rewards/chosen": -0.8137134313583374, | |
| "rewards/margins": 1.2361174821853638, | |
| "rewards/rejected": -2.049830913543701, | |
| "step": 140 | |
| }, | |
| { | |
| "epoch": 0.96, | |
| "grad_norm": 30.06950212059891, | |
| "learning_rate": 2.5446395297668287e-09, | |
| "logps/chosen": -77.057373046875, | |
| "logps/rejected": -77.57244873046875, | |
| "loss": 0.3862, | |
| "losses/dpo": 0.46156057715415955, | |
| "losses/sft": 0.3272428512573242, | |
| "losses/total": 0.46156057715415955, | |
| "ref_logps/chosen": -69.33092498779297, | |
| "ref_logps/rejected": -57.40209197998047, | |
| "rewards/accuracies": 0.78125, | |
| "rewards/chosen": -0.772644579410553, | |
| "rewards/margins": 1.2443914413452148, | |
| "rewards/rejected": -2.017035961151123, | |
| "step": 141 | |
| }, | |
| { | |
| "epoch": 0.97, | |
| "grad_norm": 22.27867702599573, | |
| "learning_rate": 1.7680288316779256e-09, | |
| "logps/chosen": -76.52747344970703, | |
| "logps/rejected": -76.79586029052734, | |
| "loss": 0.3613, | |
| "losses/dpo": 0.38265928626060486, | |
| "losses/sft": 0.330098032951355, | |
| "losses/total": 0.38265928626060486, | |
| "ref_logps/chosen": -67.75516510009766, | |
| "ref_logps/rejected": -55.386016845703125, | |
| "rewards/accuracies": 0.859375, | |
| "rewards/chosen": -0.8772311806678772, | |
| "rewards/margins": 1.263753056526184, | |
| "rewards/rejected": -2.140984058380127, | |
| "step": 142 | |
| }, | |
| { | |
| "epoch": 0.97, | |
| "grad_norm": 33.24181116137987, | |
| "learning_rate": 1.1320193567288527e-09, | |
| "logps/chosen": -68.91748809814453, | |
| "logps/rejected": -68.46461486816406, | |
| "loss": 0.4453, | |
| "losses/dpo": 0.6369032859802246, | |
| "losses/sft": 0.38167503476142883, | |
| "losses/total": 0.6369032859802246, | |
| "ref_logps/chosen": -59.43241500854492, | |
| "ref_logps/rejected": -47.996742248535156, | |
| "rewards/accuracies": 0.78125, | |
| "rewards/chosen": -0.9485068321228027, | |
| "rewards/margins": 1.0982803106307983, | |
| "rewards/rejected": -2.0467872619628906, | |
| "step": 143 | |
| }, | |
| { | |
| "epoch": 0.98, | |
| "grad_norm": 27.26573554059411, | |
| "learning_rate": 6.369713474366212e-10, | |
| "logps/chosen": -79.28267669677734, | |
| "logps/rejected": -79.29057312011719, | |
| "loss": 0.4268, | |
| "losses/dpo": 0.3871089816093445, | |
| "losses/sft": 0.3575269877910614, | |
| "losses/total": 0.3871089816093445, | |
| "ref_logps/chosen": -70.6344985961914, | |
| "ref_logps/rejected": -60.456382751464844, | |
| "rewards/accuracies": 0.8125, | |
| "rewards/chosen": -0.864818811416626, | |
| "rewards/margins": 1.018599510192871, | |
| "rewards/rejected": -1.883418321609497, | |
| "step": 144 | |
| }, | |
| { | |
| "epoch": 0.99, | |
| "grad_norm": 28.26392966654158, | |
| "learning_rate": 2.831652042480093e-10, | |
| "logps/chosen": -71.9800796508789, | |
| "logps/rejected": -73.54934692382812, | |
| "loss": 0.4465, | |
| "losses/dpo": 0.4069312512874603, | |
| "losses/sft": 0.35074400901794434, | |
| "losses/total": 0.4069312512874603, | |
| "ref_logps/chosen": -62.18163299560547, | |
| "ref_logps/rejected": -53.78350067138672, | |
| "rewards/accuracies": 0.765625, | |
| "rewards/chosen": -0.9798437356948853, | |
| "rewards/margins": 0.996740460395813, | |
| "rewards/rejected": -1.9765844345092773, | |
| "step": 145 | |
| }, | |
| { | |
| "epoch": 0.99, | |
| "grad_norm": 23.91111528235258, | |
| "learning_rate": 7.080132671774541e-11, | |
| "logps/chosen": -75.3148422241211, | |
| "logps/rejected": -74.4091567993164, | |
| "loss": 0.3508, | |
| "losses/dpo": 0.42828118801116943, | |
| "losses/sft": 0.41488099098205566, | |
| "losses/total": 0.42828118801116943, | |
| "ref_logps/chosen": -68.02532196044922, | |
| "ref_logps/rejected": -53.932071685791016, | |
| "rewards/accuracies": 0.921875, | |
| "rewards/chosen": -0.7289534211158752, | |
| "rewards/margins": 1.3187549114227295, | |
| "rewards/rejected": -2.047708511352539, | |
| "step": 146 | |
| }, | |
| { | |
| "epoch": 1.0, | |
| "grad_norm": 25.683687754039113, | |
| "learning_rate": 0.0, | |
| "logps/chosen": -77.05089569091797, | |
| "logps/rejected": -74.92003631591797, | |
| "loss": 0.4269, | |
| "losses/dpo": 0.3264519274234772, | |
| "losses/sft": 0.35349562764167786, | |
| "losses/total": 0.3264519274234772, | |
| "ref_logps/chosen": -69.42140197753906, | |
| "ref_logps/rejected": -56.2528190612793, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": -0.7629485130310059, | |
| "rewards/margins": 1.103774070739746, | |
| "rewards/rejected": -1.8667224645614624, | |
| "step": 147 | |
| }, | |
| { | |
| "epoch": 1.0, | |
| "step": 147, | |
| "total_flos": 0.0, | |
| "train_loss": 0.4901739539337807, | |
| "train_runtime": 4483.0286, | |
| "train_samples_per_second": 2.102, | |
| "train_steps_per_second": 0.033 | |
| } | |
| ], | |
| "logging_steps": 1.0, | |
| "max_steps": 147, | |
| "num_input_tokens_seen": 0, | |
| "num_train_epochs": 1, | |
| "save_steps": 10000, | |
| "total_flos": 0.0, | |
| "train_batch_size": 1, | |
| "trial_name": null, | |
| "trial_params": null | |
| } | |