| { |
| "best_global_step": null, |
| "best_metric": null, |
| "best_model_checkpoint": null, |
| "epoch": 0.99860529986053, |
| "eval_steps": 500, |
| "global_step": 358, |
| "is_hyper_param_search": false, |
| "is_local_process_zero": true, |
| "is_world_process_zero": true, |
| "log_history": [ |
| { |
| "epoch": 0.01394700139470014, |
| "grad_norm": 8.443673133850098, |
| "learning_rate": 3.1067090740782183e-06, |
| "logits/chosen": 13.965856552124023, |
| "logits/rejected": 15.354721069335938, |
| "logps/chosen": -263.2093200683594, |
| "logps/rejected": -286.745361328125, |
| "loss": 0.6931, |
| "rewards/accuracies": 0.3083333373069763, |
| "rewards/chosen": 0.001035079127177596, |
| "rewards/margins": 0.004346395842730999, |
| "rewards/rejected": -0.003311316715553403, |
| "step": 5 |
| }, |
| { |
| "epoch": 0.02789400278940028, |
| "grad_norm": 9.05903434753418, |
| "learning_rate": 6.99009541667599e-06, |
| "logits/chosen": 13.079957962036133, |
| "logits/rejected": 15.2965726852417, |
| "logps/chosen": -246.61495971679688, |
| "logps/rejected": -292.19561767578125, |
| "loss": 0.6512, |
| "rewards/accuracies": 0.6499999761581421, |
| "rewards/chosen": -0.004167428705841303, |
| "rewards/margins": 0.10341107845306396, |
| "rewards/rejected": -0.10757851600646973, |
| "step": 10 |
| }, |
| { |
| "epoch": 0.04184100418410042, |
| "grad_norm": 7.4054036140441895, |
| "learning_rate": 1.0873481759273766e-05, |
| "logits/chosen": 13.588569641113281, |
| "logits/rejected": 14.664776802062988, |
| "logps/chosen": -250.9985809326172, |
| "logps/rejected": -271.63397216796875, |
| "loss": 0.6306, |
| "rewards/accuracies": 0.6750000715255737, |
| "rewards/chosen": -0.07446910440921783, |
| "rewards/margins": 0.2632213234901428, |
| "rewards/rejected": -0.33769041299819946, |
| "step": 15 |
| }, |
| { |
| "epoch": 0.05578800557880056, |
| "grad_norm": 7.973468780517578, |
| "learning_rate": 1.4756868101871536e-05, |
| "logits/chosen": 10.776374816894531, |
| "logits/rejected": 12.654977798461914, |
| "logps/chosen": -219.0859375, |
| "logps/rejected": -279.8912048339844, |
| "loss": 0.5803, |
| "rewards/accuracies": 0.7083333730697632, |
| "rewards/chosen": -0.15387776494026184, |
| "rewards/margins": 0.5580573678016663, |
| "rewards/rejected": -0.7119351625442505, |
| "step": 20 |
| }, |
| { |
| "epoch": 0.0697350069735007, |
| "grad_norm": 8.493500709533691, |
| "learning_rate": 1.864025444446931e-05, |
| "logits/chosen": 11.028432846069336, |
| "logits/rejected": 12.664867401123047, |
| "logps/chosen": -269.8415832519531, |
| "logps/rejected": -324.97613525390625, |
| "loss": 0.6957, |
| "rewards/accuracies": 0.6833333969116211, |
| "rewards/chosen": -0.31670600175857544, |
| "rewards/margins": 0.7376400232315063, |
| "rewards/rejected": -1.0543458461761475, |
| "step": 25 |
| }, |
| { |
| "epoch": 0.08368200836820083, |
| "grad_norm": 6.57462739944458, |
| "learning_rate": 2.2523640787067085e-05, |
| "logits/chosen": 12.154653549194336, |
| "logits/rejected": 13.542457580566406, |
| "logps/chosen": -250.53341674804688, |
| "logps/rejected": -290.4168395996094, |
| "loss": 0.6559, |
| "rewards/accuracies": 0.6750000715255737, |
| "rewards/chosen": -0.11686629056930542, |
| "rewards/margins": 0.5811147093772888, |
| "rewards/rejected": -0.6979809999465942, |
| "step": 30 |
| }, |
| { |
| "epoch": 0.09762900976290098, |
| "grad_norm": 7.757661819458008, |
| "learning_rate": 2.6407027129664858e-05, |
| "logits/chosen": 11.735052108764648, |
| "logits/rejected": 12.871235847473145, |
| "logps/chosen": -196.1747589111328, |
| "logps/rejected": -223.5957794189453, |
| "loss": 0.6182, |
| "rewards/accuracies": 0.6333333253860474, |
| "rewards/chosen": -0.00016996636986732483, |
| "rewards/margins": 0.3299906551837921, |
| "rewards/rejected": -0.33016061782836914, |
| "step": 35 |
| }, |
| { |
| "epoch": 0.11157601115760112, |
| "grad_norm": 9.74497127532959, |
| "learning_rate": 2.7182958819900885e-05, |
| "logits/chosen": 10.667373657226562, |
| "logits/rejected": 12.847865104675293, |
| "logps/chosen": -208.2180633544922, |
| "logps/rejected": -302.39971923828125, |
| "loss": 0.5985, |
| "rewards/accuracies": 0.625, |
| "rewards/chosen": -0.03828881308436394, |
| "rewards/margins": 0.43971744179725647, |
| "rewards/rejected": -0.4780062735080719, |
| "step": 40 |
| }, |
| { |
| "epoch": 0.12552301255230125, |
| "grad_norm": 9.502607345581055, |
| "learning_rate": 2.7179930095042222e-05, |
| "logits/chosen": 11.666415214538574, |
| "logits/rejected": 13.399192810058594, |
| "logps/chosen": -252.95516967773438, |
| "logps/rejected": -297.4845275878906, |
| "loss": 0.6683, |
| "rewards/accuracies": 0.625, |
| "rewards/chosen": -0.2012624740600586, |
| "rewards/margins": 0.4417993426322937, |
| "rewards/rejected": -0.6430618166923523, |
| "step": 45 |
| }, |
| { |
| "epoch": 0.1394700139470014, |
| "grad_norm": 6.734437942504883, |
| "learning_rate": 2.717457231667877e-05, |
| "logits/chosen": 11.400947570800781, |
| "logits/rejected": 12.905207633972168, |
| "logps/chosen": -199.80596923828125, |
| "logps/rejected": -254.4221954345703, |
| "loss": 0.5718, |
| "rewards/accuracies": 0.6416667103767395, |
| "rewards/chosen": -0.022540345788002014, |
| "rewards/margins": 0.5749102830886841, |
| "rewards/rejected": -0.5974506139755249, |
| "step": 50 |
| }, |
| { |
| "epoch": 0.15341701534170155, |
| "grad_norm": 6.349542617797852, |
| "learning_rate": 2.7166886709384802e-05, |
| "logits/chosen": 9.220715522766113, |
| "logits/rejected": 10.65298080444336, |
| "logps/chosen": -187.50601196289062, |
| "logps/rejected": -232.32241821289062, |
| "loss": 0.6379, |
| "rewards/accuracies": 0.6583333015441895, |
| "rewards/chosen": -0.3121258318424225, |
| "rewards/margins": 0.4689728617668152, |
| "rewards/rejected": -0.7810987234115601, |
| "step": 55 |
| }, |
| { |
| "epoch": 0.16736401673640167, |
| "grad_norm": 6.804388046264648, |
| "learning_rate": 2.715687502978336e-05, |
| "logits/chosen": 10.728836059570312, |
| "logits/rejected": 11.548182487487793, |
| "logps/chosen": -220.2592010498047, |
| "logps/rejected": -248.6510009765625, |
| "loss": 0.5624, |
| "rewards/accuracies": 0.6916667222976685, |
| "rewards/chosen": -0.24719354510307312, |
| "rewards/margins": 0.5707910656929016, |
| "rewards/rejected": -0.8179847002029419, |
| "step": 60 |
| }, |
| { |
| "epoch": 0.18131101813110181, |
| "grad_norm": 8.308326721191406, |
| "learning_rate": 2.714453956614478e-05, |
| "logits/chosen": 9.632620811462402, |
| "logits/rejected": 10.766716003417969, |
| "logps/chosen": -234.59494018554688, |
| "logps/rejected": -246.2835235595703, |
| "loss": 0.6495, |
| "rewards/accuracies": 0.6500000357627869, |
| "rewards/chosen": -0.4380224347114563, |
| "rewards/margins": 0.505369246006012, |
| "rewards/rejected": -0.9433916211128235, |
| "step": 65 |
| }, |
| { |
| "epoch": 0.19525801952580196, |
| "grad_norm": 5.469903945922852, |
| "learning_rate": 2.7129883137863668e-05, |
| "logits/chosen": 10.187846183776855, |
| "logits/rejected": 11.750052452087402, |
| "logps/chosen": -201.72952270507812, |
| "logps/rejected": -254.0623779296875, |
| "loss": 0.6407, |
| "rewards/accuracies": 0.6166666746139526, |
| "rewards/chosen": -0.28986990451812744, |
| "rewards/margins": 0.4815855026245117, |
| "rewards/rejected": -0.7714553475379944, |
| "step": 70 |
| }, |
| { |
| "epoch": 0.20920502092050208, |
| "grad_norm": 6.822120189666748, |
| "learning_rate": 2.7112909094814497e-05, |
| "logits/chosen": 11.570058822631836, |
| "logits/rejected": 12.092260360717773, |
| "logps/chosen": -201.8399658203125, |
| "logps/rejected": -255.96774291992188, |
| "loss": 0.5926, |
| "rewards/accuracies": 0.6500000357627869, |
| "rewards/chosen": -0.17586487531661987, |
| "rewards/margins": 0.4662502706050873, |
| "rewards/rejected": -0.6421152353286743, |
| "step": 75 |
| }, |
| { |
| "epoch": 0.22315202231520223, |
| "grad_norm": 6.789912223815918, |
| "learning_rate": 2.7093621316585976e-05, |
| "logits/chosen": 9.428289413452148, |
| "logits/rejected": 12.114514350891113, |
| "logps/chosen": -199.76634216308594, |
| "logps/rejected": -259.3163146972656, |
| "loss": 0.6011, |
| "rewards/accuracies": 0.7083333730697632, |
| "rewards/chosen": -0.12371706962585449, |
| "rewards/margins": 0.6005428433418274, |
| "rewards/rejected": -0.7242598533630371, |
| "step": 80 |
| }, |
| { |
| "epoch": 0.23709902370990238, |
| "grad_norm": 8.106010437011719, |
| "learning_rate": 2.7072024211594312e-05, |
| "logits/chosen": 10.688722610473633, |
| "logits/rejected": 11.625075340270996, |
| "logps/chosen": -221.6556854248047, |
| "logps/rejected": -274.34808349609375, |
| "loss": 0.6676, |
| "rewards/accuracies": 0.6499999761581421, |
| "rewards/chosen": -0.237966850399971, |
| "rewards/margins": 0.4688500761985779, |
| "rewards/rejected": -0.7068168520927429, |
| "step": 85 |
| }, |
| { |
| "epoch": 0.2510460251046025, |
| "grad_norm": 7.263269901275635, |
| "learning_rate": 2.7048122716075636e-05, |
| "logits/chosen": 12.357452392578125, |
| "logits/rejected": 14.314582824707031, |
| "logps/chosen": -224.59451293945312, |
| "logps/rejected": -291.54840087890625, |
| "loss": 0.6033, |
| "rewards/accuracies": 0.6666667461395264, |
| "rewards/chosen": -0.07308916002511978, |
| "rewards/margins": 0.504932165145874, |
| "rewards/rejected": -0.5780213475227356, |
| "step": 90 |
| }, |
| { |
| "epoch": 0.2649930264993027, |
| "grad_norm": 8.925468444824219, |
| "learning_rate": 2.7021922292957776e-05, |
| "logits/chosen": 12.812413215637207, |
| "logits/rejected": 13.796060562133789, |
| "logps/chosen": -230.6768798828125, |
| "logps/rejected": -276.98828125, |
| "loss": 0.6054, |
| "rewards/accuracies": 0.6916667222976685, |
| "rewards/chosen": -0.03974637761712074, |
| "rewards/margins": 0.5660965442657471, |
| "rewards/rejected": -0.6058429479598999, |
| "step": 95 |
| }, |
| { |
| "epoch": 0.2789400278940028, |
| "grad_norm": 5.422610282897949, |
| "learning_rate": 2.6993428930611634e-05, |
| "logits/chosen": 12.078475952148438, |
| "logits/rejected": 14.003347396850586, |
| "logps/chosen": -229.6229705810547, |
| "logps/rejected": -276.1091613769531, |
| "loss": 0.5103, |
| "rewards/accuracies": 0.7916666269302368, |
| "rewards/chosen": 0.10478214919567108, |
| "rewards/margins": 0.8226801156997681, |
| "rewards/rejected": -0.7178980112075806, |
| "step": 100 |
| }, |
| { |
| "epoch": 0.2928870292887029, |
| "grad_norm": 7.437990188598633, |
| "learning_rate": 2.69626491414825e-05, |
| "logits/chosen": 11.215384483337402, |
| "logits/rejected": 14.319795608520508, |
| "logps/chosen": -253.2958221435547, |
| "logps/rejected": -353.27728271484375, |
| "loss": 0.4647, |
| "rewards/accuracies": 0.800000011920929, |
| "rewards/chosen": -0.20068514347076416, |
| "rewards/margins": 1.1105215549468994, |
| "rewards/rejected": -1.3112064599990845, |
| "step": 105 |
| }, |
| { |
| "epoch": 0.3068340306834031, |
| "grad_norm": 4.438156604766846, |
| "learning_rate": 2.6929589960601567e-05, |
| "logits/chosen": 10.04852294921875, |
| "logits/rejected": 11.834890365600586, |
| "logps/chosen": -177.2984619140625, |
| "logps/rejected": -235.3513946533203, |
| "loss": 0.5941, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": -0.14255599677562714, |
| "rewards/margins": 1.0380289554595947, |
| "rewards/rejected": -1.1805849075317383, |
| "step": 110 |
| }, |
| { |
| "epoch": 0.3207810320781032, |
| "grad_norm": 5.81158971786499, |
| "learning_rate": 2.689425894397799e-05, |
| "logits/chosen": 11.626302719116211, |
| "logits/rejected": 13.641179084777832, |
| "logps/chosen": -258.5274353027344, |
| "logps/rejected": -290.1232604980469, |
| "loss": 0.7585, |
| "rewards/accuracies": 0.6916667222976685, |
| "rewards/chosen": -0.3435746729373932, |
| "rewards/margins": 0.7051381468772888, |
| "rewards/rejected": -1.0487128496170044, |
| "step": 115 |
| }, |
| { |
| "epoch": 0.33472803347280333, |
| "grad_norm": 6.80866003036499, |
| "learning_rate": 2.685666416687189e-05, |
| "logits/chosen": 12.343037605285645, |
| "logits/rejected": 14.431178092956543, |
| "logps/chosen": -240.32119750976562, |
| "logps/rejected": -306.0838623046875, |
| "loss": 0.595, |
| "rewards/accuracies": 0.7166666984558105, |
| "rewards/chosen": 0.007438424974679947, |
| "rewards/margins": 0.7371615171432495, |
| "rewards/rejected": -0.7297230362892151, |
| "step": 120 |
| }, |
| { |
| "epoch": 0.3486750348675035, |
| "grad_norm": 5.887704372406006, |
| "learning_rate": 2.6816814221948682e-05, |
| "logits/chosen": 12.360528945922852, |
| "logits/rejected": 13.354936599731445, |
| "logps/chosen": -229.65194702148438, |
| "logps/rejected": -286.4953918457031, |
| "loss": 0.5958, |
| "rewards/accuracies": 0.7250000238418579, |
| "rewards/chosen": 0.04780165106058121, |
| "rewards/margins": 0.5758650898933411, |
| "rewards/rejected": -0.5280634164810181, |
| "step": 125 |
| }, |
| { |
| "epoch": 0.36262203626220363, |
| "grad_norm": 6.528600215911865, |
| "learning_rate": 2.6774718217315124e-05, |
| "logits/chosen": 12.22896671295166, |
| "logits/rejected": 14.30839729309082, |
| "logps/chosen": -258.0196838378906, |
| "logps/rejected": -312.25018310546875, |
| "loss": 0.5731, |
| "rewards/accuracies": 0.7166666984558105, |
| "rewards/chosen": -0.1742248833179474, |
| "rewards/margins": 0.7956671118736267, |
| "rewards/rejected": -0.9698920249938965, |
| "step": 130 |
| }, |
| { |
| "epoch": 0.37656903765690375, |
| "grad_norm": 4.8263702392578125, |
| "learning_rate": 2.6730385774437575e-05, |
| "logits/chosen": 11.001718521118164, |
| "logits/rejected": 12.369864463806152, |
| "logps/chosen": -186.54306030273438, |
| "logps/rejected": -229.1658935546875, |
| "loss": 0.5747, |
| "rewards/accuracies": 0.7333333492279053, |
| "rewards/chosen": -0.10876262187957764, |
| "rewards/margins": 0.783820390701294, |
| "rewards/rejected": -0.8925830721855164, |
| "step": 135 |
| }, |
| { |
| "epoch": 0.3905160390516039, |
| "grad_norm": 6.88525915145874, |
| "learning_rate": 2.668382702594289e-05, |
| "logits/chosen": 12.428079605102539, |
| "logits/rejected": 13.376548767089844, |
| "logps/chosen": -244.35336303710938, |
| "logps/rejected": -314.2310485839844, |
| "loss": 0.5727, |
| "rewards/accuracies": 0.675000011920929, |
| "rewards/chosen": -0.19110675156116486, |
| "rewards/margins": 0.8208184242248535, |
| "rewards/rejected": -1.011925220489502, |
| "step": 140 |
| }, |
| { |
| "epoch": 0.40446304044630405, |
| "grad_norm": 5.504042148590088, |
| "learning_rate": 2.663505261330254e-05, |
| "logits/chosen": 11.734379768371582, |
| "logits/rejected": 13.473363876342773, |
| "logps/chosen": -204.04251098632812, |
| "logps/rejected": -258.86737060546875, |
| "loss": 0.6128, |
| "rewards/accuracies": 0.6666666865348816, |
| "rewards/chosen": -0.17432288825511932, |
| "rewards/margins": 0.4802930951118469, |
| "rewards/rejected": -0.6546159982681274, |
| "step": 145 |
| }, |
| { |
| "epoch": 0.41841004184100417, |
| "grad_norm": 6.70412540435791, |
| "learning_rate": 2.6584073684400373e-05, |
| "logits/chosen": 13.045954704284668, |
| "logits/rejected": 13.6720552444458, |
| "logps/chosen": -220.8832550048828, |
| "logps/rejected": -269.233642578125, |
| "loss": 0.6032, |
| "rewards/accuracies": 0.675000011920929, |
| "rewards/chosen": -0.04318929463624954, |
| "rewards/margins": 0.618157684803009, |
| "rewards/rejected": -0.6613470315933228, |
| "step": 150 |
| }, |
| { |
| "epoch": 0.43235704323570434, |
| "grad_norm": 12.566320419311523, |
| "learning_rate": 2.653090189098466e-05, |
| "logits/chosen": 11.808868408203125, |
| "logits/rejected": 13.49694538116455, |
| "logps/chosen": -248.18212890625, |
| "logps/rejected": -286.3559265136719, |
| "loss": 0.6096, |
| "rewards/accuracies": 0.6499999761581421, |
| "rewards/chosen": -0.15974543988704681, |
| "rewards/margins": 0.6279758214950562, |
| "rewards/rejected": -0.7877213954925537, |
| "step": 155 |
| }, |
| { |
| "epoch": 0.44630404463040446, |
| "grad_norm": 6.7717509269714355, |
| "learning_rate": 2.647554938600497e-05, |
| "logits/chosen": 11.906153678894043, |
| "logits/rejected": 13.252202033996582, |
| "logps/chosen": -230.92593383789062, |
| "logps/rejected": -248.78488159179688, |
| "loss": 0.6474, |
| "rewards/accuracies": 0.6166666746139526, |
| "rewards/chosen": -0.20924147963523865, |
| "rewards/margins": 0.49998435378074646, |
| "rewards/rejected": -0.7092257738113403, |
| "step": 160 |
| }, |
| { |
| "epoch": 0.4602510460251046, |
| "grad_norm": 7.113363742828369, |
| "learning_rate": 2.6418028820834483e-05, |
| "logits/chosen": 11.409849166870117, |
| "logits/rejected": 13.500089645385742, |
| "logps/chosen": -242.9384765625, |
| "logps/rejected": -325.8098449707031, |
| "loss": 0.5703, |
| "rewards/accuracies": 0.7000000476837158, |
| "rewards/chosen": -0.4405299127101898, |
| "rewards/margins": 0.8166629076004028, |
| "rewards/rejected": -1.257192850112915, |
| "step": 165 |
| }, |
| { |
| "epoch": 0.47419804741980476, |
| "grad_norm": 4.8473429679870605, |
| "learning_rate": 2.6358353342378405e-05, |
| "logits/chosen": 8.604534149169922, |
| "logits/rejected": 11.609006881713867, |
| "logps/chosen": -185.0061492919922, |
| "logps/rejected": -256.83465576171875, |
| "loss": 0.4518, |
| "rewards/accuracies": 0.8166667222976685, |
| "rewards/chosen": -0.5300418734550476, |
| "rewards/margins": 1.1527934074401855, |
| "rewards/rejected": -1.682835340499878, |
| "step": 170 |
| }, |
| { |
| "epoch": 0.4881450488145049, |
| "grad_norm": 6.59335994720459, |
| "learning_rate": 2.6296536590069104e-05, |
| "logits/chosen": 9.987462997436523, |
| "logits/rejected": 11.572959899902344, |
| "logps/chosen": -253.36050415039062, |
| "logps/rejected": -306.0450439453125, |
| "loss": 0.7439, |
| "rewards/accuracies": 0.6666666269302368, |
| "rewards/chosen": -1.0800180435180664, |
| "rewards/margins": 0.8873661160469055, |
| "rewards/rejected": -1.9673839807510376, |
| "step": 175 |
| }, |
| { |
| "epoch": 0.502092050209205, |
| "grad_norm": 7.2572808265686035, |
| "learning_rate": 2.6232592692748676e-05, |
| "logits/chosen": 9.321784019470215, |
| "logits/rejected": 10.399984359741211, |
| "logps/chosen": -234.3979949951172, |
| "logps/rejected": -294.28961181640625, |
| "loss": 0.489, |
| "rewards/accuracies": 0.7916667461395264, |
| "rewards/chosen": -0.8451215624809265, |
| "rewards/margins": 1.1555068492889404, |
| "rewards/rejected": -2.0006284713745117, |
| "step": 180 |
| }, |
| { |
| "epoch": 0.5160390516039052, |
| "grad_norm": 6.049355506896973, |
| "learning_rate": 2.6166536265439664e-05, |
| "logits/chosen": 11.809592247009277, |
| "logits/rejected": 13.855669021606445, |
| "logps/chosen": -235.9927215576172, |
| "logps/rejected": -315.3490295410156, |
| "loss": 0.5868, |
| "rewards/accuracies": 0.6583333015441895, |
| "rewards/chosen": -0.6016494631767273, |
| "rewards/margins": 0.7112706899642944, |
| "rewards/rejected": -1.3129202127456665, |
| "step": 185 |
| }, |
| { |
| "epoch": 0.5299860529986054, |
| "grad_norm": 11.038729667663574, |
| "learning_rate": 2.609838240600464e-05, |
| "logits/chosen": 11.31627082824707, |
| "logits/rejected": 12.770919799804688, |
| "logps/chosen": -221.79501342773438, |
| "logps/rejected": -262.8851013183594, |
| "loss": 0.5937, |
| "rewards/accuracies": 0.6916667222976685, |
| "rewards/chosen": -0.5558231472969055, |
| "rewards/margins": 0.6753066778182983, |
| "rewards/rejected": -1.2311298847198486, |
| "step": 190 |
| }, |
| { |
| "epoch": 0.5439330543933054, |
| "grad_norm": 4.267746925354004, |
| "learning_rate": 2.602814669169543e-05, |
| "logits/chosen": 11.312234878540039, |
| "logits/rejected": 13.296246528625488, |
| "logps/chosen": -221.83663940429688, |
| "logps/rejected": -302.32623291015625, |
| "loss": 0.5809, |
| "rewards/accuracies": 0.7166666984558105, |
| "rewards/chosen": -0.5700381994247437, |
| "rewards/margins": 0.8077206611633301, |
| "rewards/rejected": -1.3777590990066528, |
| "step": 195 |
| }, |
| { |
| "epoch": 0.5578800557880056, |
| "grad_norm": 5.395613193511963, |
| "learning_rate": 2.5955845175592813e-05, |
| "logits/chosen": 11.410247802734375, |
| "logits/rejected": 13.515039443969727, |
| "logps/chosen": -232.266845703125, |
| "logps/rejected": -293.22467041015625, |
| "loss": 0.5553, |
| "rewards/accuracies": 0.7250000238418579, |
| "rewards/chosen": -0.5373277068138123, |
| "rewards/margins": 0.8851755857467651, |
| "rewards/rejected": -1.4225032329559326, |
| "step": 200 |
| }, |
| { |
| "epoch": 0.5718270571827058, |
| "grad_norm": 5.884355545043945, |
| "learning_rate": 2.5881494382937354e-05, |
| "logits/chosen": 11.658515930175781, |
| "logits/rejected": 13.375651359558105, |
| "logps/chosen": -212.52932739257812, |
| "logps/rejected": -286.71783447265625, |
| "loss": 0.5126, |
| "rewards/accuracies": 0.7250000238418579, |
| "rewards/chosen": -0.6171377897262573, |
| "rewards/margins": 1.1004387140274048, |
| "rewards/rejected": -1.7175763845443726, |
| "step": 205 |
| }, |
| { |
| "epoch": 0.5857740585774058, |
| "grad_norm": 8.411831855773926, |
| "learning_rate": 2.5805111307352483e-05, |
| "logits/chosen": 11.734652519226074, |
| "logits/rejected": 12.831028938293457, |
| "logps/chosen": -247.45736694335938, |
| "logps/rejected": -297.03485107421875, |
| "loss": 0.5907, |
| "rewards/accuracies": 0.7333332896232605, |
| "rewards/chosen": -0.6340610384941101, |
| "rewards/margins": 1.0966707468032837, |
| "rewards/rejected": -1.7307319641113281, |
| "step": 210 |
| }, |
| { |
| "epoch": 0.599721059972106, |
| "grad_norm": 6.429418087005615, |
| "learning_rate": 2.5726713406960365e-05, |
| "logits/chosen": 9.269506454467773, |
| "logits/rejected": 12.285856246948242, |
| "logps/chosen": -187.17591857910156, |
| "logps/rejected": -269.7003479003906, |
| "loss": 0.5349, |
| "rewards/accuracies": 0.7666667699813843, |
| "rewards/chosen": -0.5631422400474548, |
| "rewards/margins": 1.0805346965789795, |
| "rewards/rejected": -1.6436771154403687, |
| "step": 215 |
| }, |
| { |
| "epoch": 0.6136680613668062, |
| "grad_norm": 8.299524307250977, |
| "learning_rate": 2.5646318600391693e-05, |
| "logits/chosen": 10.429685592651367, |
| "logits/rejected": 11.87198543548584, |
| "logps/chosen": -234.75802612304688, |
| "logps/rejected": -284.33642578125, |
| "loss": 0.5775, |
| "rewards/accuracies": 0.6833333373069763, |
| "rewards/chosen": -0.5350970029830933, |
| "rewards/margins": 1.0643174648284912, |
| "rewards/rejected": -1.599414348602295, |
| "step": 220 |
| }, |
| { |
| "epoch": 0.6276150627615062, |
| "grad_norm": 3.9937915802001953, |
| "learning_rate": 2.556394526269021e-05, |
| "logits/chosen": 11.182074546813965, |
| "logits/rejected": 12.9561767578125, |
| "logps/chosen": -209.6283721923828, |
| "logps/rejected": -261.5816345214844, |
| "loss": 0.7124, |
| "rewards/accuracies": 0.625, |
| "rewards/chosen": -0.651893675327301, |
| "rewards/margins": 0.631694495677948, |
| "rewards/rejected": -1.283588171005249, |
| "step": 225 |
| }, |
| { |
| "epoch": 0.6415620641562064, |
| "grad_norm": 8.43625259399414, |
| "learning_rate": 2.5479612221112888e-05, |
| "logits/chosen": 11.575922966003418, |
| "logits/rejected": 12.958845138549805, |
| "logps/chosen": -241.3083953857422, |
| "logps/rejected": -291.8955993652344, |
| "loss": 0.6766, |
| "rewards/accuracies": 0.6666666269302368, |
| "rewards/chosen": -0.6825530529022217, |
| "rewards/margins": 0.6574732661247253, |
| "rewards/rejected": -1.3400263786315918, |
| "step": 230 |
| }, |
| { |
| "epoch": 0.6555090655509066, |
| "grad_norm": 4.0093817710876465, |
| "learning_rate": 2.5393338750826796e-05, |
| "logits/chosen": 12.776809692382812, |
| "logits/rejected": 14.406654357910156, |
| "logps/chosen": -229.2129669189453, |
| "logps/rejected": -300.2899475097656, |
| "loss": 0.5466, |
| "rewards/accuracies": 0.7000000476837158, |
| "rewards/chosen": -0.5617952346801758, |
| "rewards/margins": 0.752461850643158, |
| "rewards/rejected": -1.314257264137268, |
| "step": 235 |
| }, |
| { |
| "epoch": 0.6694560669456067, |
| "grad_norm": 6.296603679656982, |
| "learning_rate": 2.5305144570503554e-05, |
| "logits/chosen": 12.071104049682617, |
| "logits/rejected": 14.155171394348145, |
| "logps/chosen": -226.9972381591797, |
| "logps/rejected": -299.86993408203125, |
| "loss": 0.5546, |
| "rewards/accuracies": 0.7166667580604553, |
| "rewards/chosen": -0.5309011936187744, |
| "rewards/margins": 0.8765324354171753, |
| "rewards/rejected": -1.4074336290359497, |
| "step": 240 |
| }, |
| { |
| "epoch": 0.6834030683403068, |
| "grad_norm": 6.713784694671631, |
| "learning_rate": 2.5215049837812413e-05, |
| "logits/chosen": 11.046672821044922, |
| "logits/rejected": 13.329177856445312, |
| "logps/chosen": -222.3082275390625, |
| "logps/rejected": -302.4698181152344, |
| "loss": 0.4944, |
| "rewards/accuracies": 0.7583333253860474, |
| "rewards/chosen": -0.6500851511955261, |
| "rewards/margins": 1.216412901878357, |
| "rewards/rejected": -1.8664979934692383, |
| "step": 245 |
| }, |
| { |
| "epoch": 0.697350069735007, |
| "grad_norm": 3.64483904838562, |
| "learning_rate": 2.5123075144813044e-05, |
| "logits/chosen": 9.95425796508789, |
| "logits/rejected": 12.83178997039795, |
| "logps/chosen": -245.83987426757812, |
| "logps/rejected": -347.21807861328125, |
| "loss": 0.4497, |
| "rewards/accuracies": 0.8083333969116211, |
| "rewards/chosen": -0.7453610897064209, |
| "rewards/margins": 1.5202277898788452, |
| "rewards/rejected": -2.2655892372131348, |
| "step": 250 |
| }, |
| { |
| "epoch": 0.7112970711297071, |
| "grad_norm": 6.323943138122559, |
| "learning_rate": 2.5029241513248992e-05, |
| "logits/chosen": 10.718851089477539, |
| "logits/rejected": 11.787945747375488, |
| "logps/chosen": -243.6724853515625, |
| "logps/rejected": -275.3341064453125, |
| "loss": 0.7227, |
| "rewards/accuracies": 0.6499999761581421, |
| "rewards/chosen": -0.8060539364814758, |
| "rewards/margins": 0.8095922470092773, |
| "rewards/rejected": -1.6156460046768188, |
| "step": 255 |
| }, |
| { |
| "epoch": 0.7252440725244073, |
| "grad_norm": 7.24354887008667, |
| "learning_rate": 2.4933570389742975e-05, |
| "logits/chosen": 10.517350196838379, |
| "logits/rejected": 12.290593147277832, |
| "logps/chosen": -217.3165740966797, |
| "logps/rejected": -277.9285888671875, |
| "loss": 0.5759, |
| "rewards/accuracies": 0.7333332896232605, |
| "rewards/chosen": -0.4403415620326996, |
| "rewards/margins": 0.91583251953125, |
| "rewards/rejected": -1.3561739921569824, |
| "step": 260 |
| }, |
| { |
| "epoch": 0.7391910739191074, |
| "grad_norm": 5.851767063140869, |
| "learning_rate": 2.4836083640895016e-05, |
| "logits/chosen": 11.360766410827637, |
| "logits/rejected": 13.0468111038208, |
| "logps/chosen": -253.7987518310547, |
| "logps/rejected": -321.97967529296875, |
| "loss": 0.552, |
| "rewards/accuracies": 0.73333340883255, |
| "rewards/chosen": -0.37671542167663574, |
| "rewards/margins": 1.0203909873962402, |
| "rewards/rejected": -1.3971065282821655, |
| "step": 265 |
| }, |
| { |
| "epoch": 0.7531380753138075, |
| "grad_norm": 5.513597011566162, |
| "learning_rate": 2.473680354828461e-05, |
| "logits/chosen": 10.999284744262695, |
| "logits/rejected": 12.660871505737305, |
| "logps/chosen": -218.94735717773438, |
| "logps/rejected": -259.43133544921875, |
| "loss": 0.5662, |
| "rewards/accuracies": 0.7083333730697632, |
| "rewards/chosen": -0.4751812517642975, |
| "rewards/margins": 0.7894707322120667, |
| "rewards/rejected": -1.264651894569397, |
| "step": 270 |
| }, |
| { |
| "epoch": 0.7670850767085077, |
| "grad_norm": 5.750617504119873, |
| "learning_rate": 2.4635752803378063e-05, |
| "logits/chosen": 11.961132049560547, |
| "logits/rejected": 14.314129829406738, |
| "logps/chosen": -221.6836700439453, |
| "logps/rejected": -307.97467041015625, |
| "loss": 0.5598, |
| "rewards/accuracies": 0.7166666984558105, |
| "rewards/chosen": -0.42271748185157776, |
| "rewards/margins": 0.7973464727401733, |
| "rewards/rejected": -1.2200638055801392, |
| "step": 275 |
| }, |
| { |
| "epoch": 0.7810320781032078, |
| "grad_norm": 6.243062496185303, |
| "learning_rate": 2.453295450234211e-05, |
| "logits/chosen": 11.414163589477539, |
| "logits/rejected": 13.276272773742676, |
| "logps/chosen": -223.52529907226562, |
| "logps/rejected": -324.9430236816406, |
| "loss": 0.516, |
| "rewards/accuracies": 0.7833333015441895, |
| "rewards/chosen": -0.5223814249038696, |
| "rewards/margins": 1.0250855684280396, |
| "rewards/rejected": -1.5474669933319092, |
| "step": 280 |
| }, |
| { |
| "epoch": 0.7949790794979079, |
| "grad_norm": 7.279259204864502, |
| "learning_rate": 2.442843214076507e-05, |
| "logits/chosen": 11.585288047790527, |
| "logits/rejected": 12.664071083068848, |
| "logps/chosen": -244.99154663085938, |
| "logps/rejected": -254.84158325195312, |
| "loss": 0.5448, |
| "rewards/accuracies": 0.6916667222976685, |
| "rewards/chosen": -0.7385099530220032, |
| "rewards/margins": 0.9241247177124023, |
| "rewards/rejected": -1.6626346111297607, |
| "step": 285 |
| }, |
| { |
| "epoch": 0.8089260808926081, |
| "grad_norm": 4.353116512298584, |
| "learning_rate": 2.4322209608286686e-05, |
| "logits/chosen": 8.986726760864258, |
| "logits/rejected": 12.175418853759766, |
| "logps/chosen": -206.93594360351562, |
| "logps/rejected": -292.4324645996094, |
| "loss": 0.5075, |
| "rewards/accuracies": 0.7166666984558105, |
| "rewards/chosen": -0.7221769094467163, |
| "rewards/margins": 1.2281521558761597, |
| "rewards/rejected": -1.9503291845321655, |
| "step": 290 |
| }, |
| { |
| "epoch": 0.8228730822873083, |
| "grad_norm": 8.33502197265625, |
| "learning_rate": 2.421431118313789e-05, |
| "logits/chosen": 10.475484848022461, |
| "logits/rejected": 11.844476699829102, |
| "logps/chosen": -240.02206420898438, |
| "logps/rejected": -279.66064453125, |
| "loss": 0.7693, |
| "rewards/accuracies": 0.7250000238418579, |
| "rewards/chosen": -1.0663460493087769, |
| "rewards/margins": 0.940434455871582, |
| "rewards/rejected": -2.0067803859710693, |
| "step": 295 |
| }, |
| { |
| "epoch": 0.8368200836820083, |
| "grad_norm": 7.870750904083252, |
| "learning_rate": 2.41047615265918e-05, |
| "logits/chosen": 11.179079055786133, |
| "logits/rejected": 13.722526550292969, |
| "logps/chosen": -243.4619140625, |
| "logps/rejected": -338.6166687011719, |
| "loss": 0.5564, |
| "rewards/accuracies": 0.7333333492279053, |
| "rewards/chosen": -0.5845211744308472, |
| "rewards/margins": 1.252018690109253, |
| "rewards/rejected": -1.8365398645401, |
| "step": 300 |
| }, |
| { |
| "epoch": 0.8507670850767085, |
| "grad_norm": 5.815814018249512, |
| "learning_rate": 2.3993585677327107e-05, |
| "logits/chosen": 13.206197738647461, |
| "logits/rejected": 13.943652153015137, |
| "logps/chosen": -276.7415466308594, |
| "logps/rejected": -329.16949462890625, |
| "loss": 0.5341, |
| "rewards/accuracies": 0.7250000238418579, |
| "rewards/chosen": -0.4414283335208893, |
| "rewards/margins": 0.9674088358879089, |
| "rewards/rejected": -1.408837080001831, |
| "step": 305 |
| }, |
| { |
| "epoch": 0.8647140864714087, |
| "grad_norm": 8.953941345214844, |
| "learning_rate": 2.3880809045705262e-05, |
| "logits/chosen": 12.374226570129395, |
| "logits/rejected": 13.989924430847168, |
| "logps/chosen": -244.1103515625, |
| "logps/rejected": -294.4126281738281, |
| "loss": 0.5197, |
| "rewards/accuracies": 0.7416666746139526, |
| "rewards/chosen": -0.3111744523048401, |
| "rewards/margins": 1.076432228088379, |
| "rewards/rejected": -1.3876066207885742, |
| "step": 310 |
| }, |
| { |
| "epoch": 0.8786610878661087, |
| "grad_norm": 5.921610355377197, |
| "learning_rate": 2.3766457407962654e-05, |
| "logits/chosen": 11.312451362609863, |
| "logits/rejected": 13.527206420898438, |
| "logps/chosen": -234.7072296142578, |
| "logps/rejected": -290.8281555175781, |
| "loss": 0.6344, |
| "rewards/accuracies": 0.6583333611488342, |
| "rewards/chosen": -0.5945696234703064, |
| "rewards/margins": 0.7386828660964966, |
| "rewards/rejected": -1.3332524299621582, |
| "step": 315 |
| }, |
| { |
| "epoch": 0.8926080892608089, |
| "grad_norm": 7.662965774536133, |
| "learning_rate": 2.3650556900319204e-05, |
| "logits/chosen": 11.144214630126953, |
| "logits/rejected": 13.931811332702637, |
| "logps/chosen": -198.17422485351562, |
| "logps/rejected": -286.37152099609375, |
| "loss": 0.5295, |
| "rewards/accuracies": 0.7166666388511658, |
| "rewards/chosen": -0.3581700325012207, |
| "rewards/margins": 1.0556398630142212, |
| "rewards/rejected": -1.4138100147247314, |
| "step": 320 |
| }, |
| { |
| "epoch": 0.9065550906555091, |
| "grad_norm": 6.2693071365356445, |
| "learning_rate": 2.3533134013004666e-05, |
| "logits/chosen": 11.203069686889648, |
| "logits/rejected": 11.721672058105469, |
| "logps/chosen": -192.05380249023438, |
| "logps/rejected": -221.40042114257812, |
| "loss": 0.6338, |
| "rewards/accuracies": 0.7083333730697632, |
| "rewards/chosen": -0.56825852394104, |
| "rewards/margins": 0.6851181983947754, |
| "rewards/rejected": -1.253376841545105, |
| "step": 325 |
| }, |
| { |
| "epoch": 0.9205020920502092, |
| "grad_norm": 4.502702236175537, |
| "learning_rate": 2.341421558420403e-05, |
| "logits/chosen": 10.94641399383545, |
| "logits/rejected": 13.1921968460083, |
| "logps/chosen": -214.18911743164062, |
| "logps/rejected": -298.7375183105469, |
| "loss": 0.4812, |
| "rewards/accuracies": 0.7416666746139526, |
| "rewards/chosen": -0.6049180030822754, |
| "rewards/margins": 1.2558748722076416, |
| "rewards/rejected": -1.860793113708496, |
| "step": 330 |
| }, |
| { |
| "epoch": 0.9344490934449093, |
| "grad_norm": 6.939857482910156, |
| "learning_rate": 2.3293828793923365e-05, |
| "logits/chosen": 11.944158554077148, |
| "logits/rejected": 13.312121391296387, |
| "logps/chosen": -258.62774658203125, |
| "logps/rejected": -314.3907165527344, |
| "loss": 0.619, |
| "rewards/accuracies": 0.6583333015441895, |
| "rewards/chosen": -0.5666269063949585, |
| "rewards/margins": 1.0656505823135376, |
| "rewards/rejected": -1.632277488708496, |
| "step": 335 |
| }, |
| { |
| "epoch": 0.9483960948396095, |
| "grad_norm": 4.783708572387695, |
| "learning_rate": 2.3172001157777566e-05, |
| "logits/chosen": 11.259064674377441, |
| "logits/rejected": 12.700227737426758, |
| "logps/chosen": -222.77395629882812, |
| "logps/rejected": -291.24884033203125, |
| "loss": 0.6497, |
| "rewards/accuracies": 0.658333420753479, |
| "rewards/chosen": -0.5113255977630615, |
| "rewards/margins": 0.866672158241272, |
| "rewards/rejected": -1.377997636795044, |
| "step": 340 |
| }, |
| { |
| "epoch": 0.9623430962343096, |
| "grad_norm": 4.230240821838379, |
| "learning_rate": 2.3048760520701374e-05, |
| "logits/chosen": 11.741998672485352, |
| "logits/rejected": 13.120841979980469, |
| "logps/chosen": -246.1442108154297, |
| "logps/rejected": -311.65972900390625, |
| "loss": 0.485, |
| "rewards/accuracies": 0.7416666746139526, |
| "rewards/chosen": -0.4931299090385437, |
| "rewards/margins": 1.0040347576141357, |
| "rewards/rejected": -1.4971646070480347, |
| "step": 345 |
| }, |
| { |
| "epoch": 0.9762900976290098, |
| "grad_norm": 4.901912212371826, |
| "learning_rate": 2.2924135050585152e-05, |
| "logits/chosen": 11.310202598571777, |
| "logits/rejected": 13.040916442871094, |
| "logps/chosen": -223.5089111328125, |
| "logps/rejected": -253.26953125, |
| "loss": 0.5638, |
| "rewards/accuracies": 0.7416666746139526, |
| "rewards/chosen": -0.6479132771492004, |
| "rewards/margins": 0.7726107835769653, |
| "rewards/rejected": -1.4205242395401, |
| "step": 350 |
| }, |
| { |
| "epoch": 0.9902370990237099, |
| "grad_norm": 4.653822898864746, |
| "learning_rate": 2.2798153231836813e-05, |
| "logits/chosen": 12.201348304748535, |
| "logits/rejected": 13.601194381713867, |
| "logps/chosen": -268.86328125, |
| "logps/rejected": -299.11578369140625, |
| "loss": 0.5367, |
| "rewards/accuracies": 0.7416666746139526, |
| "rewards/chosen": -0.5867849588394165, |
| "rewards/margins": 1.1348934173583984, |
| "rewards/rejected": -1.721678376197815, |
| "step": 355 |
| }, |
| { |
| "epoch": 0.99860529986053, |
| "eval_logits/chosen": 11.91541862487793, |
| "eval_logits/rejected": 12.808130264282227, |
| "eval_logps/chosen": -229.4397430419922, |
| "eval_logps/rejected": -286.59234619140625, |
| "eval_loss": 0.6298339366912842, |
| "eval_rewards/accuracies": 0.7099999785423279, |
| "eval_rewards/chosen": -0.7923385500907898, |
| "eval_rewards/margins": 0.9493054151535034, |
| "eval_rewards/rejected": -1.7416436672210693, |
| "eval_runtime": 24.5973, |
| "eval_samples_per_second": 8.131, |
| "eval_steps_per_second": 8.131, |
| "step": 358 |
| } |
| ], |
| "logging_steps": 5, |
| "max_steps": 1074, |
| "num_input_tokens_seen": 0, |
| "num_train_epochs": 3, |
| "save_steps": 500, |
| "stateful_callbacks": { |
| "TrainerControl": { |
| "args": { |
| "should_epoch_stop": false, |
| "should_evaluate": false, |
| "should_log": false, |
| "should_save": true, |
| "should_training_stop": false |
| }, |
| "attributes": {} |
| } |
| }, |
| "total_flos": 0.0, |
| "train_batch_size": 12, |
| "trial_name": null, |
| "trial_params": null |
| } |
|
|
