{ "best_metric": null, "best_model_checkpoint": null, "epoch": 1.0, "eval_steps": 100, "global_step": 391, "is_hyper_param_search": false, "is_local_process_zero": true, "is_world_process_zero": true, "log_history": [ { "epoch": 0.0, "grad_norm": 421.7748930851862, "learning_rate": 2.5e-09, "logits/chosen": -4.623842239379883, "logits/rejected": -4.85917854309082, "logps/chosen": -239.31422424316406, "logps/rejected": -207.56365966796875, "loss": 0.693, "rewards/accuracies": 0.0, "rewards/chosen": 0.0, "rewards/margins": 0.0, "rewards/rejected": 0.0, "step": 1 }, { "epoch": 0.03, "grad_norm": 395.00460150231794, "learning_rate": 2.5e-08, "logits/chosen": -4.33427619934082, "logits/rejected": -4.64423942565918, "logps/chosen": -265.1573791503906, "logps/rejected": -215.7504425048828, "loss": 0.693, "rewards/accuracies": 0.4444444477558136, "rewards/chosen": -0.0050499276258051395, "rewards/margins": -0.0008213710389100015, "rewards/rejected": -0.004228556528687477, "step": 10 }, { "epoch": 0.05, "grad_norm": 398.86324560087934, "learning_rate": 5e-08, "logits/chosen": -4.5085954666137695, "logits/rejected": -4.7428812980651855, "logps/chosen": -267.77325439453125, "logps/rejected": -216.7412567138672, "loss": 0.6821, "rewards/accuracies": 0.643750011920929, "rewards/chosen": 0.016667194664478302, "rewards/margins": 0.029861312359571457, "rewards/rejected": -0.01319411676377058, "step": 20 }, { "epoch": 0.08, "grad_norm": 388.1898348935921, "learning_rate": 7.5e-08, "logits/chosen": -4.594970703125, "logits/rejected": -4.776912689208984, "logps/chosen": -257.5873107910156, "logps/rejected": -215.4453582763672, "loss": 0.6195, "rewards/accuracies": 0.7875000238418579, "rewards/chosen": 0.09455005824565887, "rewards/margins": 0.18580853939056396, "rewards/rejected": -0.09125850349664688, "step": 30 }, { "epoch": 0.1, "grad_norm": 267.0875315068977, "learning_rate": 1e-07, "logits/chosen": -4.637397766113281, "logits/rejected": -4.731802940368652, "logps/chosen": -249.911865234375, "logps/rejected": -222.6023712158203, "loss": 0.4813, "rewards/accuracies": 0.8374999761581421, "rewards/chosen": 0.36873525381088257, "rewards/margins": 0.661862313747406, "rewards/rejected": -0.29312705993652344, "step": 40 }, { "epoch": 0.13, "grad_norm": 193.9339508971275, "learning_rate": 9.979985922607475e-08, "logits/chosen": -4.531577110290527, "logits/rejected": -4.7472710609436035, "logps/chosen": -265.0107116699219, "logps/rejected": -231.0427703857422, "loss": 0.3721, "rewards/accuracies": 0.893750011920929, "rewards/chosen": 0.6958638429641724, "rewards/margins": 1.2129887342453003, "rewards/rejected": -0.5171248316764832, "step": 50 }, { "epoch": 0.15, "grad_norm": 197.66681943180416, "learning_rate": 9.92010391574745e-08, "logits/chosen": -4.6342668533325195, "logits/rejected": -4.783431053161621, "logps/chosen": -230.5199432373047, "logps/rejected": -224.8899383544922, "loss": 0.281, "rewards/accuracies": 0.875, "rewards/chosen": 1.0602517127990723, "rewards/margins": 2.0471043586730957, "rewards/rejected": -0.9868523478507996, "step": 60 }, { "epoch": 0.18, "grad_norm": 162.90653777538404, "learning_rate": 9.820833372667812e-08, "logits/chosen": -4.519268989562988, "logits/rejected": -4.774823188781738, "logps/chosen": -240.6663818359375, "logps/rejected": -230.677490234375, "loss": 0.2583, "rewards/accuracies": 0.9312499761581421, "rewards/chosen": 1.247557520866394, "rewards/margins": 2.539060592651367, "rewards/rejected": -1.2915031909942627, "step": 70 }, { "epoch": 0.2, "grad_norm": 139.86229845766394, "learning_rate": 9.682969016701356e-08, "logits/chosen": -4.514620780944824, "logits/rejected": -4.760144233703613, "logps/chosen": -247.26364135742188, "logps/rejected": -244.28866577148438, "loss": 0.2248, "rewards/accuracies": 0.8999999761581421, "rewards/chosen": 1.3956347703933716, "rewards/margins": 2.7926278114318848, "rewards/rejected": -1.3969930410385132, "step": 80 }, { "epoch": 0.23, "grad_norm": 208.01416582057018, "learning_rate": 9.507614539004081e-08, "logits/chosen": -4.604180812835693, "logits/rejected": -4.849738597869873, "logps/chosen": -236.37899780273438, "logps/rejected": -221.75564575195312, "loss": 0.2139, "rewards/accuracies": 0.9312499761581421, "rewards/chosen": 1.4833731651306152, "rewards/margins": 3.3759377002716064, "rewards/rejected": -1.8925644159317017, "step": 90 }, { "epoch": 0.26, "grad_norm": 168.83388514254116, "learning_rate": 9.296173762811083e-08, "logits/chosen": -4.470773220062256, "logits/rejected": -4.786902904510498, "logps/chosen": -240.86477661132812, "logps/rejected": -247.6592559814453, "loss": 0.216, "rewards/accuracies": 0.893750011920929, "rewards/chosen": 1.5853464603424072, "rewards/margins": 3.6403656005859375, "rewards/rejected": -2.0550191402435303, "step": 100 }, { "epoch": 0.26, "eval_logits/chosen": -4.555809497833252, "eval_logits/rejected": -4.771756172180176, "eval_logps/chosen": -391.56207275390625, "eval_logps/rejected": -513.7159423828125, "eval_loss": 1.1906155347824097, "eval_rewards/accuracies": 0.3984375, "eval_rewards/chosen": -0.10876215249300003, "eval_rewards/margins": -0.575933575630188, "eval_rewards/rejected": 0.46717149019241333, "eval_runtime": 97.8557, "eval_samples_per_second": 20.438, "eval_steps_per_second": 0.327, "step": 100 }, { "epoch": 0.28, "grad_norm": 158.8626687509829, "learning_rate": 9.050339404945832e-08, "logits/chosen": -4.515485763549805, "logits/rejected": -4.795956134796143, "logps/chosen": -232.4200439453125, "logps/rejected": -236.90969848632812, "loss": 0.1985, "rewards/accuracies": 0.925000011920929, "rewards/chosen": 1.6765697002410889, "rewards/margins": 3.698953628540039, "rewards/rejected": -2.02238392829895, "step": 110 }, { "epoch": 0.31, "grad_norm": 135.50801896928647, "learning_rate": 8.77207952455395e-08, "logits/chosen": -4.454637050628662, "logits/rejected": -4.722542762756348, "logps/chosen": -256.7890930175781, "logps/rejected": -248.82601928710938, "loss": 0.1975, "rewards/accuracies": 0.9624999761581421, "rewards/chosen": 1.9200481176376343, "rewards/margins": 3.9417896270751953, "rewards/rejected": -2.0217413902282715, "step": 120 }, { "epoch": 0.33, "grad_norm": 154.49491996909583, "learning_rate": 8.463621767547997e-08, "logits/chosen": -4.527877330780029, "logits/rejected": -4.81399393081665, "logps/chosen": -241.6123809814453, "logps/rejected": -239.6664276123047, "loss": 0.1993, "rewards/accuracies": 0.8999999761581421, "rewards/chosen": 1.683153748512268, "rewards/margins": 4.116929054260254, "rewards/rejected": -2.4337756633758545, "step": 130 }, { "epoch": 0.36, "grad_norm": 161.00485250742125, "learning_rate": 8.127435532896387e-08, "logits/chosen": -4.5556840896606445, "logits/rejected": -4.849120140075684, "logps/chosen": -268.4659118652344, "logps/rejected": -258.8057556152344, "loss": 0.1874, "rewards/accuracies": 0.925000011920929, "rewards/chosen": 1.5729143619537354, "rewards/margins": 4.236142158508301, "rewards/rejected": -2.6632275581359863, "step": 140 }, { "epoch": 0.38, "grad_norm": 157.59104792116975, "learning_rate": 7.766212203526569e-08, "logits/chosen": -4.545467853546143, "logits/rejected": -4.822617053985596, "logps/chosen": -235.0808563232422, "logps/rejected": -246.62606811523438, "loss": 0.1878, "rewards/accuracies": 0.949999988079071, "rewards/chosen": 1.7337404489517212, "rewards/margins": 4.484419822692871, "rewards/rejected": -2.7506790161132812, "step": 150 }, { "epoch": 0.41, "grad_norm": 153.9528654470227, "learning_rate": 7.382843600106538e-08, "logits/chosen": -4.591027736663818, "logits/rejected": -4.801583290100098, "logps/chosen": -234.0863494873047, "logps/rejected": -241.58761596679688, "loss": 0.1687, "rewards/accuracies": 0.9375, "rewards/chosen": 1.7370154857635498, "rewards/margins": 4.334869861602783, "rewards/rejected": -2.5978541374206543, "step": 160 }, { "epoch": 0.43, "grad_norm": 134.86505617626943, "learning_rate": 6.980398830195784e-08, "logits/chosen": -4.457674980163574, "logits/rejected": -4.7707014083862305, "logps/chosen": -239.03787231445312, "logps/rejected": -248.50393676757812, "loss": 0.1647, "rewards/accuracies": 0.9437500238418579, "rewards/chosen": 2.203636407852173, "rewards/margins": 5.073732852935791, "rewards/rejected": -2.870095729827881, "step": 170 }, { "epoch": 0.46, "grad_norm": 134.91302963185322, "learning_rate": 6.562099718102787e-08, "logits/chosen": -4.556507110595703, "logits/rejected": -4.814910411834717, "logps/chosen": -217.7527618408203, "logps/rejected": -222.974365234375, "loss": 0.1885, "rewards/accuracies": 0.9375, "rewards/chosen": 1.9110908508300781, "rewards/margins": 4.606502532958984, "rewards/rejected": -2.695411443710327, "step": 180 }, { "epoch": 0.49, "grad_norm": 164.38739923975325, "learning_rate": 6.131295012148612e-08, "logits/chosen": -4.534274101257324, "logits/rejected": -4.6962666511535645, "logps/chosen": -241.34304809570312, "logps/rejected": -259.50860595703125, "loss": 0.1854, "rewards/accuracies": 0.8999999761581421, "rewards/chosen": 1.9549798965454102, "rewards/margins": 4.389232158660889, "rewards/rejected": -2.434252977371216, "step": 190 }, { "epoch": 0.51, "grad_norm": 145.61398569600198, "learning_rate": 5.691433575823665e-08, "logits/chosen": -4.528763771057129, "logits/rejected": -4.724316596984863, "logps/chosen": -234.06503295898438, "logps/rejected": -241.2750701904297, "loss": 0.1925, "rewards/accuracies": 0.875, "rewards/chosen": 1.8074201345443726, "rewards/margins": 4.400271892547607, "rewards/rejected": -2.5928521156311035, "step": 200 }, { "epoch": 0.51, "eval_logits/chosen": -4.5165863037109375, "eval_logits/rejected": -4.742655277252197, "eval_logps/chosen": -394.47015380859375, "eval_logps/rejected": -515.5682983398438, "eval_loss": 1.3562896251678467, "eval_rewards/accuracies": 0.375, "eval_rewards/chosen": -0.39956870675086975, "eval_rewards/margins": -0.6815085411071777, "eval_rewards/rejected": 0.28193986415863037, "eval_runtime": 97.991, "eval_samples_per_second": 20.41, "eval_steps_per_second": 0.327, "step": 200 }, { "epoch": 0.54, "grad_norm": 126.33823591196861, "learning_rate": 5.2460367774593905e-08, "logits/chosen": -4.592769622802734, "logits/rejected": -4.851358413696289, "logps/chosen": -244.64755249023438, "logps/rejected": -259.65142822265625, "loss": 0.1489, "rewards/accuracies": 0.9437500238418579, "rewards/chosen": 2.1237542629241943, "rewards/margins": 5.224299907684326, "rewards/rejected": -3.10054612159729, "step": 210 }, { "epoch": 0.56, "grad_norm": 139.63020748735394, "learning_rate": 4.798670299452925e-08, "logits/chosen": -4.435277462005615, "logits/rejected": -4.771773338317871, "logps/chosen": -242.7268524169922, "logps/rejected": -258.1542053222656, "loss": 0.1754, "rewards/accuracies": 0.9312499761581421, "rewards/chosen": 2.0084481239318848, "rewards/margins": 5.356553077697754, "rewards/rejected": -3.348104953765869, "step": 220 }, { "epoch": 0.59, "grad_norm": 146.4580429105822, "learning_rate": 4.3529155927297226e-08, "logits/chosen": -4.511857986450195, "logits/rejected": -4.845773696899414, "logps/chosen": -241.4152374267578, "logps/rejected": -257.61944580078125, "loss": 0.1815, "rewards/accuracies": 0.9312499761581421, "rewards/chosen": 1.9553873538970947, "rewards/margins": 5.346222877502441, "rewards/rejected": -3.390836000442505, "step": 230 }, { "epoch": 0.61, "grad_norm": 120.36905618473185, "learning_rate": 3.9123412049691636e-08, "logits/chosen": -4.488260269165039, "logits/rejected": -4.767331123352051, "logps/chosen": -252.67538452148438, "logps/rejected": -255.2183837890625, "loss": 0.1687, "rewards/accuracies": 0.9375, "rewards/chosen": 2.014777421951294, "rewards/margins": 5.443254470825195, "rewards/rejected": -3.4284775257110596, "step": 240 }, { "epoch": 0.64, "grad_norm": 143.11323504040695, "learning_rate": 3.480474212128766e-08, "logits/chosen": -4.618422031402588, "logits/rejected": -4.878515243530273, "logps/chosen": -231.21737670898438, "logps/rejected": -236.189208984375, "loss": 0.1614, "rewards/accuracies": 0.949999988079071, "rewards/chosen": 1.7484108209609985, "rewards/margins": 4.606410503387451, "rewards/rejected": -2.857999324798584, "step": 250 }, { "epoch": 0.66, "grad_norm": 193.1124896787253, "learning_rate": 3.060771981975726e-08, "logits/chosen": -4.48922872543335, "logits/rejected": -4.790060997009277, "logps/chosen": -234.9071502685547, "logps/rejected": -258.86639404296875, "loss": 0.1653, "rewards/accuracies": 0.949999988079071, "rewards/chosen": 1.9141381978988647, "rewards/margins": 5.825932025909424, "rewards/rejected": -3.9117939472198486, "step": 260 }, { "epoch": 0.69, "grad_norm": 194.5027032928207, "learning_rate": 2.6565944956764818e-08, "logits/chosen": -4.580440044403076, "logits/rejected": -4.814294815063477, "logps/chosen": -242.62802124023438, "logps/rejected": -251.90444946289062, "loss": 0.1739, "rewards/accuracies": 0.9375, "rewards/chosen": 1.84052312374115, "rewards/margins": 5.5284624099731445, "rewards/rejected": -3.687939405441284, "step": 270 }, { "epoch": 0.72, "grad_norm": 134.789590992645, "learning_rate": 2.2711774490274766e-08, "logits/chosen": -4.531357765197754, "logits/rejected": -4.778365135192871, "logps/chosen": -244.4646453857422, "logps/rejected": -277.8075866699219, "loss": 0.148, "rewards/accuracies": 0.96875, "rewards/chosen": 2.004220485687256, "rewards/margins": 5.491039276123047, "rewards/rejected": -3.48681902885437, "step": 280 }, { "epoch": 0.74, "grad_norm": 281.0472172318649, "learning_rate": 1.9076063486687256e-08, "logits/chosen": -4.403564453125, "logits/rejected": -4.759608268737793, "logps/chosen": -251.33419799804688, "logps/rejected": -248.35458374023438, "loss": 0.1633, "rewards/accuracies": 0.925000011920929, "rewards/chosen": 2.1274170875549316, "rewards/margins": 5.399934768676758, "rewards/rejected": -3.272517681121826, "step": 290 }, { "epoch": 0.77, "grad_norm": 153.20971602377742, "learning_rate": 1.5687918106563324e-08, "logits/chosen": -4.5167622566223145, "logits/rejected": -4.735228061676025, "logps/chosen": -232.205322265625, "logps/rejected": -250.81884765625, "loss": 0.179, "rewards/accuracies": 0.8812500238418579, "rewards/chosen": 2.1202826499938965, "rewards/margins": 5.605620861053467, "rewards/rejected": -3.4853386878967285, "step": 300 }, { "epoch": 0.77, "eval_logits/chosen": -4.521942615509033, "eval_logits/rejected": -4.748414039611816, "eval_logps/chosen": -397.169189453125, "eval_logps/rejected": -518.8555297851562, "eval_loss": 1.3649907112121582, "eval_rewards/accuracies": 0.3828125, "eval_rewards/chosen": -0.6694751977920532, "eval_rewards/margins": -0.6226927638053894, "eval_rewards/rejected": -0.04678243398666382, "eval_runtime": 97.9822, "eval_samples_per_second": 20.412, "eval_steps_per_second": 0.327, "step": 300 }, { "epoch": 0.79, "grad_norm": 149.9412754602278, "learning_rate": 1.257446259144494e-08, "logits/chosen": -4.436882972717285, "logits/rejected": -4.769654750823975, "logps/chosen": -239.6269073486328, "logps/rejected": -259.68951416015625, "loss": 0.1671, "rewards/accuracies": 0.9437500238418579, "rewards/chosen": 2.1512722969055176, "rewards/margins": 5.90042781829834, "rewards/rejected": -3.7491557598114014, "step": 310 }, { "epoch": 0.82, "grad_norm": 162.84992879153398, "learning_rate": 9.760622117187234e-09, "logits/chosen": -4.495532512664795, "logits/rejected": -4.839716911315918, "logps/chosen": -226.40927124023438, "logps/rejected": -241.6173553466797, "loss": 0.1706, "rewards/accuracies": 0.9312499761581421, "rewards/chosen": 1.8755052089691162, "rewards/margins": 5.392090797424316, "rewards/rejected": -3.5165855884552, "step": 320 }, { "epoch": 0.84, "grad_norm": 140.15174845446813, "learning_rate": 7.2689232521989885e-09, "logits/chosen": -4.450541019439697, "logits/rejected": -4.755129814147949, "logps/chosen": -248.2396240234375, "logps/rejected": -268.4003601074219, "loss": 0.159, "rewards/accuracies": 0.918749988079071, "rewards/chosen": 1.9909223318099976, "rewards/margins": 5.5609917640686035, "rewards/rejected": -3.5700690746307373, "step": 330 }, { "epoch": 0.87, "grad_norm": 126.08085826498804, "learning_rate": 5.119313618049309e-09, "logits/chosen": -4.463766574859619, "logits/rejected": -4.806758403778076, "logps/chosen": -251.0575714111328, "logps/rejected": -241.99609375, "loss": 0.1558, "rewards/accuracies": 0.9312499761581421, "rewards/chosen": 2.308844804763794, "rewards/margins": 5.830248832702637, "rewards/rejected": -3.5214037895202637, "step": 340 }, { "epoch": 0.9, "grad_norm": 122.26451804586765, "learning_rate": 3.3290021961708158e-09, "logits/chosen": -4.487333297729492, "logits/rejected": -4.672482967376709, "logps/chosen": -237.46456909179688, "logps/rejected": -258.9417419433594, "loss": 0.1724, "rewards/accuracies": 0.925000011920929, "rewards/chosen": 1.8006614446640015, "rewards/margins": 4.9178361892700195, "rewards/rejected": -3.117175340652466, "step": 350 }, { "epoch": 0.92, "grad_norm": 144.944818423744, "learning_rate": 1.9123215591052013e-09, "logits/chosen": -4.482793807983398, "logits/rejected": -4.7083001136779785, "logps/chosen": -243.4811248779297, "logps/rejected": -260.84857177734375, "loss": 0.1742, "rewards/accuracies": 0.918749988079071, "rewards/chosen": 1.8153793811798096, "rewards/margins": 5.153982162475586, "rewards/rejected": -3.3386025428771973, "step": 360 }, { "epoch": 0.95, "grad_norm": 132.15693545315082, "learning_rate": 8.806131292167618e-10, "logits/chosen": -4.498129367828369, "logits/rejected": -4.671444892883301, "logps/chosen": -237.79995727539062, "logps/rejected": -269.40106201171875, "loss": 0.1751, "rewards/accuracies": 0.887499988079071, "rewards/chosen": 1.9021259546279907, "rewards/margins": 5.157528877258301, "rewards/rejected": -3.2554030418395996, "step": 370 }, { "epoch": 0.97, "grad_norm": 150.94805422584076, "learning_rate": 2.4213638345040867e-10, "logits/chosen": -4.600647926330566, "logits/rejected": -4.883923053741455, "logps/chosen": -242.57833862304688, "logps/rejected": -258.09075927734375, "loss": 0.1659, "rewards/accuracies": 0.925000011920929, "rewards/chosen": 1.9504365921020508, "rewards/margins": 5.70705509185791, "rewards/rejected": -3.7566189765930176, "step": 380 }, { "epoch": 1.0, "grad_norm": 216.65022888736644, "learning_rate": 2.0027310073833516e-12, "logits/chosen": -4.585878372192383, "logits/rejected": -4.851449966430664, "logps/chosen": -237.5348663330078, "logps/rejected": -255.65542602539062, "loss": 0.1632, "rewards/accuracies": 0.9312499761581421, "rewards/chosen": 2.213772773742676, "rewards/margins": 5.727804183959961, "rewards/rejected": -3.514031171798706, "step": 390 }, { "epoch": 1.0, "step": 391, "total_flos": 0.0, "train_loss": 0.23296360492401416, "train_runtime": 6150.4195, "train_samples_per_second": 8.13, "train_steps_per_second": 0.064 } ], "logging_steps": 10, "max_steps": 391, "num_input_tokens_seen": 0, "num_train_epochs": 1, "save_steps": 100, "total_flos": 0.0, "train_batch_size": 8, "trial_name": null, "trial_params": null }