| { | |
| "best_metric": null, | |
| "best_model_checkpoint": null, | |
| "epoch": 0.9996020692399522, | |
| "eval_steps": 100, | |
| "global_step": 1256, | |
| "is_hyper_param_search": false, | |
| "is_local_process_zero": true, | |
| "is_world_process_zero": true, | |
| "log_history": [ | |
| { | |
| "debug/losses": 0.23031963407993317, | |
| "debug/policy_weights": 0.3322809934616089, | |
| "debug/raw_losses": 0.6931471824645996, | |
| "epoch": 0.0007958615200955034, | |
| "grad_norm": 1.6286401468051575, | |
| "learning_rate": 3.968253968253968e-09, | |
| "logits/chosen": -2.735659122467041, | |
| "logits/rejected": -2.7581238746643066, | |
| "logps/chosen": -124.62968444824219, | |
| "logps/rejected": -168.09475708007812, | |
| "loss": 0.2239, | |
| "rewards/accuracies": 0.0, | |
| "rewards/chosen": 0.0, | |
| "rewards/margins": 0.0, | |
| "rewards/rejected": 0.0, | |
| "step": 1 | |
| }, | |
| { | |
| "debug/losses": 0.23611842095851898, | |
| "debug/policy_weights": 0.34072285890579224, | |
| "debug/raw_losses": 0.6929619312286377, | |
| "epoch": 0.007958615200955034, | |
| "grad_norm": 1.6881469544577703, | |
| "learning_rate": 3.968253968253968e-08, | |
| "logits/chosen": -2.738783836364746, | |
| "logits/rejected": -2.7277822494506836, | |
| "logps/chosen": -146.68910217285156, | |
| "logps/rejected": -131.2349395751953, | |
| "loss": 0.2295, | |
| "rewards/accuracies": 0.4375, | |
| "rewards/chosen": 0.00013088027480989695, | |
| "rewards/margins": 0.00037292600609362125, | |
| "rewards/rejected": -0.00024204571673180908, | |
| "step": 10 | |
| }, | |
| { | |
| "debug/losses": 0.2265315055847168, | |
| "debug/policy_weights": 0.3267918825149536, | |
| "debug/raw_losses": 0.693217396736145, | |
| "epoch": 0.01591723040191007, | |
| "grad_norm": 1.5615628005517825, | |
| "learning_rate": 7.936507936507936e-08, | |
| "logits/chosen": -2.706993341445923, | |
| "logits/rejected": -2.703998327255249, | |
| "logps/chosen": -129.48587036132812, | |
| "logps/rejected": -130.25735473632812, | |
| "loss": 0.2238, | |
| "rewards/accuracies": 0.44999998807907104, | |
| "rewards/chosen": -0.00017557166574988514, | |
| "rewards/margins": -0.000137387789436616, | |
| "rewards/rejected": -3.818388358922675e-05, | |
| "step": 20 | |
| }, | |
| { | |
| "debug/losses": 0.2132270336151123, | |
| "debug/policy_weights": 0.30768805742263794, | |
| "debug/raw_losses": 0.6929622888565063, | |
| "epoch": 0.0238758456028651, | |
| "grad_norm": 1.5392325714344397, | |
| "learning_rate": 1.1904761904761903e-07, | |
| "logits/chosen": -2.6836955547332764, | |
| "logits/rejected": -2.680663824081421, | |
| "logps/chosen": -141.81492614746094, | |
| "logps/rejected": -155.6810760498047, | |
| "loss": 0.2264, | |
| "rewards/accuracies": 0.581250011920929, | |
| "rewards/chosen": 0.00016820887685753405, | |
| "rewards/margins": 0.0003723005356732756, | |
| "rewards/rejected": -0.00020409165881574154, | |
| "step": 30 | |
| }, | |
| { | |
| "debug/losses": 0.2176501303911209, | |
| "debug/policy_weights": 0.3141174018383026, | |
| "debug/raw_losses": 0.6928995847702026, | |
| "epoch": 0.03183446080382014, | |
| "grad_norm": 1.638997322742743, | |
| "learning_rate": 1.5873015873015872e-07, | |
| "logits/chosen": -2.6918673515319824, | |
| "logits/rejected": -2.6841723918914795, | |
| "logps/chosen": -154.97286987304688, | |
| "logps/rejected": -164.1558837890625, | |
| "loss": 0.221, | |
| "rewards/accuracies": 0.5625, | |
| "rewards/chosen": -0.001508228713646531, | |
| "rewards/margins": 0.0005008662701584399, | |
| "rewards/rejected": -0.0020090951584279537, | |
| "step": 40 | |
| }, | |
| { | |
| "debug/losses": 0.2295212298631668, | |
| "debug/policy_weights": 0.33170244097709656, | |
| "debug/raw_losses": 0.6918987035751343, | |
| "epoch": 0.03979307600477517, | |
| "grad_norm": 1.4818468652902261, | |
| "learning_rate": 1.984126984126984e-07, | |
| "logits/chosen": -2.7066802978515625, | |
| "logits/rejected": -2.6879210472106934, | |
| "logps/chosen": -144.00912475585938, | |
| "logps/rejected": -137.74859619140625, | |
| "loss": 0.2233, | |
| "rewards/accuracies": 0.612500011920929, | |
| "rewards/chosen": -0.002855247352272272, | |
| "rewards/margins": 0.0025138729251921177, | |
| "rewards/rejected": -0.005369120743125677, | |
| "step": 50 | |
| }, | |
| { | |
| "debug/losses": 0.22746217250823975, | |
| "debug/policy_weights": 0.3287571668624878, | |
| "debug/raw_losses": 0.6916018724441528, | |
| "epoch": 0.0477516912057302, | |
| "grad_norm": 1.4868186691240965, | |
| "learning_rate": 2.3809523809523806e-07, | |
| "logits/chosen": -2.715649127960205, | |
| "logits/rejected": -2.716421604156494, | |
| "logps/chosen": -145.9324951171875, | |
| "logps/rejected": -159.49798583984375, | |
| "loss": 0.2191, | |
| "rewards/accuracies": 0.5687500238418579, | |
| "rewards/chosen": -0.003058128524571657, | |
| "rewards/margins": 0.0031352252699434757, | |
| "rewards/rejected": -0.006193353794515133, | |
| "step": 60 | |
| }, | |
| { | |
| "debug/losses": 0.2176016867160797, | |
| "debug/policy_weights": 0.3151531517505646, | |
| "debug/raw_losses": 0.6905783414840698, | |
| "epoch": 0.055710306406685235, | |
| "grad_norm": 1.5355880647836464, | |
| "learning_rate": 2.7777777777777776e-07, | |
| "logits/chosen": -2.736604690551758, | |
| "logits/rejected": -2.727555990219116, | |
| "logps/chosen": -149.36459350585938, | |
| "logps/rejected": -143.43045043945312, | |
| "loss": 0.2113, | |
| "rewards/accuracies": 0.625, | |
| "rewards/chosen": -0.017263194546103477, | |
| "rewards/margins": 0.005380765534937382, | |
| "rewards/rejected": -0.022643957287073135, | |
| "step": 70 | |
| }, | |
| { | |
| "debug/losses": 0.1883382946252823, | |
| "debug/policy_weights": 0.2724798023700714, | |
| "debug/raw_losses": 0.6915446519851685, | |
| "epoch": 0.06366892160764027, | |
| "grad_norm": 1.5327858491562372, | |
| "learning_rate": 3.1746031746031743e-07, | |
| "logits/chosen": -2.7107303142547607, | |
| "logits/rejected": -2.6923205852508545, | |
| "logps/chosen": -157.98587036132812, | |
| "logps/rejected": -149.39295959472656, | |
| "loss": 0.2011, | |
| "rewards/accuracies": 0.5249999761581421, | |
| "rewards/chosen": -0.04851604253053665, | |
| "rewards/margins": 0.003855167655274272, | |
| "rewards/rejected": -0.05237121507525444, | |
| "step": 80 | |
| }, | |
| { | |
| "debug/losses": 0.18660078942775726, | |
| "debug/policy_weights": 0.2731621563434601, | |
| "debug/raw_losses": 0.6829166412353516, | |
| "epoch": 0.07162753680859531, | |
| "grad_norm": 1.6166482985332293, | |
| "learning_rate": 3.5714285714285716e-07, | |
| "logits/chosen": -2.7208943367004395, | |
| "logits/rejected": -2.727738618850708, | |
| "logps/chosen": -152.66610717773438, | |
| "logps/rejected": -173.5172882080078, | |
| "loss": 0.185, | |
| "rewards/accuracies": 0.606249988079071, | |
| "rewards/chosen": -0.06819866597652435, | |
| "rewards/margins": 0.022195402532815933, | |
| "rewards/rejected": -0.09039406478404999, | |
| "step": 90 | |
| }, | |
| { | |
| "debug/losses": 0.16780678927898407, | |
| "debug/policy_weights": 0.24758808314800262, | |
| "debug/raw_losses": 0.6784400939941406, | |
| "epoch": 0.07958615200955034, | |
| "grad_norm": 1.5200040949416276, | |
| "learning_rate": 3.968253968253968e-07, | |
| "logits/chosen": -2.6890692710876465, | |
| "logits/rejected": -2.6723039150238037, | |
| "logps/chosen": -149.3301239013672, | |
| "logps/rejected": -143.3484344482422, | |
| "loss": 0.1733, | |
| "rewards/accuracies": 0.5625, | |
| "rewards/chosen": -0.11518146842718124, | |
| "rewards/margins": 0.03295627981424332, | |
| "rewards/rejected": -0.14813776314258575, | |
| "step": 100 | |
| }, | |
| { | |
| "epoch": 0.07958615200955034, | |
| "eval_debug/losses": 0.1618107706308365, | |
| "eval_debug/policy_weights": 0.23841436207294464, | |
| "eval_debug/raw_losses": 0.6786514520645142, | |
| "eval_logits/chosen": -2.711678981781006, | |
| "eval_logits/rejected": -2.703619956970215, | |
| "eval_logps/chosen": -158.4381103515625, | |
| "eval_logps/rejected": -168.3044891357422, | |
| "eval_loss": 0.1633094847202301, | |
| "eval_rewards/accuracies": 0.5979477763175964, | |
| "eval_rewards/chosen": -0.1419461965560913, | |
| "eval_rewards/margins": 0.03410893306136131, | |
| "eval_rewards/rejected": -0.17605511844158173, | |
| "eval_runtime": 152.723, | |
| "eval_samples_per_second": 55.997, | |
| "eval_steps_per_second": 0.877, | |
| "step": 100 | |
| }, | |
| { | |
| "debug/losses": 0.15096870064735413, | |
| "debug/policy_weights": 0.2191181182861328, | |
| "debug/raw_losses": 0.6909996867179871, | |
| "epoch": 0.08754476721050537, | |
| "grad_norm": 1.5046677998036695, | |
| "learning_rate": 4.365079365079365e-07, | |
| "logits/chosen": -2.6899752616882324, | |
| "logits/rejected": -2.671255350112915, | |
| "logps/chosen": -179.54322814941406, | |
| "logps/rejected": -165.19168090820312, | |
| "loss": 0.1473, | |
| "rewards/accuracies": 0.5375000238418579, | |
| "rewards/chosen": -0.21252474188804626, | |
| "rewards/margins": 0.011284739710390568, | |
| "rewards/rejected": -0.22380945086479187, | |
| "step": 110 | |
| }, | |
| { | |
| "debug/losses": 0.1395491510629654, | |
| "debug/policy_weights": 0.20841534435749054, | |
| "debug/raw_losses": 0.6783668398857117, | |
| "epoch": 0.0955033824114604, | |
| "grad_norm": 1.388858214651452, | |
| "learning_rate": 4.761904761904761e-07, | |
| "logits/chosen": -2.663153886795044, | |
| "logits/rejected": -2.6503779888153076, | |
| "logps/chosen": -167.6672821044922, | |
| "logps/rejected": -172.83718872070312, | |
| "loss": 0.1346, | |
| "rewards/accuracies": 0.574999988079071, | |
| "rewards/chosen": -0.2853550910949707, | |
| "rewards/margins": 0.04338715597987175, | |
| "rewards/rejected": -0.32874220609664917, | |
| "step": 120 | |
| }, | |
| { | |
| "debug/losses": 0.11419715732336044, | |
| "debug/policy_weights": 0.17497313022613525, | |
| "debug/raw_losses": 0.6557947397232056, | |
| "epoch": 0.10346199761241544, | |
| "grad_norm": 2.1752818607208244, | |
| "learning_rate": 4.999845414634076e-07, | |
| "logits/chosen": -2.6734774112701416, | |
| "logits/rejected": -2.645984172821045, | |
| "logps/chosen": -189.3183135986328, | |
| "logps/rejected": -179.66036987304688, | |
| "loss": 0.117, | |
| "rewards/accuracies": 0.6000000238418579, | |
| "rewards/chosen": -0.34408754110336304, | |
| "rewards/margins": 0.09557478129863739, | |
| "rewards/rejected": -0.439662367105484, | |
| "step": 130 | |
| }, | |
| { | |
| "debug/losses": 0.0952613353729248, | |
| "debug/policy_weights": 0.14768476784229279, | |
| "debug/raw_losses": 0.6522240042686462, | |
| "epoch": 0.11142061281337047, | |
| "grad_norm": 2.442265971491927, | |
| "learning_rate": 4.998106548810311e-07, | |
| "logits/chosen": -2.6368653774261475, | |
| "logits/rejected": -2.6138510704040527, | |
| "logps/chosen": -192.3903045654297, | |
| "logps/rejected": -183.68685913085938, | |
| "loss": 0.1083, | |
| "rewards/accuracies": 0.65625, | |
| "rewards/chosen": -0.46787652373313904, | |
| "rewards/margins": 0.11621709167957306, | |
| "rewards/rejected": -0.5840936303138733, | |
| "step": 140 | |
| }, | |
| { | |
| "debug/losses": 0.07598518580198288, | |
| "debug/policy_weights": 0.12855970859527588, | |
| "debug/raw_losses": 0.6079715490341187, | |
| "epoch": 0.1193792280143255, | |
| "grad_norm": 5.777724458868344, | |
| "learning_rate": 4.994436933879359e-07, | |
| "logits/chosen": -2.6171412467956543, | |
| "logits/rejected": -2.610034942626953, | |
| "logps/chosen": -196.5837860107422, | |
| "logps/rejected": -225.31173706054688, | |
| "loss": 0.0881, | |
| "rewards/accuracies": 0.6875, | |
| "rewards/chosen": -0.5704454183578491, | |
| "rewards/margins": 0.22279813885688782, | |
| "rewards/rejected": -0.7932435870170593, | |
| "step": 150 | |
| }, | |
| { | |
| "debug/losses": 0.10459884256124496, | |
| "debug/policy_weights": 0.1645442694425583, | |
| "debug/raw_losses": 0.6207043528556824, | |
| "epoch": 0.12733784321528055, | |
| "grad_norm": 1.6699884563484173, | |
| "learning_rate": 4.988839406031596e-07, | |
| "logits/chosen": -2.6161184310913086, | |
| "logits/rejected": -2.6225523948669434, | |
| "logps/chosen": -160.10772705078125, | |
| "logps/rejected": -207.2742156982422, | |
| "loss": 0.1069, | |
| "rewards/accuracies": 0.6812499761581421, | |
| "rewards/chosen": -0.3946920931339264, | |
| "rewards/margins": 0.20463863015174866, | |
| "rewards/rejected": -0.599330723285675, | |
| "step": 160 | |
| }, | |
| { | |
| "debug/losses": 0.099858358502388, | |
| "debug/policy_weights": 0.1572486311197281, | |
| "debug/raw_losses": 0.6404817700386047, | |
| "epoch": 0.13529645841623558, | |
| "grad_norm": 1.696036642690539, | |
| "learning_rate": 4.981318291512395e-07, | |
| "logits/chosen": -2.5810608863830566, | |
| "logits/rejected": -2.578481912612915, | |
| "logps/chosen": -174.32598876953125, | |
| "logps/rejected": -198.0808868408203, | |
| "loss": 0.1132, | |
| "rewards/accuracies": 0.6312500238418579, | |
| "rewards/chosen": -0.379153311252594, | |
| "rewards/margins": 0.15221598744392395, | |
| "rewards/rejected": -0.5313693284988403, | |
| "step": 170 | |
| }, | |
| { | |
| "debug/losses": 0.09473618865013123, | |
| "debug/policy_weights": 0.13851876556873322, | |
| "debug/raw_losses": 0.672447681427002, | |
| "epoch": 0.14325507361719061, | |
| "grad_norm": 1.9630112175338894, | |
| "learning_rate": 4.971879403278432e-07, | |
| "logits/chosen": -2.5959112644195557, | |
| "logits/rejected": -2.5829200744628906, | |
| "logps/chosen": -202.05296325683594, | |
| "logps/rejected": -207.42648315429688, | |
| "loss": 0.0946, | |
| "rewards/accuracies": 0.5874999761581421, | |
| "rewards/chosen": -0.5352639555931091, | |
| "rewards/margins": 0.0995597094297409, | |
| "rewards/rejected": -0.6348236203193665, | |
| "step": 180 | |
| }, | |
| { | |
| "debug/losses": 0.08237513154745102, | |
| "debug/policy_weights": 0.12221841514110565, | |
| "debug/raw_losses": 0.6608506441116333, | |
| "epoch": 0.15121368881814565, | |
| "grad_norm": 1.9225034938066397, | |
| "learning_rate": 4.960530036504941e-07, | |
| "logits/chosen": -2.579829692840576, | |
| "logits/rejected": -2.5656394958496094, | |
| "logps/chosen": -222.9535369873047, | |
| "logps/rejected": -232.0922393798828, | |
| "loss": 0.0793, | |
| "rewards/accuracies": 0.574999988079071, | |
| "rewards/chosen": -0.7860406637191772, | |
| "rewards/margins": 0.1566535085439682, | |
| "rewards/rejected": -0.9426941871643066, | |
| "step": 190 | |
| }, | |
| { | |
| "debug/losses": 0.08729609102010727, | |
| "debug/policy_weights": 0.13735465705394745, | |
| "debug/raw_losses": 0.6344841718673706, | |
| "epoch": 0.15917230401910068, | |
| "grad_norm": 1.4158943423202177, | |
| "learning_rate": 4.947278962947386e-07, | |
| "logits/chosen": -2.5332934856414795, | |
| "logits/rejected": -2.5345585346221924, | |
| "logps/chosen": -217.6802215576172, | |
| "logps/rejected": -252.038330078125, | |
| "loss": 0.0826, | |
| "rewards/accuracies": 0.643750011920929, | |
| "rewards/chosen": -0.6853083372116089, | |
| "rewards/margins": 0.20577910542488098, | |
| "rewards/rejected": -0.8910874128341675, | |
| "step": 200 | |
| }, | |
| { | |
| "epoch": 0.15917230401910068, | |
| "eval_debug/losses": 0.07361924648284912, | |
| "eval_debug/policy_weights": 0.11718404293060303, | |
| "eval_debug/raw_losses": 0.6317233443260193, | |
| "eval_logits/chosen": -2.550771474838257, | |
| "eval_logits/rejected": -2.540393590927124, | |
| "eval_logps/chosen": -223.08139038085938, | |
| "eval_logps/rejected": -250.74151611328125, | |
| "eval_loss": 0.07500571012496948, | |
| "eval_rewards/accuracies": 0.6361940503120422, | |
| "eval_rewards/chosen": -0.7883791327476501, | |
| "eval_rewards/margins": 0.21204641461372375, | |
| "eval_rewards/rejected": -1.0004255771636963, | |
| "eval_runtime": 153.1417, | |
| "eval_samples_per_second": 55.844, | |
| "eval_steps_per_second": 0.875, | |
| "step": 200 | |
| }, | |
| { | |
| "debug/losses": 0.06145762279629707, | |
| "debug/policy_weights": 0.10109380632638931, | |
| "debug/raw_losses": 0.6302995681762695, | |
| "epoch": 0.1671309192200557, | |
| "grad_norm": 1.435571888650618, | |
| "learning_rate": 4.932136424161899e-07, | |
| "logits/chosen": -2.521623134613037, | |
| "logits/rejected": -2.50742769241333, | |
| "logps/chosen": -217.82498168945312, | |
| "logps/rejected": -248.6222381591797, | |
| "loss": 0.0666, | |
| "rewards/accuracies": 0.643750011920929, | |
| "rewards/chosen": -0.922324538230896, | |
| "rewards/margins": 0.24683670699596405, | |
| "rewards/rejected": -1.169161319732666, | |
| "step": 210 | |
| }, | |
| { | |
| "debug/losses": 0.04205631464719772, | |
| "debug/policy_weights": 0.07819141447544098, | |
| "debug/raw_losses": 0.5699876546859741, | |
| "epoch": 0.17508953442101075, | |
| "grad_norm": 1.336610780534668, | |
| "learning_rate": 4.915114123589732e-07, | |
| "logits/chosen": -2.550821304321289, | |
| "logits/rejected": -2.526845932006836, | |
| "logps/chosen": -245.73306274414062, | |
| "logps/rejected": -283.7450256347656, | |
| "loss": 0.0522, | |
| "rewards/accuracies": 0.6812499761581421, | |
| "rewards/chosen": -1.0883342027664185, | |
| "rewards/margins": 0.3927660584449768, | |
| "rewards/rejected": -1.48110032081604, | |
| "step": 220 | |
| }, | |
| { | |
| "debug/losses": 0.04529764503240585, | |
| "debug/policy_weights": 0.07175824046134949, | |
| "debug/raw_losses": 0.6204754114151001, | |
| "epoch": 0.18304814962196578, | |
| "grad_norm": 1.4784857016304622, | |
| "learning_rate": 4.896225217511849e-07, | |
| "logits/chosen": -2.553422212600708, | |
| "logits/rejected": -2.5453922748565674, | |
| "logps/chosen": -263.7728576660156, | |
| "logps/rejected": -298.7444152832031, | |
| "loss": 0.0493, | |
| "rewards/accuracies": 0.637499988079071, | |
| "rewards/chosen": -1.2447619438171387, | |
| "rewards/margins": 0.2553327679634094, | |
| "rewards/rejected": -1.5000946521759033, | |
| "step": 230 | |
| }, | |
| { | |
| "debug/losses": 0.06539560854434967, | |
| "debug/policy_weights": 0.10245949029922485, | |
| "debug/raw_losses": 0.6345073580741882, | |
| "epoch": 0.1910067648229208, | |
| "grad_norm": 1.9058271009788008, | |
| "learning_rate": 4.875484304880629e-07, | |
| "logits/chosen": -2.576491594314575, | |
| "logits/rejected": -2.558601140975952, | |
| "logps/chosen": -257.90118408203125, | |
| "logps/rejected": -279.42266845703125, | |
| "loss": 0.0646, | |
| "rewards/accuracies": 0.643750011920929, | |
| "rewards/chosen": -0.9614318609237671, | |
| "rewards/margins": 0.2736155390739441, | |
| "rewards/rejected": -1.2350473403930664, | |
| "step": 240 | |
| }, | |
| { | |
| "debug/losses": 0.06740079820156097, | |
| "debug/policy_weights": 0.11094751209020615, | |
| "debug/raw_losses": 0.6248602271080017, | |
| "epoch": 0.19896538002387584, | |
| "grad_norm": 1.5384599137611568, | |
| "learning_rate": 4.852907416036558e-07, | |
| "logits/chosen": -2.521726131439209, | |
| "logits/rejected": -2.512889862060547, | |
| "logps/chosen": -222.5371856689453, | |
| "logps/rejected": -262.0302429199219, | |
| "loss": 0.0757, | |
| "rewards/accuracies": 0.668749988079071, | |
| "rewards/chosen": -0.8324035406112671, | |
| "rewards/margins": 0.2716120183467865, | |
| "rewards/rejected": -1.1040157079696655, | |
| "step": 250 | |
| }, | |
| { | |
| "debug/losses": 0.06672520935535431, | |
| "debug/policy_weights": 0.11189812421798706, | |
| "debug/raw_losses": 0.5824211239814758, | |
| "epoch": 0.20692399522483088, | |
| "grad_norm": 1.4195949507142585, | |
| "learning_rate": 4.828512000318616e-07, | |
| "logits/chosen": -2.535505533218384, | |
| "logits/rejected": -2.4970450401306152, | |
| "logps/chosen": -254.94741821289062, | |
| "logps/rejected": -278.82440185546875, | |
| "loss": 0.0639, | |
| "rewards/accuracies": 0.706250011920929, | |
| "rewards/chosen": -0.9024659395217896, | |
| "rewards/margins": 0.3342443108558655, | |
| "rewards/rejected": -1.2367103099822998, | |
| "step": 260 | |
| }, | |
| { | |
| "debug/losses": 0.05524461343884468, | |
| "debug/policy_weights": 0.09467937052249908, | |
| "debug/raw_losses": 0.600980281829834, | |
| "epoch": 0.2148826104257859, | |
| "grad_norm": 2.09985567586005, | |
| "learning_rate": 4.802316912577946e-07, | |
| "logits/chosen": -2.4608120918273926, | |
| "logits/rejected": -2.4260432720184326, | |
| "logps/chosen": -242.8441925048828, | |
| "logps/rejected": -261.7018127441406, | |
| "loss": 0.0597, | |
| "rewards/accuracies": 0.6625000238418579, | |
| "rewards/chosen": -0.9519554972648621, | |
| "rewards/margins": 0.32279661297798157, | |
| "rewards/rejected": -1.2747520208358765, | |
| "step": 270 | |
| }, | |
| { | |
| "debug/losses": 0.07073532044887543, | |
| "debug/policy_weights": 0.11124340444803238, | |
| "debug/raw_losses": 0.6386277079582214, | |
| "epoch": 0.22284122562674094, | |
| "grad_norm": 2.53125193502766, | |
| "learning_rate": 4.774342398605221e-07, | |
| "logits/chosen": -2.4330108165740967, | |
| "logits/rejected": -2.4124581813812256, | |
| "logps/chosen": -238.8041229248047, | |
| "logps/rejected": -258.5404357910156, | |
| "loss": 0.0674, | |
| "rewards/accuracies": 0.612500011920929, | |
| "rewards/chosen": -0.9319969415664673, | |
| "rewards/margins": 0.2547939419746399, | |
| "rewards/rejected": -1.186790943145752, | |
| "step": 280 | |
| }, | |
| { | |
| "debug/losses": 0.06495990604162216, | |
| "debug/policy_weights": 0.10757051408290863, | |
| "debug/raw_losses": 0.5930755138397217, | |
| "epoch": 0.23079984082769597, | |
| "grad_norm": 2.544321863170394, | |
| "learning_rate": 4.744610079482978e-07, | |
| "logits/chosen": -2.4562830924987793, | |
| "logits/rejected": -2.4185662269592285, | |
| "logps/chosen": -274.9333190917969, | |
| "logps/rejected": -300.23583984375, | |
| "loss": 0.0558, | |
| "rewards/accuracies": 0.6625000238418579, | |
| "rewards/chosen": -1.0925931930541992, | |
| "rewards/margins": 0.3425787389278412, | |
| "rewards/rejected": -1.4351718425750732, | |
| "step": 290 | |
| }, | |
| { | |
| "debug/losses": 0.04091046005487442, | |
| "debug/policy_weights": 0.07422160357236862, | |
| "debug/raw_losses": 0.5923314094543457, | |
| "epoch": 0.238758456028651, | |
| "grad_norm": 2.8319076712554905, | |
| "learning_rate": 4.713142934875005e-07, | |
| "logits/chosen": -2.3712522983551025, | |
| "logits/rejected": -2.329404592514038, | |
| "logps/chosen": -278.4283752441406, | |
| "logps/rejected": -303.9710388183594, | |
| "loss": 0.0515, | |
| "rewards/accuracies": 0.6625000238418579, | |
| "rewards/chosen": -1.30873441696167, | |
| "rewards/margins": 0.4222927689552307, | |
| "rewards/rejected": -1.7310272455215454, | |
| "step": 300 | |
| }, | |
| { | |
| "epoch": 0.238758456028651, | |
| "eval_debug/losses": 0.0557919405400753, | |
| "eval_debug/policy_weights": 0.09022855013608932, | |
| "eval_debug/raw_losses": 0.6134340167045593, | |
| "eval_logits/chosen": -2.3552868366241455, | |
| "eval_logits/rejected": -2.339716911315918, | |
| "eval_logps/chosen": -268.7242736816406, | |
| "eval_logps/rejected": -311.3038635253906, | |
| "eval_loss": 0.05731642618775368, | |
| "eval_rewards/accuracies": 0.6567164063453674, | |
| "eval_rewards/chosen": -1.2448077201843262, | |
| "eval_rewards/margins": 0.36124110221862793, | |
| "eval_rewards/rejected": -1.6060487031936646, | |
| "eval_runtime": 153.2008, | |
| "eval_samples_per_second": 55.822, | |
| "eval_steps_per_second": 0.875, | |
| "step": 300 | |
| }, | |
| { | |
| "debug/losses": 0.0425337590277195, | |
| "debug/policy_weights": 0.06980495154857635, | |
| "debug/raw_losses": 0.588487982749939, | |
| "epoch": 0.24671707122960604, | |
| "grad_norm": 1.7020687918550936, | |
| "learning_rate": 4.679965285265706e-07, | |
| "logits/chosen": -2.2830398082733154, | |
| "logits/rejected": -2.270655632019043, | |
| "logps/chosen": -256.02520751953125, | |
| "logps/rejected": -309.3921813964844, | |
| "loss": 0.0508, | |
| "rewards/accuracies": 0.643750011920929, | |
| "rewards/chosen": -1.3847135305404663, | |
| "rewards/margins": 0.4131029546260834, | |
| "rewards/rejected": -1.7978166341781616, | |
| "step": 310 | |
| }, | |
| { | |
| "debug/losses": 0.030957188457250595, | |
| "debug/policy_weights": 0.05928220599889755, | |
| "debug/raw_losses": 0.5176903009414673, | |
| "epoch": 0.2546756864305611, | |
| "grad_norm": 1.8070710565204129, | |
| "learning_rate": 4.64510277316316e-07, | |
| "logits/chosen": -2.213348150253296, | |
| "logits/rejected": -2.1686458587646484, | |
| "logps/chosen": -307.92364501953125, | |
| "logps/rejected": -366.7274169921875, | |
| "loss": 0.0327, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": -1.6229156255722046, | |
| "rewards/margins": 0.6521427631378174, | |
| "rewards/rejected": -2.2750582695007324, | |
| "step": 320 | |
| }, | |
| { | |
| "debug/losses": 0.03477818891406059, | |
| "debug/policy_weights": 0.05532154440879822, | |
| "debug/raw_losses": 0.6181614398956299, | |
| "epoch": 0.26263430163151613, | |
| "grad_norm": 1.6888122864648538, | |
| "learning_rate": 4.6085823432804137e-07, | |
| "logits/chosen": -2.2881710529327393, | |
| "logits/rejected": -2.259243965148926, | |
| "logps/chosen": -319.549560546875, | |
| "logps/rejected": -354.44024658203125, | |
| "loss": 0.0355, | |
| "rewards/accuracies": 0.6312500238418579, | |
| "rewards/chosen": -1.757887840270996, | |
| "rewards/margins": 0.3513242304325104, | |
| "rewards/rejected": -2.1092123985290527, | |
| "step": 330 | |
| }, | |
| { | |
| "debug/losses": 0.036134570837020874, | |
| "debug/policy_weights": 0.0601133331656456, | |
| "debug/raw_losses": 0.6012001633644104, | |
| "epoch": 0.27059291683247116, | |
| "grad_norm": 0.9082619220908468, | |
| "learning_rate": 4.570432221710314e-07, | |
| "logits/chosen": -2.345127820968628, | |
| "logits/rejected": -2.328043222427368, | |
| "logps/chosen": -311.11541748046875, | |
| "logps/rejected": -360.7956237792969, | |
| "loss": 0.0329, | |
| "rewards/accuracies": 0.625, | |
| "rewards/chosen": -1.6493396759033203, | |
| "rewards/margins": 0.4032120704650879, | |
| "rewards/rejected": -2.052551746368408, | |
| "step": 340 | |
| }, | |
| { | |
| "debug/losses": 0.038389407098293304, | |
| "debug/policy_weights": 0.0665694996714592, | |
| "debug/raw_losses": 0.5971581339836121, | |
| "epoch": 0.2785515320334262, | |
| "grad_norm": 1.7673927299473366, | |
| "learning_rate": 4.5306818941099866e-07, | |
| "logits/chosen": -2.3454818725585938, | |
| "logits/rejected": -2.2886033058166504, | |
| "logps/chosen": -309.58526611328125, | |
| "logps/rejected": -334.3452453613281, | |
| "loss": 0.0377, | |
| "rewards/accuracies": 0.675000011920929, | |
| "rewards/chosen": -1.5936106443405151, | |
| "rewards/margins": 0.41180863976478577, | |
| "rewards/rejected": -2.0054192543029785, | |
| "step": 350 | |
| }, | |
| { | |
| "debug/losses": 0.03970758244395256, | |
| "debug/policy_weights": 0.06552158296108246, | |
| "debug/raw_losses": 0.5963835716247559, | |
| "epoch": 0.28651014723438123, | |
| "grad_norm": 1.165368953491439, | |
| "learning_rate": 4.4893620829118124e-07, | |
| "logits/chosen": -2.3107028007507324, | |
| "logits/rejected": -2.2786478996276855, | |
| "logps/chosen": -296.3944091796875, | |
| "logps/rejected": -329.440185546875, | |
| "loss": 0.042, | |
| "rewards/accuracies": 0.675000011920929, | |
| "rewards/chosen": -1.4465725421905518, | |
| "rewards/margins": 0.39386042952537537, | |
| "rewards/rejected": -1.840433120727539, | |
| "step": 360 | |
| }, | |
| { | |
| "debug/losses": 0.031909000128507614, | |
| "debug/policy_weights": 0.05640440434217453, | |
| "debug/raw_losses": 0.632055401802063, | |
| "epoch": 0.29446876243533626, | |
| "grad_norm": 1.4365386513442289, | |
| "learning_rate": 4.4465047235785185e-07, | |
| "logits/chosen": -2.3353095054626465, | |
| "logits/rejected": -2.299114942550659, | |
| "logps/chosen": -316.80322265625, | |
| "logps/rejected": -334.6367492675781, | |
| "loss": 0.038, | |
| "rewards/accuracies": 0.6625000238418579, | |
| "rewards/chosen": -1.719588041305542, | |
| "rewards/margins": 0.34410953521728516, | |
| "rewards/rejected": -2.063697576522827, | |
| "step": 370 | |
| }, | |
| { | |
| "debug/losses": 0.03546866774559021, | |
| "debug/policy_weights": 0.05660950019955635, | |
| "debug/raw_losses": 0.6386741399765015, | |
| "epoch": 0.3024273776362913, | |
| "grad_norm": 1.6981468490942673, | |
| "learning_rate": 4.40214293992074e-07, | |
| "logits/chosen": -2.302638530731201, | |
| "logits/rejected": -2.2782890796661377, | |
| "logps/chosen": -302.3027038574219, | |
| "logps/rejected": -331.7867736816406, | |
| "loss": 0.0384, | |
| "rewards/accuracies": 0.6499999761581421, | |
| "rewards/chosen": -1.649095892906189, | |
| "rewards/margins": 0.3226591646671295, | |
| "rewards/rejected": -1.971754789352417, | |
| "step": 380 | |
| }, | |
| { | |
| "debug/losses": 0.04937135428190231, | |
| "debug/policy_weights": 0.07896497845649719, | |
| "debug/raw_losses": 0.6217660903930664, | |
| "epoch": 0.3103859928372463, | |
| "grad_norm": 1.7063718152736185, | |
| "learning_rate": 4.3563110184961234e-07, | |
| "logits/chosen": -2.327444076538086, | |
| "logits/rejected": -2.305551052093506, | |
| "logps/chosen": -294.81793212890625, | |
| "logps/rejected": -332.69305419921875, | |
| "loss": 0.0437, | |
| "rewards/accuracies": 0.643750011920929, | |
| "rewards/chosen": -1.4561359882354736, | |
| "rewards/margins": 0.34932222962379456, | |
| "rewards/rejected": -1.8054578304290771, | |
| "step": 390 | |
| }, | |
| { | |
| "debug/losses": 0.03210610896348953, | |
| "debug/policy_weights": 0.06029694527387619, | |
| "debug/raw_losses": 0.5555212497711182, | |
| "epoch": 0.31834460803820136, | |
| "grad_norm": 1.053077631697114, | |
| "learning_rate": 4.3090443821097566e-07, | |
| "logits/chosen": -2.3126392364501953, | |
| "logits/rejected": -2.290933847427368, | |
| "logps/chosen": -292.9631042480469, | |
| "logps/rejected": -358.627685546875, | |
| "loss": 0.0343, | |
| "rewards/accuracies": 0.699999988079071, | |
| "rewards/chosen": -1.5689239501953125, | |
| "rewards/margins": 0.5151135325431824, | |
| "rewards/rejected": -2.0840373039245605, | |
| "step": 400 | |
| }, | |
| { | |
| "epoch": 0.31834460803820136, | |
| "eval_debug/losses": 0.028384286910295486, | |
| "eval_debug/policy_weights": 0.04820017144083977, | |
| "eval_debug/raw_losses": 0.59935063123703, | |
| "eval_logits/chosen": -2.3006632328033447, | |
| "eval_logits/rejected": -2.2854816913604736, | |
| "eval_logps/chosen": -321.4913330078125, | |
| "eval_logps/rejected": -364.0837097167969, | |
| "eval_loss": 0.030249595642089844, | |
| "eval_rewards/accuracies": 0.6623134613037109, | |
| "eval_rewards/chosen": -1.7724782228469849, | |
| "eval_rewards/margins": 0.3613690733909607, | |
| "eval_rewards/rejected": -2.133847236633301, | |
| "eval_runtime": 153.4178, | |
| "eval_samples_per_second": 55.743, | |
| "eval_steps_per_second": 0.873, | |
| "step": 400 | |
| }, | |
| { | |
| "debug/losses": 0.029884925112128258, | |
| "debug/policy_weights": 0.05188853666186333, | |
| "debug/raw_losses": 0.5859032869338989, | |
| "epoch": 0.3263032232391564, | |
| "grad_norm": 1.1682793896386887, | |
| "learning_rate": 4.2603795624364195e-07, | |
| "logits/chosen": -2.279048204421997, | |
| "logits/rejected": -2.2422895431518555, | |
| "logps/chosen": -302.4944152832031, | |
| "logps/rejected": -329.99310302734375, | |
| "loss": 0.0303, | |
| "rewards/accuracies": 0.6937500238418579, | |
| "rewards/chosen": -1.6612056493759155, | |
| "rewards/margins": 0.3638765215873718, | |
| "rewards/rejected": -2.0250821113586426, | |
| "step": 410 | |
| }, | |
| { | |
| "debug/losses": 0.039774149656295776, | |
| "debug/policy_weights": 0.06452381610870361, | |
| "debug/raw_losses": 0.5788149833679199, | |
| "epoch": 0.3342618384401114, | |
| "grad_norm": 1.1186339494698643, | |
| "learning_rate": 4.210354171785795e-07, | |
| "logits/chosen": -2.3332016468048096, | |
| "logits/rejected": -2.331998109817505, | |
| "logps/chosen": -291.0079040527344, | |
| "logps/rejected": -344.42462158203125, | |
| "loss": 0.0361, | |
| "rewards/accuracies": 0.7124999761581421, | |
| "rewards/chosen": -1.488434910774231, | |
| "rewards/margins": 0.40906715393066406, | |
| "rewards/rejected": -1.8975019454956055, | |
| "step": 420 | |
| }, | |
| { | |
| "debug/losses": 0.03884187713265419, | |
| "debug/policy_weights": 0.06631166487932205, | |
| "debug/raw_losses": 0.5891371369361877, | |
| "epoch": 0.34222045364106646, | |
| "grad_norm": 1.8153925533673183, | |
| "learning_rate": 4.15900687403248e-07, | |
| "logits/chosen": -2.3279812335968018, | |
| "logits/rejected": -2.3141000270843506, | |
| "logps/chosen": -287.6651916503906, | |
| "logps/rejected": -336.90863037109375, | |
| "loss": 0.0393, | |
| "rewards/accuracies": 0.643750011920929, | |
| "rewards/chosen": -1.4973087310791016, | |
| "rewards/margins": 0.46901196241378784, | |
| "rewards/rejected": -1.9663206338882446, | |
| "step": 430 | |
| }, | |
| { | |
| "debug/losses": 0.03900137543678284, | |
| "debug/policy_weights": 0.06590630114078522, | |
| "debug/raw_losses": 0.6019418239593506, | |
| "epoch": 0.3501790688420215, | |
| "grad_norm": 1.3373117259952034, | |
| "learning_rate": 4.1063773547332584e-07, | |
| "logits/chosen": -2.332784652709961, | |
| "logits/rejected": -2.311373233795166, | |
| "logps/chosen": -290.93402099609375, | |
| "logps/rejected": -337.70306396484375, | |
| "loss": 0.0411, | |
| "rewards/accuracies": 0.6499999761581421, | |
| "rewards/chosen": -1.4399511814117432, | |
| "rewards/margins": 0.4576480984687805, | |
| "rewards/rejected": -1.897599458694458, | |
| "step": 440 | |
| }, | |
| { | |
| "debug/losses": 0.034195203334093094, | |
| "debug/policy_weights": 0.06440727412700653, | |
| "debug/raw_losses": 0.5752583742141724, | |
| "epoch": 0.3581376840429765, | |
| "grad_norm": 1.4642214637111448, | |
| "learning_rate": 4.0525062904547276e-07, | |
| "logits/chosen": -2.274158000946045, | |
| "logits/rejected": -2.241867780685425, | |
| "logps/chosen": -274.84783935546875, | |
| "logps/rejected": -318.5943908691406, | |
| "loss": 0.0424, | |
| "rewards/accuracies": 0.6937500238418579, | |
| "rewards/chosen": -1.4228484630584717, | |
| "rewards/margins": 0.5137365460395813, | |
| "rewards/rejected": -1.9365848302841187, | |
| "step": 450 | |
| }, | |
| { | |
| "debug/losses": 0.042845163494348526, | |
| "debug/policy_weights": 0.07054711878299713, | |
| "debug/raw_losses": 0.5637282729148865, | |
| "epoch": 0.36609629924393156, | |
| "grad_norm": 1.493914915308785, | |
| "learning_rate": 3.997435317334988e-07, | |
| "logits/chosen": -2.3144097328186035, | |
| "logits/rejected": -2.2954559326171875, | |
| "logps/chosen": -297.01629638671875, | |
| "logps/rejected": -354.07861328125, | |
| "loss": 0.0384, | |
| "rewards/accuracies": 0.6875, | |
| "rewards/chosen": -1.5000154972076416, | |
| "rewards/margins": 0.5090978145599365, | |
| "rewards/rejected": -2.009113311767578, | |
| "step": 460 | |
| }, | |
| { | |
| "debug/losses": 0.03523220494389534, | |
| "debug/policy_weights": 0.060066692531108856, | |
| "debug/raw_losses": 0.5853220820426941, | |
| "epoch": 0.3740549144448866, | |
| "grad_norm": 1.5606293995092184, | |
| "learning_rate": 3.941206998903701e-07, | |
| "logits/chosen": -2.290365219116211, | |
| "logits/rejected": -2.2592685222625732, | |
| "logps/chosen": -331.3341369628906, | |
| "logps/rejected": -367.52630615234375, | |
| "loss": 0.031, | |
| "rewards/accuracies": 0.6937500238418579, | |
| "rewards/chosen": -1.7656469345092773, | |
| "rewards/margins": 0.38830217719078064, | |
| "rewards/rejected": -2.153949022293091, | |
| "step": 470 | |
| }, | |
| { | |
| "debug/losses": 0.042910732328891754, | |
| "debug/policy_weights": 0.06740695238113403, | |
| "debug/raw_losses": 0.5751327276229858, | |
| "epoch": 0.3820135296458416, | |
| "grad_norm": 2.0400801936994357, | |
| "learning_rate": 3.8838647931853684e-07, | |
| "logits/chosen": -2.2397708892822266, | |
| "logits/rejected": -2.2155957221984863, | |
| "logps/chosen": -265.8719177246094, | |
| "logps/rejected": -317.6902770996094, | |
| "loss": 0.0417, | |
| "rewards/accuracies": 0.7124999761581421, | |
| "rewards/chosen": -1.4397175312042236, | |
| "rewards/margins": 0.4852531850337982, | |
| "rewards/rejected": -1.9249706268310547, | |
| "step": 480 | |
| }, | |
| { | |
| "debug/losses": 0.040277738124132156, | |
| "debug/policy_weights": 0.07625563442707062, | |
| "debug/raw_losses": 0.5467380881309509, | |
| "epoch": 0.38997214484679665, | |
| "grad_norm": 1.5717280943877756, | |
| "learning_rate": 3.825453019111281e-07, | |
| "logits/chosen": -2.246934175491333, | |
| "logits/rejected": -2.2185163497924805, | |
| "logps/chosen": -281.0770568847656, | |
| "logps/rejected": -348.90753173828125, | |
| "loss": 0.0441, | |
| "rewards/accuracies": 0.71875, | |
| "rewards/chosen": -1.3660808801651, | |
| "rewards/margins": 0.5780023336410522, | |
| "rewards/rejected": -1.9440832138061523, | |
| "step": 490 | |
| }, | |
| { | |
| "debug/losses": 0.0474771223962307, | |
| "debug/policy_weights": 0.08785078674554825, | |
| "debug/raw_losses": 0.5391517281532288, | |
| "epoch": 0.3979307600477517, | |
| "grad_norm": 1.89593573498899, | |
| "learning_rate": 3.7660168222660824e-07, | |
| "logits/chosen": -2.312988519668579, | |
| "logits/rejected": -2.2645742893218994, | |
| "logps/chosen": -305.178466796875, | |
| "logps/rejected": -341.88360595703125, | |
| "loss": 0.0432, | |
| "rewards/accuracies": 0.706250011920929, | |
| "rewards/chosen": -1.3672921657562256, | |
| "rewards/margins": 0.5291160941123962, | |
| "rewards/rejected": -1.8964084386825562, | |
| "step": 500 | |
| }, | |
| { | |
| "epoch": 0.3979307600477517, | |
| "eval_debug/losses": 0.0406743586063385, | |
| "eval_debug/policy_weights": 0.07022719830274582, | |
| "eval_debug/raw_losses": 0.5892212986946106, | |
| "eval_logits/chosen": -2.264326333999634, | |
| "eval_logits/rejected": -2.240631580352783, | |
| "eval_logps/chosen": -294.8950500488281, | |
| "eval_logps/rejected": -349.04681396484375, | |
| "eval_loss": 0.043212514370679855, | |
| "eval_rewards/accuracies": 0.6800373196601868, | |
| "eval_rewards/chosen": -1.5065159797668457, | |
| "eval_rewards/margins": 0.4769621789455414, | |
| "eval_rewards/rejected": -1.9834781885147095, | |
| "eval_runtime": 153.3333, | |
| "eval_samples_per_second": 55.774, | |
| "eval_steps_per_second": 0.874, | |
| "step": 500 | |
| }, | |
| { | |
| "debug/losses": 0.04252048209309578, | |
| "debug/policy_weights": 0.07039301842451096, | |
| "debug/raw_losses": 0.6011983156204224, | |
| "epoch": 0.4058893752487067, | |
| "grad_norm": 2.2077694886776533, | |
| "learning_rate": 3.705602139995416e-07, | |
| "logits/chosen": -2.2369987964630127, | |
| "logits/rejected": -2.203916072845459, | |
| "logps/chosen": -297.1964111328125, | |
| "logps/rejected": -339.70843505859375, | |
| "loss": 0.0499, | |
| "rewards/accuracies": 0.6625000238418579, | |
| "rewards/chosen": -1.4493403434753418, | |
| "rewards/margins": 0.44409775733947754, | |
| "rewards/rejected": -1.8934379816055298, | |
| "step": 510 | |
| }, | |
| { | |
| "debug/losses": 0.045193079859018326, | |
| "debug/policy_weights": 0.07069429010152817, | |
| "debug/raw_losses": 0.6190410256385803, | |
| "epoch": 0.41384799044966175, | |
| "grad_norm": 1.3854536697917073, | |
| "learning_rate": 3.6442556659016475e-07, | |
| "logits/chosen": -2.264357328414917, | |
| "logits/rejected": -2.24722957611084, | |
| "logps/chosen": -291.65087890625, | |
| "logps/rejected": -329.7967224121094, | |
| "loss": 0.0443, | |
| "rewards/accuracies": 0.699999988079071, | |
| "rewards/chosen": -1.4414405822753906, | |
| "rewards/margins": 0.3577377200126648, | |
| "rewards/rejected": -1.7991783618927002, | |
| "step": 520 | |
| }, | |
| { | |
| "debug/losses": 0.03731568530201912, | |
| "debug/policy_weights": 0.06065317243337631, | |
| "debug/raw_losses": 0.6242992281913757, | |
| "epoch": 0.4218066056506168, | |
| "grad_norm": 1.0635014002404581, | |
| "learning_rate": 3.582024813755076e-07, | |
| "logits/chosen": -2.1829025745391846, | |
| "logits/rejected": -2.159717082977295, | |
| "logps/chosen": -333.71392822265625, | |
| "logps/rejected": -360.9361877441406, | |
| "loss": 0.0308, | |
| "rewards/accuracies": 0.668749988079071, | |
| "rewards/chosen": -1.8308143615722656, | |
| "rewards/margins": 0.3124064803123474, | |
| "rewards/rejected": -2.143220901489258, | |
| "step": 530 | |
| }, | |
| { | |
| "debug/losses": 0.02375711500644684, | |
| "debug/policy_weights": 0.03976626321673393, | |
| "debug/raw_losses": 0.5894423127174377, | |
| "epoch": 0.4297652208515718, | |
| "grad_norm": 0.8777673540587257, | |
| "learning_rate": 3.5189576808485404e-07, | |
| "logits/chosen": -2.1102588176727295, | |
| "logits/rejected": -2.0886712074279785, | |
| "logps/chosen": -356.2108154296875, | |
| "logps/rejected": -402.5682373046875, | |
| "loss": 0.0241, | |
| "rewards/accuracies": 0.7124999761581421, | |
| "rewards/chosen": -2.0562400817871094, | |
| "rewards/margins": 0.4145965576171875, | |
| "rewards/rejected": -2.470837116241455, | |
| "step": 540 | |
| }, | |
| { | |
| "debug/losses": 0.026336893439292908, | |
| "debug/policy_weights": 0.04056663066148758, | |
| "debug/raw_losses": 0.6279866099357605, | |
| "epoch": 0.43772383605252685, | |
| "grad_norm": 1.3575157514235345, | |
| "learning_rate": 3.4551030108237433e-07, | |
| "logits/chosen": -2.0719475746154785, | |
| "logits/rejected": -2.0144124031066895, | |
| "logps/chosen": -366.66064453125, | |
| "logps/rejected": -384.7783508300781, | |
| "loss": 0.0252, | |
| "rewards/accuracies": 0.6625000238418579, | |
| "rewards/chosen": -2.140892505645752, | |
| "rewards/margins": 0.3489706516265869, | |
| "rewards/rejected": -2.4898629188537598, | |
| "step": 550 | |
| }, | |
| { | |
| "debug/losses": 0.023122292011976242, | |
| "debug/policy_weights": 0.045903079211711884, | |
| "debug/raw_losses": 0.567331850528717, | |
| "epoch": 0.4456824512534819, | |
| "grad_norm": 1.197996404456996, | |
| "learning_rate": 3.390510155998023e-07, | |
| "logits/chosen": -2.1358299255371094, | |
| "logits/rejected": -2.1018567085266113, | |
| "logps/chosen": -359.2723693847656, | |
| "logps/rejected": -411.61187744140625, | |
| "loss": 0.0241, | |
| "rewards/accuracies": 0.6875, | |
| "rewards/chosen": -2.050138473510742, | |
| "rewards/margins": 0.5038273334503174, | |
| "rewards/rejected": -2.5539660453796387, | |
| "step": 560 | |
| }, | |
| { | |
| "debug/losses": 0.029903370887041092, | |
| "debug/policy_weights": 0.050540171563625336, | |
| "debug/raw_losses": 0.5872690081596375, | |
| "epoch": 0.4536410664544369, | |
| "grad_norm": 1.5301014116065141, | |
| "learning_rate": 3.325229039220684e-07, | |
| "logits/chosen": -2.0992767810821533, | |
| "logits/rejected": -2.076805830001831, | |
| "logps/chosen": -361.57342529296875, | |
| "logps/rejected": -405.8957214355469, | |
| "loss": 0.0286, | |
| "rewards/accuracies": 0.6625000238418579, | |
| "rewards/chosen": -2.0752933025360107, | |
| "rewards/margins": 0.4622616171836853, | |
| "rewards/rejected": -2.537554979324341, | |
| "step": 570 | |
| }, | |
| { | |
| "debug/losses": 0.03479158133268356, | |
| "debug/policy_weights": 0.053991250693798065, | |
| "debug/raw_losses": 0.604568600654602, | |
| "epoch": 0.46159968165539195, | |
| "grad_norm": 1.41615105697594, | |
| "learning_rate": 3.2593101152883795e-07, | |
| "logits/chosen": -2.110628366470337, | |
| "logits/rejected": -2.0749945640563965, | |
| "logps/chosen": -349.9741516113281, | |
| "logps/rejected": -388.0547180175781, | |
| "loss": 0.0249, | |
| "rewards/accuracies": 0.6187499761581421, | |
| "rewards/chosen": -2.010434150695801, | |
| "rewards/margins": 0.40317314863204956, | |
| "rewards/rejected": -2.413607120513916, | |
| "step": 580 | |
| }, | |
| { | |
| "debug/losses": 0.03422771021723747, | |
| "debug/policy_weights": 0.05810137465596199, | |
| "debug/raw_losses": 0.5813102126121521, | |
| "epoch": 0.469558296856347, | |
| "grad_norm": 1.4269890028062786, | |
| "learning_rate": 3.192804331949349e-07, | |
| "logits/chosen": -2.1150734424591064, | |
| "logits/rejected": -2.0873661041259766, | |
| "logps/chosen": -322.30841064453125, | |
| "logps/rejected": -366.505859375, | |
| "loss": 0.0311, | |
| "rewards/accuracies": 0.706250011920929, | |
| "rewards/chosen": -1.7967876195907593, | |
| "rewards/margins": 0.4749871790409088, | |
| "rewards/rejected": -2.2717747688293457, | |
| "step": 590 | |
| }, | |
| { | |
| "debug/losses": 0.03016613982617855, | |
| "debug/policy_weights": 0.05547459051012993, | |
| "debug/raw_losses": 0.5693393349647522, | |
| "epoch": 0.477516912057302, | |
| "grad_norm": 1.2863972010725921, | |
| "learning_rate": 3.125763090526674e-07, | |
| "logits/chosen": -2.1585888862609863, | |
| "logits/rejected": -2.11460542678833, | |
| "logps/chosen": -328.352783203125, | |
| "logps/rejected": -376.36846923828125, | |
| "loss": 0.0342, | |
| "rewards/accuracies": 0.6937500238418579, | |
| "rewards/chosen": -1.735060691833496, | |
| "rewards/margins": 0.5392208099365234, | |
| "rewards/rejected": -2.2742817401885986, | |
| "step": 600 | |
| }, | |
| { | |
| "epoch": 0.477516912057302, | |
| "eval_debug/losses": 0.030185788869857788, | |
| "eval_debug/policy_weights": 0.05269056186079979, | |
| "eval_debug/raw_losses": 0.5812317132949829, | |
| "eval_logits/chosen": -2.13511323928833, | |
| "eval_logits/rejected": -2.11336612701416, | |
| "eval_logps/chosen": -327.0503234863281, | |
| "eval_logps/rejected": -381.1920471191406, | |
| "eval_loss": 0.032057907432317734, | |
| "eval_rewards/accuracies": 0.6875, | |
| "eval_rewards/chosen": -1.8280682563781738, | |
| "eval_rewards/margins": 0.4768621623516083, | |
| "eval_rewards/rejected": -2.3049304485321045, | |
| "eval_runtime": 153.2709, | |
| "eval_samples_per_second": 55.797, | |
| "eval_steps_per_second": 0.874, | |
| "step": 600 | |
| }, | |
| { | |
| "debug/losses": 0.02869519032537937, | |
| "debug/policy_weights": 0.045216310769319534, | |
| "debug/raw_losses": 0.6201798319816589, | |
| "epoch": 0.48547552725825704, | |
| "grad_norm": 1.0413008003733695, | |
| "learning_rate": 3.0582382061909623e-07, | |
| "logits/chosen": -2.144442081451416, | |
| "logits/rejected": -2.114197015762329, | |
| "logps/chosen": -334.3606262207031, | |
| "logps/rejected": -370.7452087402344, | |
| "loss": 0.031, | |
| "rewards/accuracies": 0.643750011920929, | |
| "rewards/chosen": -1.9323441982269287, | |
| "rewards/margins": 0.3452734053134918, | |
| "rewards/rejected": -2.277617931365967, | |
| "step": 610 | |
| }, | |
| { | |
| "debug/losses": 0.02686266042292118, | |
| "debug/policy_weights": 0.04530490189790726, | |
| "debug/raw_losses": 0.5958613753318787, | |
| "epoch": 0.4934341424592121, | |
| "grad_norm": 1.20614262286929, | |
| "learning_rate": 2.9902818679131775e-07, | |
| "logits/chosen": -2.1490912437438965, | |
| "logits/rejected": -2.1047191619873047, | |
| "logps/chosen": -345.37994384765625, | |
| "logps/rejected": -382.47808837890625, | |
| "loss": 0.0274, | |
| "rewards/accuracies": 0.637499988079071, | |
| "rewards/chosen": -1.9065704345703125, | |
| "rewards/margins": 0.42660021781921387, | |
| "rewards/rejected": -2.3331706523895264, | |
| "step": 620 | |
| }, | |
| { | |
| "debug/losses": 0.02654409408569336, | |
| "debug/policy_weights": 0.04730083793401718, | |
| "debug/raw_losses": 0.5461606979370117, | |
| "epoch": 0.5013927576601671, | |
| "grad_norm": 1.8099237202122547, | |
| "learning_rate": 2.921946598128571e-07, | |
| "logits/chosen": -2.12278413772583, | |
| "logits/rejected": -2.0931880474090576, | |
| "logps/chosen": -322.9065246582031, | |
| "logps/rejected": -354.53228759765625, | |
| "loss": 0.0319, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": -1.7012741565704346, | |
| "rewards/margins": 0.5234018564224243, | |
| "rewards/rejected": -2.2246758937835693, | |
| "step": 630 | |
| }, | |
| { | |
| "debug/losses": 0.037343092262744904, | |
| "debug/policy_weights": 0.053458474576473236, | |
| "debug/raw_losses": 0.6420666575431824, | |
| "epoch": 0.5093513728611222, | |
| "grad_norm": 1.6865475152716571, | |
| "learning_rate": 2.8532852121428733e-07, | |
| "logits/chosen": -2.1078145503997803, | |
| "logits/rejected": -2.0716660022735596, | |
| "logps/chosen": -296.3316345214844, | |
| "logps/rejected": -330.1187744140625, | |
| "loss": 0.0344, | |
| "rewards/accuracies": 0.5874999761581421, | |
| "rewards/chosen": -1.6080780029296875, | |
| "rewards/margins": 0.36063456535339355, | |
| "rewards/rejected": -1.968712568283081, | |
| "step": 640 | |
| }, | |
| { | |
| "debug/losses": 0.035708196461200714, | |
| "debug/policy_weights": 0.05902569368481636, | |
| "debug/raw_losses": 0.5638469457626343, | |
| "epoch": 0.5173099880620772, | |
| "grad_norm": 1.7034498752676623, | |
| "learning_rate": 2.7843507773121414e-07, | |
| "logits/chosen": -2.1405997276306152, | |
| "logits/rejected": -2.108238697052002, | |
| "logps/chosen": -292.4433288574219, | |
| "logps/rejected": -354.64459228515625, | |
| "loss": 0.0376, | |
| "rewards/accuracies": 0.768750011920929, | |
| "rewards/chosen": -1.5522750616073608, | |
| "rewards/margins": 0.5501760840415955, | |
| "rewards/rejected": -2.1024510860443115, | |
| "step": 650 | |
| }, | |
| { | |
| "debug/losses": 0.029026631265878677, | |
| "debug/policy_weights": 0.05990219861268997, | |
| "debug/raw_losses": 0.531947135925293, | |
| "epoch": 0.5252686032630323, | |
| "grad_norm": 1.5938360839379533, | |
| "learning_rate": 2.715196572027789e-07, | |
| "logits/chosen": -2.1009621620178223, | |
| "logits/rejected": -2.0724639892578125, | |
| "logps/chosen": -312.18450927734375, | |
| "logps/rejected": -384.11865234375, | |
| "loss": 0.0339, | |
| "rewards/accuracies": 0.768750011920929, | |
| "rewards/chosen": -1.6771888732910156, | |
| "rewards/margins": 0.6227995753288269, | |
| "rewards/rejected": -2.2999885082244873, | |
| "step": 660 | |
| }, | |
| { | |
| "debug/losses": 0.029919123277068138, | |
| "debug/policy_weights": 0.05170232057571411, | |
| "debug/raw_losses": 0.6014626026153564, | |
| "epoch": 0.5332272184639872, | |
| "grad_norm": 1.5125072558657653, | |
| "learning_rate": 2.645876044538521e-07, | |
| "logits/chosen": -2.1004645824432373, | |
| "logits/rejected": -2.0667197704315186, | |
| "logps/chosen": -330.8302307128906, | |
| "logps/rejected": -369.1127624511719, | |
| "loss": 0.0319, | |
| "rewards/accuracies": 0.675000011920929, | |
| "rewards/chosen": -1.8169825077056885, | |
| "rewards/margins": 0.46573132276535034, | |
| "rewards/rejected": -2.2827141284942627, | |
| "step": 670 | |
| }, | |
| { | |
| "debug/losses": 0.038384515792131424, | |
| "debug/policy_weights": 0.06169766187667847, | |
| "debug/raw_losses": 0.5807624459266663, | |
| "epoch": 0.5411858336649423, | |
| "grad_norm": 1.2825455723938728, | |
| "learning_rate": 2.5764427716409815e-07, | |
| "logits/chosen": -2.126770496368408, | |
| "logits/rejected": -2.0941073894500732, | |
| "logps/chosen": -320.5771484375, | |
| "logps/rejected": -363.2606506347656, | |
| "loss": 0.0316, | |
| "rewards/accuracies": 0.7124999761581421, | |
| "rewards/chosen": -1.6580756902694702, | |
| "rewards/margins": 0.4379847049713135, | |
| "rewards/rejected": -2.096060276031494, | |
| "step": 680 | |
| }, | |
| { | |
| "debug/losses": 0.03115171566605568, | |
| "debug/policy_weights": 0.051781851798295975, | |
| "debug/raw_losses": 0.6048017144203186, | |
| "epoch": 0.5491444488658973, | |
| "grad_norm": 1.312607359472945, | |
| "learning_rate": 2.5069504172710494e-07, | |
| "logits/chosen": -2.095470905303955, | |
| "logits/rejected": -2.08461332321167, | |
| "logps/chosen": -335.7181091308594, | |
| "logps/rejected": -388.94171142578125, | |
| "loss": 0.0288, | |
| "rewards/accuracies": 0.6937500238418579, | |
| "rewards/chosen": -1.873830795288086, | |
| "rewards/margins": 0.3542255759239197, | |
| "rewards/rejected": -2.2280564308166504, | |
| "step": 690 | |
| }, | |
| { | |
| "debug/losses": 0.032391004264354706, | |
| "debug/policy_weights": 0.04837799817323685, | |
| "debug/raw_losses": 0.63223797082901, | |
| "epoch": 0.5571030640668524, | |
| "grad_norm": 1.2365349964087866, | |
| "learning_rate": 2.4374526910277886e-07, | |
| "logits/chosen": -2.010230541229248, | |
| "logits/rejected": -1.984021782875061, | |
| "logps/chosen": -325.3149108886719, | |
| "logps/rejected": -352.573974609375, | |
| "loss": 0.0283, | |
| "rewards/accuracies": 0.59375, | |
| "rewards/chosen": -1.8190257549285889, | |
| "rewards/margins": 0.3168783187866211, | |
| "rewards/rejected": -2.13590407371521, | |
| "step": 700 | |
| }, | |
| { | |
| "epoch": 0.5571030640668524, | |
| "eval_debug/losses": 0.026820143684744835, | |
| "eval_debug/policy_weights": 0.04667918384075165, | |
| "eval_debug/raw_losses": 0.5765854716300964, | |
| "eval_logits/chosen": -1.9899719953536987, | |
| "eval_logits/rejected": -1.9676791429519653, | |
| "eval_logps/chosen": -328.6565856933594, | |
| "eval_logps/rejected": -378.7768859863281, | |
| "eval_loss": 0.028307339176535606, | |
| "eval_rewards/accuracies": 0.6940298676490784, | |
| "eval_rewards/chosen": -1.8441307544708252, | |
| "eval_rewards/margins": 0.4366479218006134, | |
| "eval_rewards/rejected": -2.280778408050537, | |
| "eval_runtime": 153.3492, | |
| "eval_samples_per_second": 55.768, | |
| "eval_steps_per_second": 0.874, | |
| "step": 700 | |
| }, | |
| { | |
| "debug/losses": 0.026440713554620743, | |
| "debug/policy_weights": 0.04469674825668335, | |
| "debug/raw_losses": 0.5913997292518616, | |
| "epoch": 0.5650616792678074, | |
| "grad_norm": 1.7865897282724608, | |
| "learning_rate": 2.368003306662104e-07, | |
| "logits/chosen": -1.9654159545898438, | |
| "logits/rejected": -1.9266160726547241, | |
| "logps/chosen": -352.88787841796875, | |
| "logps/rejected": -381.1947326660156, | |
| "loss": 0.0276, | |
| "rewards/accuracies": 0.6499999761581421, | |
| "rewards/chosen": -1.9729446172714233, | |
| "rewards/margins": 0.3949921727180481, | |
| "rewards/rejected": -2.367936611175537, | |
| "step": 710 | |
| }, | |
| { | |
| "debug/losses": 0.028649378567934036, | |
| "debug/policy_weights": 0.05744954198598862, | |
| "debug/raw_losses": 0.5734940767288208, | |
| "epoch": 0.5730202944687625, | |
| "grad_norm": 1.4165928688447773, | |
| "learning_rate": 2.2986559405621886e-07, | |
| "logits/chosen": -1.9602625370025635, | |
| "logits/rejected": -1.916290283203125, | |
| "logps/chosen": -343.48406982421875, | |
| "logps/rejected": -386.95989990234375, | |
| "loss": 0.0308, | |
| "rewards/accuracies": 0.6875, | |
| "rewards/chosen": -1.8104501962661743, | |
| "rewards/margins": 0.4908137917518616, | |
| "rewards/rejected": -2.3012638092041016, | |
| "step": 720 | |
| }, | |
| { | |
| "debug/losses": 0.022339364513754845, | |
| "debug/policy_weights": 0.03942544013261795, | |
| "debug/raw_losses": 0.5562313795089722, | |
| "epoch": 0.5809789096697174, | |
| "grad_norm": 1.382543972122978, | |
| "learning_rate": 2.2294641902678443e-07, | |
| "logits/chosen": -1.9571382999420166, | |
| "logits/rejected": -1.9248653650283813, | |
| "logps/chosen": -327.402587890625, | |
| "logps/rejected": -379.947998046875, | |
| "loss": 0.0232, | |
| "rewards/accuracies": 0.737500011920929, | |
| "rewards/chosen": -1.9223827123641968, | |
| "rewards/margins": 0.558407723903656, | |
| "rewards/rejected": -2.480790615081787, | |
| "step": 730 | |
| }, | |
| { | |
| "debug/losses": 0.01860680803656578, | |
| "debug/policy_weights": 0.03574604541063309, | |
| "debug/raw_losses": 0.5566806793212891, | |
| "epoch": 0.5889375248706725, | |
| "grad_norm": 1.468728949042428, | |
| "learning_rate": 2.160481533045751e-07, | |
| "logits/chosen": -1.921987533569336, | |
| "logits/rejected": -1.873268485069275, | |
| "logps/chosen": -362.5789489746094, | |
| "logps/rejected": -399.66748046875, | |
| "loss": 0.0225, | |
| "rewards/accuracies": 0.737500011920929, | |
| "rewards/chosen": -2.1244444847106934, | |
| "rewards/margins": 0.46507740020751953, | |
| "rewards/rejected": -2.589521884918213, | |
| "step": 740 | |
| }, | |
| { | |
| "debug/losses": 0.021815910935401917, | |
| "debug/policy_weights": 0.039903424680233, | |
| "debug/raw_losses": 0.5822666883468628, | |
| "epoch": 0.5968961400716275, | |
| "grad_norm": 1.4132116728769042, | |
| "learning_rate": 2.0917612845576882e-07, | |
| "logits/chosen": -1.9699146747589111, | |
| "logits/rejected": -1.9025242328643799, | |
| "logps/chosen": -354.04644775390625, | |
| "logps/rejected": -385.09075927734375, | |
| "loss": 0.024, | |
| "rewards/accuracies": 0.706250011920929, | |
| "rewards/chosen": -1.975638747215271, | |
| "rewards/margins": 0.5243066549301147, | |
| "rewards/rejected": -2.4999454021453857, | |
| "step": 750 | |
| }, | |
| { | |
| "debug/losses": 0.02490827441215515, | |
| "debug/policy_weights": 0.04055667296051979, | |
| "debug/raw_losses": 0.6043840050697327, | |
| "epoch": 0.6048547552725826, | |
| "grad_norm": 1.5603724313512064, | |
| "learning_rate": 2.0233565576536564e-07, | |
| "logits/chosen": -1.9306650161743164, | |
| "logits/rejected": -1.9156144857406616, | |
| "logps/chosen": -329.97711181640625, | |
| "logps/rejected": -378.48834228515625, | |
| "loss": 0.0283, | |
| "rewards/accuracies": 0.65625, | |
| "rewards/chosen": -1.8800745010375977, | |
| "rewards/margins": 0.4346494674682617, | |
| "rewards/rejected": -2.3147239685058594, | |
| "step": 760 | |
| }, | |
| { | |
| "debug/losses": 0.02524595521390438, | |
| "debug/policy_weights": 0.04561594873666763, | |
| "debug/raw_losses": 0.5465348958969116, | |
| "epoch": 0.6128133704735376, | |
| "grad_norm": 1.4684425781915056, | |
| "learning_rate": 1.9553202213217537e-07, | |
| "logits/chosen": -1.9351495504379272, | |
| "logits/rejected": -1.901551604270935, | |
| "logps/chosen": -314.9850769042969, | |
| "logps/rejected": -377.1813049316406, | |
| "loss": 0.0247, | |
| "rewards/accuracies": 0.7437499761581421, | |
| "rewards/chosen": -1.8458607196807861, | |
| "rewards/margins": 0.5721122026443481, | |
| "rewards/rejected": -2.417973041534424, | |
| "step": 770 | |
| }, | |
| { | |
| "debug/losses": 0.024128446355462074, | |
| "debug/policy_weights": 0.04677557945251465, | |
| "debug/raw_losses": 0.5592825412750244, | |
| "epoch": 0.6207719856744927, | |
| "grad_norm": 1.055203900824062, | |
| "learning_rate": 1.887704859826528e-07, | |
| "logits/chosen": -1.976380705833435, | |
| "logits/rejected": -1.9334523677825928, | |
| "logps/chosen": -364.484375, | |
| "logps/rejected": -424.99957275390625, | |
| "loss": 0.0247, | |
| "rewards/accuracies": 0.71875, | |
| "rewards/chosen": -2.032318353652954, | |
| "rewards/margins": 0.5795284509658813, | |
| "rewards/rejected": -2.611846923828125, | |
| "step": 780 | |
| }, | |
| { | |
| "debug/losses": 0.024484191089868546, | |
| "debug/policy_weights": 0.04067380353808403, | |
| "debug/raw_losses": 0.5770185589790344, | |
| "epoch": 0.6287306008754476, | |
| "grad_norm": 1.216416574340141, | |
| "learning_rate": 1.8205627320673836e-07, | |
| "logits/chosen": -1.9579511880874634, | |
| "logits/rejected": -1.9054477214813232, | |
| "logps/chosen": -366.54754638671875, | |
| "logps/rejected": -416.3515625, | |
| "loss": 0.0229, | |
| "rewards/accuracies": 0.7124999761581421, | |
| "rewards/chosen": -2.1298985481262207, | |
| "rewards/margins": 0.5691512823104858, | |
| "rewards/rejected": -2.699049472808838, | |
| "step": 790 | |
| }, | |
| { | |
| "debug/losses": 0.02110939845442772, | |
| "debug/policy_weights": 0.03945222496986389, | |
| "debug/raw_losses": 0.6036224365234375, | |
| "epoch": 0.6366892160764027, | |
| "grad_norm": 1.1130648846777194, | |
| "learning_rate": 1.7539457311884675e-07, | |
| "logits/chosen": -1.9825928211212158, | |
| "logits/rejected": -1.9254627227783203, | |
| "logps/chosen": -373.7814025878906, | |
| "logps/rejected": -404.980712890625, | |
| "loss": 0.023, | |
| "rewards/accuracies": 0.668749988079071, | |
| "rewards/chosen": -2.1609199047088623, | |
| "rewards/margins": 0.4238300919532776, | |
| "rewards/rejected": -2.584749698638916, | |
| "step": 800 | |
| }, | |
| { | |
| "epoch": 0.6366892160764027, | |
| "eval_debug/losses": 0.02278236113488674, | |
| "eval_debug/policy_weights": 0.04001852124929428, | |
| "eval_debug/raw_losses": 0.5787393450737, | |
| "eval_logits/chosen": -1.9514880180358887, | |
| "eval_logits/rejected": -1.9267523288726807, | |
| "eval_logps/chosen": -350.94134521484375, | |
| "eval_logps/rejected": -407.4722595214844, | |
| "eval_loss": 0.02435474470257759, | |
| "eval_rewards/accuracies": 0.6884328126907349, | |
| "eval_rewards/chosen": -2.0669784545898438, | |
| "eval_rewards/margins": 0.5007539987564087, | |
| "eval_rewards/rejected": -2.567732334136963, | |
| "eval_runtime": 153.2113, | |
| "eval_samples_per_second": 55.818, | |
| "eval_steps_per_second": 0.875, | |
| "step": 800 | |
| }, | |
| { | |
| "debug/losses": 0.031348057091236115, | |
| "debug/policy_weights": 0.05042605847120285, | |
| "debug/raw_losses": 0.617703914642334, | |
| "epoch": 0.6446478312773577, | |
| "grad_norm": 1.2829017385682178, | |
| "learning_rate": 1.687905344471226e-07, | |
| "logits/chosen": -1.9726717472076416, | |
| "logits/rejected": -1.9506819248199463, | |
| "logps/chosen": -356.71759033203125, | |
| "logps/rejected": -399.9237060546875, | |
| "loss": 0.0275, | |
| "rewards/accuracies": 0.625, | |
| "rewards/chosen": -1.9306268692016602, | |
| "rewards/margins": 0.41608279943466187, | |
| "rewards/rejected": -2.346709966659546, | |
| "step": 810 | |
| }, | |
| { | |
| "debug/losses": 0.031708650290966034, | |
| "debug/policy_weights": 0.04735485464334488, | |
| "debug/raw_losses": 0.6054385900497437, | |
| "epoch": 0.6526064464783128, | |
| "grad_norm": 1.2622583124342848, | |
| "learning_rate": 1.6224926135406693e-07, | |
| "logits/chosen": -1.9768617153167725, | |
| "logits/rejected": -1.9380643367767334, | |
| "logps/chosen": -340.0035095214844, | |
| "logps/rejected": -370.344970703125, | |
| "loss": 0.0259, | |
| "rewards/accuracies": 0.6499999761581421, | |
| "rewards/chosen": -1.954310655593872, | |
| "rewards/margins": 0.4057098925113678, | |
| "rewards/rejected": -2.360020399093628, | |
| "step": 820 | |
| }, | |
| { | |
| "debug/losses": 0.028013870120048523, | |
| "debug/policy_weights": 0.04989269748330116, | |
| "debug/raw_losses": 0.5807904005050659, | |
| "epoch": 0.6605650616792678, | |
| "grad_norm": 1.2623708525947415, | |
| "learning_rate": 1.557758094916053e-07, | |
| "logits/chosen": -1.9997406005859375, | |
| "logits/rejected": -1.9534574747085571, | |
| "logps/chosen": -351.2829895019531, | |
| "logps/rejected": -399.71087646484375, | |
| "loss": 0.0271, | |
| "rewards/accuracies": 0.706250011920929, | |
| "rewards/chosen": -1.9434627294540405, | |
| "rewards/margins": 0.4895332455635071, | |
| "rewards/rejected": -2.4329960346221924, | |
| "step": 830 | |
| }, | |
| { | |
| "debug/losses": 0.031273625791072845, | |
| "debug/policy_weights": 0.05692868307232857, | |
| "debug/raw_losses": 0.5609691739082336, | |
| "epoch": 0.6685236768802229, | |
| "grad_norm": 1.0866287225682587, | |
| "learning_rate": 1.4937518209365108e-07, | |
| "logits/chosen": -2.0409183502197266, | |
| "logits/rejected": -1.9729896783828735, | |
| "logps/chosen": -362.9765319824219, | |
| "logps/rejected": -389.9468994140625, | |
| "loss": 0.0327, | |
| "rewards/accuracies": 0.7250000238418579, | |
| "rewards/chosen": -1.8453567028045654, | |
| "rewards/margins": 0.5211337804794312, | |
| "rewards/rejected": -2.366490602493286, | |
| "step": 840 | |
| }, | |
| { | |
| "debug/losses": 0.029767373576760292, | |
| "debug/policy_weights": 0.052564360201358795, | |
| "debug/raw_losses": 0.6102725863456726, | |
| "epoch": 0.6764822920811778, | |
| "grad_norm": 1.394943527609451, | |
| "learning_rate": 1.4305232610918045e-07, | |
| "logits/chosen": -2.0147321224212646, | |
| "logits/rejected": -1.9801139831542969, | |
| "logps/chosen": -339.0684509277344, | |
| "logps/rejected": -375.1840515136719, | |
| "loss": 0.0307, | |
| "rewards/accuracies": 0.668749988079071, | |
| "rewards/chosen": -1.9301865100860596, | |
| "rewards/margins": 0.3946223855018616, | |
| "rewards/rejected": -2.3248085975646973, | |
| "step": 850 | |
| }, | |
| { | |
| "debug/losses": 0.03315902501344681, | |
| "debug/policy_weights": 0.05502952262759209, | |
| "debug/raw_losses": 0.5980508923530579, | |
| "epoch": 0.6844409072821329, | |
| "grad_norm": 1.1275097393699727, | |
| "learning_rate": 1.3681212837880977e-07, | |
| "logits/chosen": -2.002835512161255, | |
| "logits/rejected": -1.995160698890686, | |
| "logps/chosen": -307.8199157714844, | |
| "logps/rejected": -369.13763427734375, | |
| "loss": 0.0318, | |
| "rewards/accuracies": 0.675000011920929, | |
| "rewards/chosen": -1.7375946044921875, | |
| "rewards/margins": 0.43073099851608276, | |
| "rewards/rejected": -2.168325901031494, | |
| "step": 860 | |
| }, | |
| { | |
| "debug/losses": 0.029089778661727905, | |
| "debug/policy_weights": 0.050317078828811646, | |
| "debug/raw_losses": 0.5906243324279785, | |
| "epoch": 0.6923995224830879, | |
| "grad_norm": 1.4878858135274058, | |
| "learning_rate": 1.3065941185782977e-07, | |
| "logits/chosen": -1.9642223119735718, | |
| "logits/rejected": -1.927920937538147, | |
| "logps/chosen": -334.91162109375, | |
| "logps/rejected": -361.5536193847656, | |
| "loss": 0.0306, | |
| "rewards/accuracies": 0.675000011920929, | |
| "rewards/chosen": -1.8268492221832275, | |
| "rewards/margins": 0.42832955718040466, | |
| "rewards/rejected": -2.255178928375244, | |
| "step": 870 | |
| }, | |
| { | |
| "debug/losses": 0.029998168349266052, | |
| "debug/policy_weights": 0.0653611272573471, | |
| "debug/raw_losses": 0.47929373383522034, | |
| "epoch": 0.700358137684043, | |
| "grad_norm": 1.29187319176411, | |
| "learning_rate": 1.2459893188861613e-07, | |
| "logits/chosen": -2.0278587341308594, | |
| "logits/rejected": -1.9883801937103271, | |
| "logps/chosen": -307.253662109375, | |
| "logps/rejected": -392.42962646484375, | |
| "loss": 0.0333, | |
| "rewards/accuracies": 0.78125, | |
| "rewards/chosen": -1.5876758098602295, | |
| "rewards/margins": 0.7120193839073181, | |
| "rewards/rejected": -2.2996952533721924, | |
| "step": 880 | |
| }, | |
| { | |
| "debug/losses": 0.031164586544036865, | |
| "debug/policy_weights": 0.05676507204771042, | |
| "debug/raw_losses": 0.5348513722419739, | |
| "epoch": 0.708316752884998, | |
| "grad_norm": 1.2972584564770755, | |
| "learning_rate": 1.1863537252529548e-07, | |
| "logits/chosen": -1.9931986331939697, | |
| "logits/rejected": -1.9309186935424805, | |
| "logps/chosen": -337.57659912109375, | |
| "logps/rejected": -380.300048828125, | |
| "loss": 0.0307, | |
| "rewards/accuracies": 0.699999988079071, | |
| "rewards/chosen": -1.793096899986267, | |
| "rewards/margins": 0.5044206380844116, | |
| "rewards/rejected": -2.297517776489258, | |
| "step": 890 | |
| }, | |
| { | |
| "debug/losses": 0.0308552123606205, | |
| "debug/policy_weights": 0.05725027993321419, | |
| "debug/raw_losses": 0.5641220808029175, | |
| "epoch": 0.716275368085953, | |
| "grad_norm": 1.406582703436064, | |
| "learning_rate": 1.1277334291351145e-07, | |
| "logits/chosen": -1.92514169216156, | |
| "logits/rejected": -1.8894935846328735, | |
| "logps/chosen": -308.63641357421875, | |
| "logps/rejected": -367.8720397949219, | |
| "loss": 0.032, | |
| "rewards/accuracies": 0.6499999761581421, | |
| "rewards/chosen": -1.7129472494125366, | |
| "rewards/margins": 0.5332741737365723, | |
| "rewards/rejected": -2.2462215423583984, | |
| "step": 900 | |
| }, | |
| { | |
| "epoch": 0.716275368085953, | |
| "eval_debug/losses": 0.03159477189183235, | |
| "eval_debug/policy_weights": 0.055877406150102615, | |
| "eval_debug/raw_losses": 0.5719749927520752, | |
| "eval_logits/chosen": -1.952079176902771, | |
| "eval_logits/rejected": -1.9262397289276123, | |
| "eval_logps/chosen": -318.9172668457031, | |
| "eval_logps/rejected": -378.01251220703125, | |
| "eval_loss": 0.03346230462193489, | |
| "eval_rewards/accuracies": 0.6847015023231506, | |
| "eval_rewards/chosen": -1.7467377185821533, | |
| "eval_rewards/margins": 0.5263976454734802, | |
| "eval_rewards/rejected": -2.2731354236602783, | |
| "eval_runtime": 153.3496, | |
| "eval_samples_per_second": 55.768, | |
| "eval_steps_per_second": 0.874, | |
| "step": 900 | |
| }, | |
| { | |
| "debug/losses": 0.031236503273248672, | |
| "debug/policy_weights": 0.06273458153009415, | |
| "debug/raw_losses": 0.5026707053184509, | |
| "epoch": 0.724233983286908, | |
| "grad_norm": 1.4167921520949656, | |
| "learning_rate": 1.0701737372808431e-07, | |
| "logits/chosen": -1.9520496129989624, | |
| "logits/rejected": -1.926009178161621, | |
| "logps/chosen": -301.79803466796875, | |
| "logps/rejected": -384.8517150878906, | |
| "loss": 0.0301, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": -1.5805127620697021, | |
| "rewards/margins": 0.7013577818870544, | |
| "rewards/rejected": -2.2818703651428223, | |
| "step": 910 | |
| }, | |
| { | |
| "debug/losses": 0.028870219364762306, | |
| "debug/policy_weights": 0.05118779093027115, | |
| "debug/raw_losses": 0.6152342557907104, | |
| "epoch": 0.7321925984878631, | |
| "grad_norm": 1.3852811285813345, | |
| "learning_rate": 1.0137191367132078e-07, | |
| "logits/chosen": -1.9522676467895508, | |
| "logits/rejected": -1.9203464984893799, | |
| "logps/chosen": -363.1385192871094, | |
| "logps/rejected": -410.9398498535156, | |
| "loss": 0.0272, | |
| "rewards/accuracies": 0.699999988079071, | |
| "rewards/chosen": -2.070452928543091, | |
| "rewards/margins": 0.43279123306274414, | |
| "rewards/rejected": -2.503243923187256, | |
| "step": 920 | |
| }, | |
| { | |
| "debug/losses": 0.02983178198337555, | |
| "debug/policy_weights": 0.04729953408241272, | |
| "debug/raw_losses": 0.6190310716629028, | |
| "epoch": 0.7401512136888182, | |
| "grad_norm": 1.1110786407319169, | |
| "learning_rate": 9.584132603467827e-08, | |
| "logits/chosen": -1.9343547821044922, | |
| "logits/rejected": -1.8795759677886963, | |
| "logps/chosen": -377.31103515625, | |
| "logps/rejected": -408.9967346191406, | |
| "loss": 0.0268, | |
| "rewards/accuracies": 0.637499988079071, | |
| "rewards/chosen": -2.1014113426208496, | |
| "rewards/margins": 0.44571709632873535, | |
| "rewards/rejected": -2.547128438949585, | |
| "step": 930 | |
| }, | |
| { | |
| "debug/losses": 0.025911280885338783, | |
| "debug/policy_weights": 0.04492679983377457, | |
| "debug/raw_losses": 0.5681049227714539, | |
| "epoch": 0.7481098288897732, | |
| "grad_norm": 1.282258686519084, | |
| "learning_rate": 9.042988532644249e-08, | |
| "logits/chosen": -1.9340381622314453, | |
| "logits/rejected": -1.903543472290039, | |
| "logps/chosen": -344.40826416015625, | |
| "logps/rejected": -411.30517578125, | |
| "loss": 0.0266, | |
| "rewards/accuracies": 0.699999988079071, | |
| "rewards/chosen": -2.0105528831481934, | |
| "rewards/margins": 0.5551122426986694, | |
| "rewards/rejected": -2.5656652450561523, | |
| "step": 940 | |
| }, | |
| { | |
| "debug/losses": 0.029090652242302895, | |
| "debug/policy_weights": 0.04689662158489227, | |
| "debug/raw_losses": 0.5897936820983887, | |
| "epoch": 0.7560684440907283, | |
| "grad_norm": 1.1204653206310022, | |
| "learning_rate": 8.514177396802428e-08, | |
| "logits/chosen": -1.953285574913025, | |
| "logits/rejected": -1.9266679286956787, | |
| "logps/chosen": -352.0096130371094, | |
| "logps/rejected": -408.1578063964844, | |
| "loss": 0.0257, | |
| "rewards/accuracies": 0.699999988079071, | |
| "rewards/chosen": -2.1213886737823486, | |
| "rewards/margins": 0.49127835035324097, | |
| "rewards/rejected": -2.612666606903076, | |
| "step": 950 | |
| }, | |
| { | |
| "debug/losses": 0.026548391208052635, | |
| "debug/policy_weights": 0.04522048681974411, | |
| "debug/raw_losses": 0.5894621014595032, | |
| "epoch": 0.7640270592916832, | |
| "grad_norm": 0.898387217227339, | |
| "learning_rate": 7.998107906142839e-08, | |
| "logits/chosen": -1.9560728073120117, | |
| "logits/rejected": -1.9274494647979736, | |
| "logps/chosen": -343.21002197265625, | |
| "logps/rejected": -379.6694030761719, | |
| "loss": 0.0259, | |
| "rewards/accuracies": 0.6499999761581421, | |
| "rewards/chosen": -2.0018842220306396, | |
| "rewards/margins": 0.4287974238395691, | |
| "rewards/rejected": -2.4306817054748535, | |
| "step": 960 | |
| }, | |
| { | |
| "debug/losses": 0.02163025364279747, | |
| "debug/policy_weights": 0.04079737886786461, | |
| "debug/raw_losses": 0.5705414414405823, | |
| "epoch": 0.7719856744926383, | |
| "grad_norm": 1.3955639157776591, | |
| "learning_rate": 7.495178923039396e-08, | |
| "logits/chosen": -1.9478362798690796, | |
| "logits/rejected": -1.9617118835449219, | |
| "logps/chosen": -331.72784423828125, | |
| "logps/rejected": -410.7064514160156, | |
| "loss": 0.0264, | |
| "rewards/accuracies": 0.6499999761581421, | |
| "rewards/chosen": -2.030539035797119, | |
| "rewards/margins": 0.5192916989326477, | |
| "rewards/rejected": -2.549830913543701, | |
| "step": 970 | |
| }, | |
| { | |
| "debug/losses": 0.023149430751800537, | |
| "debug/policy_weights": 0.041884638369083405, | |
| "debug/raw_losses": 0.5447368025779724, | |
| "epoch": 0.7799442896935933, | |
| "grad_norm": 1.5585746856477063, | |
| "learning_rate": 7.005779153764682e-08, | |
| "logits/chosen": -1.9719524383544922, | |
| "logits/rejected": -1.9254512786865234, | |
| "logps/chosen": -337.332275390625, | |
| "logps/rejected": -387.82318115234375, | |
| "loss": 0.028, | |
| "rewards/accuracies": 0.706250011920929, | |
| "rewards/chosen": -1.9454883337020874, | |
| "rewards/margins": 0.5578633546829224, | |
| "rewards/rejected": -2.5033516883850098, | |
| "step": 980 | |
| }, | |
| { | |
| "debug/losses": 0.027449458837509155, | |
| "debug/policy_weights": 0.04992467164993286, | |
| "debug/raw_losses": 0.551071286201477, | |
| "epoch": 0.7879029048945484, | |
| "grad_norm": 1.6695068228207648, | |
| "learning_rate": 6.530286848064698e-08, | |
| "logits/chosen": -1.9756189584732056, | |
| "logits/rejected": -1.950194001197815, | |
| "logps/chosen": -336.4566955566406, | |
| "logps/rejected": -397.8472595214844, | |
| "loss": 0.0264, | |
| "rewards/accuracies": 0.6937500238418579, | |
| "rewards/chosen": -1.8330726623535156, | |
| "rewards/margins": 0.5761424899101257, | |
| "rewards/rejected": -2.4092154502868652, | |
| "step": 990 | |
| }, | |
| { | |
| "debug/losses": 0.026408571749925613, | |
| "debug/policy_weights": 0.04767809808254242, | |
| "debug/raw_losses": 0.5441979169845581, | |
| "epoch": 0.7958615200955034, | |
| "grad_norm": 1.6016926469447665, | |
| "learning_rate": 6.069069506815325e-08, | |
| "logits/chosen": -1.964962363243103, | |
| "logits/rejected": -1.9171581268310547, | |
| "logps/chosen": -334.23931884765625, | |
| "logps/rejected": -392.36944580078125, | |
| "loss": 0.0294, | |
| "rewards/accuracies": 0.731249988079071, | |
| "rewards/chosen": -1.8932774066925049, | |
| "rewards/margins": 0.6260205507278442, | |
| "rewards/rejected": -2.5192978382110596, | |
| "step": 1000 | |
| }, | |
| { | |
| "epoch": 0.7958615200955034, | |
| "eval_debug/losses": 0.02711501345038414, | |
| "eval_debug/policy_weights": 0.048435505479574203, | |
| "eval_debug/raw_losses": 0.5694997906684875, | |
| "eval_logits/chosen": -1.9579750299453735, | |
| "eval_logits/rejected": -1.9318426847457886, | |
| "eval_logps/chosen": -338.3062438964844, | |
| "eval_logps/rejected": -398.16033935546875, | |
| "eval_loss": 0.02892610803246498, | |
| "eval_rewards/accuracies": 0.6865671873092651, | |
| "eval_rewards/chosen": -1.9406273365020752, | |
| "eval_rewards/margins": 0.5339860916137695, | |
| "eval_rewards/rejected": -2.474613666534424, | |
| "eval_runtime": 153.1415, | |
| "eval_samples_per_second": 55.844, | |
| "eval_steps_per_second": 0.875, | |
| "step": 1000 | |
| }, | |
| { | |
| "debug/losses": 0.03183472901582718, | |
| "debug/policy_weights": 0.049464497715234756, | |
| "debug/raw_losses": 0.6195154786109924, | |
| "epoch": 0.8038201352964585, | |
| "grad_norm": 1.4391778362577514, | |
| "learning_rate": 5.6224835979863714e-08, | |
| "logits/chosen": -1.9442205429077148, | |
| "logits/rejected": -1.9024133682250977, | |
| "logps/chosen": -348.94781494140625, | |
| "logps/rejected": -381.45281982421875, | |
| "loss": 0.0284, | |
| "rewards/accuracies": 0.6187499761581421, | |
| "rewards/chosen": -1.9863436222076416, | |
| "rewards/margins": 0.40313720703125, | |
| "rewards/rejected": -2.3894805908203125, | |
| "step": 1010 | |
| }, | |
| { | |
| "debug/losses": 0.022611474618315697, | |
| "debug/policy_weights": 0.044095687568187714, | |
| "debug/raw_losses": 0.5569943785667419, | |
| "epoch": 0.8117787504974134, | |
| "grad_norm": 1.8837439228048556, | |
| "learning_rate": 5.190874281132851e-08, | |
| "logits/chosen": -1.944157600402832, | |
| "logits/rejected": -1.92160165309906, | |
| "logps/chosen": -325.661865234375, | |
| "logps/rejected": -386.25885009765625, | |
| "loss": 0.0285, | |
| "rewards/accuracies": 0.71875, | |
| "rewards/chosen": -1.8595483303070068, | |
| "rewards/margins": 0.5777700543403625, | |
| "rewards/rejected": -2.4373183250427246, | |
| "step": 1020 | |
| }, | |
| { | |
| "debug/losses": 0.0247525442391634, | |
| "debug/policy_weights": 0.045843105763196945, | |
| "debug/raw_losses": 0.5282943844795227, | |
| "epoch": 0.8197373656983685, | |
| "grad_norm": 1.3079251608813307, | |
| "learning_rate": 4.774575140626316e-08, | |
| "logits/chosen": -1.9093300104141235, | |
| "logits/rejected": -1.858944296836853, | |
| "logps/chosen": -322.7106018066406, | |
| "logps/rejected": -379.23748779296875, | |
| "loss": 0.0288, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": -1.826472282409668, | |
| "rewards/margins": 0.6417751312255859, | |
| "rewards/rejected": -2.468247413635254, | |
| "step": 1030 | |
| }, | |
| { | |
| "debug/losses": 0.029871661216020584, | |
| "debug/policy_weights": 0.05007849261164665, | |
| "debug/raw_losses": 0.6156904101371765, | |
| "epoch": 0.8276959808993235, | |
| "grad_norm": 1.4623406105385353, | |
| "learning_rate": 4.373907927832513e-08, | |
| "logits/chosen": -1.9410374164581299, | |
| "logits/rejected": -1.9204730987548828, | |
| "logps/chosen": -313.7291564941406, | |
| "logps/rejected": -367.6692199707031, | |
| "loss": 0.0313, | |
| "rewards/accuracies": 0.65625, | |
| "rewards/chosen": -1.8602615594863892, | |
| "rewards/margins": 0.42712441086769104, | |
| "rewards/rejected": -2.287385940551758, | |
| "step": 1040 | |
| }, | |
| { | |
| "debug/losses": 0.029429305344820023, | |
| "debug/policy_weights": 0.054835814982652664, | |
| "debug/raw_losses": 0.5309990048408508, | |
| "epoch": 0.8356545961002786, | |
| "grad_norm": 1.3217568428330395, | |
| "learning_rate": 3.9891823124345665e-08, | |
| "logits/chosen": -1.975529670715332, | |
| "logits/rejected": -1.9376213550567627, | |
| "logps/chosen": -324.4565124511719, | |
| "logps/rejected": -385.99835205078125, | |
| "loss": 0.032, | |
| "rewards/accuracies": 0.71875, | |
| "rewards/chosen": -1.7343988418579102, | |
| "rewards/margins": 0.6463450193405151, | |
| "rewards/rejected": -2.3807437419891357, | |
| "step": 1050 | |
| }, | |
| { | |
| "debug/losses": 0.026505377143621445, | |
| "debug/policy_weights": 0.050394363701343536, | |
| "debug/raw_losses": 0.5719414949417114, | |
| "epoch": 0.8436132113012336, | |
| "grad_norm": 1.3080390048525132, | |
| "learning_rate": 3.620695643093924e-08, | |
| "logits/chosen": -1.9717638492584229, | |
| "logits/rejected": -1.9583126306533813, | |
| "logps/chosen": -307.21038818359375, | |
| "logps/rejected": -371.9794616699219, | |
| "loss": 0.0294, | |
| "rewards/accuracies": 0.6875, | |
| "rewards/chosen": -1.7042350769042969, | |
| "rewards/margins": 0.5117149353027344, | |
| "rewards/rejected": -2.2159502506256104, | |
| "step": 1060 | |
| }, | |
| { | |
| "debug/losses": 0.03151816874742508, | |
| "debug/policy_weights": 0.051329899579286575, | |
| "debug/raw_losses": 0.5482734441757202, | |
| "epoch": 0.8515718265021887, | |
| "grad_norm": 1.9249185694718636, | |
| "learning_rate": 3.268732717634032e-08, | |
| "logits/chosen": -1.942237138748169, | |
| "logits/rejected": -1.908158540725708, | |
| "logps/chosen": -311.0814514160156, | |
| "logps/rejected": -363.3550720214844, | |
| "loss": 0.0315, | |
| "rewards/accuracies": 0.71875, | |
| "rewards/chosen": -1.746870756149292, | |
| "rewards/margins": 0.545746386051178, | |
| "rewards/rejected": -2.292617082595825, | |
| "step": 1070 | |
| }, | |
| { | |
| "debug/losses": 0.0361117348074913, | |
| "debug/policy_weights": 0.056406814604997635, | |
| "debug/raw_losses": 0.6171997785568237, | |
| "epoch": 0.8595304417031436, | |
| "grad_norm": 1.2978569483513427, | |
| "learning_rate": 2.9335655629243645e-08, | |
| "logits/chosen": -1.9778077602386475, | |
| "logits/rejected": -1.9576594829559326, | |
| "logps/chosen": -338.1204833984375, | |
| "logps/rejected": -390.57769775390625, | |
| "loss": 0.032, | |
| "rewards/accuracies": 0.6312500238418579, | |
| "rewards/chosen": -1.8773784637451172, | |
| "rewards/margins": 0.41199570894241333, | |
| "rewards/rejected": -2.2893738746643066, | |
| "step": 1080 | |
| }, | |
| { | |
| "debug/losses": 0.02704630419611931, | |
| "debug/policy_weights": 0.04527381435036659, | |
| "debug/raw_losses": 0.6039454340934753, | |
| "epoch": 0.8674890569040987, | |
| "grad_norm": 1.4432056512216735, | |
| "learning_rate": 2.6154532246349476e-08, | |
| "logits/chosen": -1.9391145706176758, | |
| "logits/rejected": -1.882846474647522, | |
| "logps/chosen": -329.9732360839844, | |
| "logps/rejected": -354.43182373046875, | |
| "loss": 0.0324, | |
| "rewards/accuracies": 0.643750011920929, | |
| "rewards/chosen": -1.8840316534042358, | |
| "rewards/margins": 0.42414146661758423, | |
| "rewards/rejected": -2.3081729412078857, | |
| "step": 1090 | |
| }, | |
| { | |
| "debug/losses": 0.03607811778783798, | |
| "debug/policy_weights": 0.060448456555604935, | |
| "debug/raw_losses": 0.6365524530410767, | |
| "epoch": 0.8754476721050537, | |
| "grad_norm": 2.2478985377156913, | |
| "learning_rate": 2.31464156702382e-08, | |
| "logits/chosen": -1.9797563552856445, | |
| "logits/rejected": -1.9243662357330322, | |
| "logps/chosen": -340.63897705078125, | |
| "logps/rejected": -371.7919006347656, | |
| "loss": 0.0308, | |
| "rewards/accuracies": 0.606249988079071, | |
| "rewards/chosen": -1.8404861688613892, | |
| "rewards/margins": 0.3901959955692291, | |
| "rewards/rejected": -2.230681896209717, | |
| "step": 1100 | |
| }, | |
| { | |
| "epoch": 0.8754476721050537, | |
| "eval_debug/losses": 0.029148757457733154, | |
| "eval_debug/policy_weights": 0.052018992602825165, | |
| "eval_debug/raw_losses": 0.5656781196594238, | |
| "eval_logits/chosen": -1.9813780784606934, | |
| "eval_logits/rejected": -1.9553614854812622, | |
| "eval_logps/chosen": -325.3560485839844, | |
| "eval_logps/rejected": -384.33758544921875, | |
| "eval_loss": 0.031092733144760132, | |
| "eval_rewards/accuracies": 0.700559675693512, | |
| "eval_rewards/chosen": -1.81112539768219, | |
| "eval_rewards/margins": 0.5252605676651001, | |
| "eval_rewards/rejected": -2.336386203765869, | |
| "eval_runtime": 153.3772, | |
| "eval_samples_per_second": 55.758, | |
| "eval_steps_per_second": 0.874, | |
| "step": 1100 | |
| }, | |
| { | |
| "debug/losses": 0.028289441019296646, | |
| "debug/policy_weights": 0.050502192229032516, | |
| "debug/raw_losses": 0.5873227119445801, | |
| "epoch": 0.8834062873060088, | |
| "grad_norm": 1.2493044367365553, | |
| "learning_rate": 2.031363082912252e-08, | |
| "logits/chosen": -1.9731247425079346, | |
| "logits/rejected": -1.9583030939102173, | |
| "logps/chosen": -312.7198181152344, | |
| "logps/rejected": -365.84539794921875, | |
| "loss": 0.0313, | |
| "rewards/accuracies": 0.65625, | |
| "rewards/chosen": -1.811852216720581, | |
| "rewards/margins": 0.43998369574546814, | |
| "rewards/rejected": -2.2518362998962402, | |
| "step": 1110 | |
| }, | |
| { | |
| "debug/losses": 0.028473522514104843, | |
| "debug/policy_weights": 0.049161095172166824, | |
| "debug/raw_losses": 0.613974392414093, | |
| "epoch": 0.8913649025069638, | |
| "grad_norm": 1.5694190986130379, | |
| "learning_rate": 1.7658367139945228e-08, | |
| "logits/chosen": -1.9810594320297241, | |
| "logits/rejected": -1.9477195739746094, | |
| "logps/chosen": -323.91619873046875, | |
| "logps/rejected": -368.2818298339844, | |
| "loss": 0.0294, | |
| "rewards/accuracies": 0.675000011920929, | |
| "rewards/chosen": -1.7960774898529053, | |
| "rewards/margins": 0.45412111282348633, | |
| "rewards/rejected": -2.2501983642578125, | |
| "step": 1120 | |
| }, | |
| { | |
| "debug/losses": 0.03077780269086361, | |
| "debug/policy_weights": 0.055907707661390305, | |
| "debug/raw_losses": 0.5727918148040771, | |
| "epoch": 0.8993235177079189, | |
| "grad_norm": 1.736668937190508, | |
| "learning_rate": 1.5182676816211632e-08, | |
| "logits/chosen": -1.9711143970489502, | |
| "logits/rejected": -1.9426500797271729, | |
| "logps/chosen": -326.6500549316406, | |
| "logps/rejected": -389.47027587890625, | |
| "loss": 0.0322, | |
| "rewards/accuracies": 0.675000011920929, | |
| "rewards/chosen": -1.8423681259155273, | |
| "rewards/margins": 0.4901112914085388, | |
| "rewards/rejected": -2.332479476928711, | |
| "step": 1130 | |
| }, | |
| { | |
| "debug/losses": 0.04067676141858101, | |
| "debug/policy_weights": 0.06354663521051407, | |
| "debug/raw_losses": 0.6167309880256653, | |
| "epoch": 0.9072821329088738, | |
| "grad_norm": 1.311295756466762, | |
| "learning_rate": 1.2888473281864597e-08, | |
| "logits/chosen": -2.0066215991973877, | |
| "logits/rejected": -1.978035569190979, | |
| "logps/chosen": -334.54376220703125, | |
| "logps/rejected": -368.7451171875, | |
| "loss": 0.0321, | |
| "rewards/accuracies": 0.6499999761581421, | |
| "rewards/chosen": -1.7672863006591797, | |
| "rewards/margins": 0.3908054232597351, | |
| "rewards/rejected": -2.1580917835235596, | |
| "step": 1140 | |
| }, | |
| { | |
| "debug/losses": 0.029732922092080116, | |
| "debug/policy_weights": 0.05216806009411812, | |
| "debug/raw_losses": 0.5487850904464722, | |
| "epoch": 0.9152407481098289, | |
| "grad_norm": 1.2474371694735222, | |
| "learning_rate": 1.0777529692427679e-08, | |
| "logits/chosen": -1.9626529216766357, | |
| "logits/rejected": -1.924966812133789, | |
| "logps/chosen": -318.5897521972656, | |
| "logps/rejected": -362.83251953125, | |
| "loss": 0.0311, | |
| "rewards/accuracies": 0.7437499761581421, | |
| "rewards/chosen": -1.6781198978424072, | |
| "rewards/margins": 0.5509058833122253, | |
| "rewards/rejected": -2.2290260791778564, | |
| "step": 1150 | |
| }, | |
| { | |
| "debug/losses": 0.034698087722063065, | |
| "debug/policy_weights": 0.054052066057920456, | |
| "debug/raw_losses": 0.599198579788208, | |
| "epoch": 0.9231993633107839, | |
| "grad_norm": 1.4791914719616432, | |
| "learning_rate": 8.851477564560061e-09, | |
| "logits/chosen": -1.9579505920410156, | |
| "logits/rejected": -1.9253816604614258, | |
| "logps/chosen": -311.4327392578125, | |
| "logps/rejected": -374.3013610839844, | |
| "loss": 0.0312, | |
| "rewards/accuracies": 0.6812499761581421, | |
| "rewards/chosen": -1.692578911781311, | |
| "rewards/margins": 0.5192922353744507, | |
| "rewards/rejected": -2.2118711471557617, | |
| "step": 1160 | |
| }, | |
| { | |
| "debug/losses": 0.030673867091536522, | |
| "debug/policy_weights": 0.05762176960706711, | |
| "debug/raw_losses": 0.5920447111129761, | |
| "epoch": 0.931157978511739, | |
| "grad_norm": 1.2297627601023153, | |
| "learning_rate": 7.111805515081531e-09, | |
| "logits/chosen": -1.9873937368392944, | |
| "logits/rejected": -1.928577184677124, | |
| "logps/chosen": -349.22552490234375, | |
| "logps/rejected": -393.4672546386719, | |
| "loss": 0.0309, | |
| "rewards/accuracies": 0.699999988079071, | |
| "rewards/chosen": -1.9474347829818726, | |
| "rewards/margins": 0.4757717251777649, | |
| "rewards/rejected": -2.423206329345703, | |
| "step": 1170 | |
| }, | |
| { | |
| "debug/losses": 0.03575451672077179, | |
| "debug/policy_weights": 0.0605660155415535, | |
| "debug/raw_losses": 0.5854865312576294, | |
| "epoch": 0.939116593712694, | |
| "grad_norm": 1.1864649900689144, | |
| "learning_rate": 5.559858110443016e-09, | |
| "logits/chosen": -2.000704288482666, | |
| "logits/rejected": -1.9662271738052368, | |
| "logps/chosen": -329.9343566894531, | |
| "logps/rejected": -382.48822021484375, | |
| "loss": 0.0302, | |
| "rewards/accuracies": 0.731249988079071, | |
| "rewards/chosen": -1.740262746810913, | |
| "rewards/margins": 0.504480242729187, | |
| "rewards/rejected": -2.2447433471679688, | |
| "step": 1180 | |
| }, | |
| { | |
| "debug/losses": 0.024325475096702576, | |
| "debug/policy_weights": 0.04755675792694092, | |
| "debug/raw_losses": 0.549831211566925, | |
| "epoch": 0.947075208913649, | |
| "grad_norm": 1.4675887378312884, | |
| "learning_rate": 4.196834827531276e-09, | |
| "logits/chosen": -1.9613635540008545, | |
| "logits/rejected": -1.926835060119629, | |
| "logps/chosen": -328.3395690917969, | |
| "logps/rejected": -393.16650390625, | |
| "loss": 0.0289, | |
| "rewards/accuracies": 0.6875, | |
| "rewards/chosen": -1.8523941040039062, | |
| "rewards/margins": 0.6523604393005371, | |
| "rewards/rejected": -2.5047545433044434, | |
| "step": 1190 | |
| }, | |
| { | |
| "debug/losses": 0.024997171014547348, | |
| "debug/policy_weights": 0.04472526162862778, | |
| "debug/raw_losses": 0.5522318482398987, | |
| "epoch": 0.955033824114604, | |
| "grad_norm": 1.2840605704052697, | |
| "learning_rate": 3.023789126611137e-09, | |
| "logits/chosen": -1.9729950428009033, | |
| "logits/rejected": -1.9184396266937256, | |
| "logps/chosen": -324.0898742675781, | |
| "logps/rejected": -372.49151611328125, | |
| "loss": 0.0303, | |
| "rewards/accuracies": 0.6625000238418579, | |
| "rewards/chosen": -1.7675983905792236, | |
| "rewards/margins": 0.5561206936836243, | |
| "rewards/rejected": -2.323719024658203, | |
| "step": 1200 | |
| }, | |
| { | |
| "epoch": 0.955033824114604, | |
| "eval_debug/losses": 0.029483934864401817, | |
| "eval_debug/policy_weights": 0.05266140401363373, | |
| "eval_debug/raw_losses": 0.5652690529823303, | |
| "eval_logits/chosen": -1.9727920293807983, | |
| "eval_logits/rejected": -1.946227788925171, | |
| "eval_logps/chosen": -324.4056091308594, | |
| "eval_logps/rejected": -384.5557556152344, | |
| "eval_loss": 0.03144846484065056, | |
| "eval_rewards/accuracies": 0.6996268630027771, | |
| "eval_rewards/chosen": -1.8016209602355957, | |
| "eval_rewards/margins": 0.536946713924408, | |
| "eval_rewards/rejected": -2.3385674953460693, | |
| "eval_runtime": 153.1595, | |
| "eval_samples_per_second": 55.837, | |
| "eval_steps_per_second": 0.875, | |
| "step": 1200 | |
| }, | |
| { | |
| "debug/losses": 0.027663961052894592, | |
| "debug/policy_weights": 0.04848761484026909, | |
| "debug/raw_losses": 0.5848130583763123, | |
| "epoch": 0.9629924393155591, | |
| "grad_norm": 1.2701407512231324, | |
| "learning_rate": 2.041627637121929e-09, | |
| "logits/chosen": -1.950147032737732, | |
| "logits/rejected": -1.9268693923950195, | |
| "logps/chosen": -327.3438415527344, | |
| "logps/rejected": -394.26251220703125, | |
| "loss": 0.0305, | |
| "rewards/accuracies": 0.6625000238418579, | |
| "rewards/chosen": -1.858901023864746, | |
| "rewards/margins": 0.4966161251068115, | |
| "rewards/rejected": -2.3555169105529785, | |
| "step": 1210 | |
| }, | |
| { | |
| "debug/losses": 0.03097696043550968, | |
| "debug/policy_weights": 0.05295131728053093, | |
| "debug/raw_losses": 0.5842943787574768, | |
| "epoch": 0.9709510545165141, | |
| "grad_norm": 1.7444108884419736, | |
| "learning_rate": 1.2511094569571668e-09, | |
| "logits/chosen": -1.9334291219711304, | |
| "logits/rejected": -1.87801194190979, | |
| "logps/chosen": -323.7477722167969, | |
| "logps/rejected": -347.8150634765625, | |
| "loss": 0.0313, | |
| "rewards/accuracies": 0.668749988079071, | |
| "rewards/chosen": -1.7532447576522827, | |
| "rewards/margins": 0.47265228629112244, | |
| "rewards/rejected": -2.2258970737457275, | |
| "step": 1220 | |
| }, | |
| { | |
| "debug/losses": 0.02778841182589531, | |
| "debug/policy_weights": 0.04463866725564003, | |
| "debug/raw_losses": 0.5899677276611328, | |
| "epoch": 0.9789096697174692, | |
| "grad_norm": 1.2986395101895525, | |
| "learning_rate": 6.528455657691112e-10, | |
| "logits/chosen": -1.9225807189941406, | |
| "logits/rejected": -1.9189189672470093, | |
| "logps/chosen": -331.2619934082031, | |
| "logps/rejected": -391.13671875, | |
| "loss": 0.0287, | |
| "rewards/accuracies": 0.6937500238418579, | |
| "rewards/chosen": -1.9252440929412842, | |
| "rewards/margins": 0.48622721433639526, | |
| "rewards/rejected": -2.411471128463745, | |
| "step": 1230 | |
| }, | |
| { | |
| "debug/losses": 0.032337166368961334, | |
| "debug/policy_weights": 0.051962029188871384, | |
| "debug/raw_losses": 0.5668946504592896, | |
| "epoch": 0.9868682849184242, | |
| "grad_norm": 1.4772091582386668, | |
| "learning_rate": 2.4729835275189016e-10, | |
| "logits/chosen": -1.9424164295196533, | |
| "logits/rejected": -1.9091637134552002, | |
| "logps/chosen": -328.1160583496094, | |
| "logps/rejected": -392.4992370605469, | |
| "loss": 0.0303, | |
| "rewards/accuracies": 0.675000011920929, | |
| "rewards/chosen": -1.8741003274917603, | |
| "rewards/margins": 0.5802657604217529, | |
| "rewards/rejected": -2.4543659687042236, | |
| "step": 1240 | |
| }, | |
| { | |
| "debug/losses": 0.027117431163787842, | |
| "debug/policy_weights": 0.04854784160852432, | |
| "debug/raw_losses": 0.5413271188735962, | |
| "epoch": 0.9948269001193792, | |
| "grad_norm": 1.622815424856795, | |
| "learning_rate": 3.478125926756337e-11, | |
| "logits/chosen": -1.9423835277557373, | |
| "logits/rejected": -1.9216960668563843, | |
| "logps/chosen": -327.0545349121094, | |
| "logps/rejected": -397.98040771484375, | |
| "loss": 0.0289, | |
| "rewards/accuracies": 0.71875, | |
| "rewards/chosen": -1.8533722162246704, | |
| "rewards/margins": 0.5928901433944702, | |
| "rewards/rejected": -2.4462623596191406, | |
| "step": 1250 | |
| }, | |
| { | |
| "epoch": 0.9996020692399522, | |
| "step": 1256, | |
| "total_flos": 0.0, | |
| "train_loss": 0.05419013021620595, | |
| "train_runtime": 10439.8283, | |
| "train_samples_per_second": 15.403, | |
| "train_steps_per_second": 0.12 | |
| } | |
| ], | |
| "logging_steps": 10, | |
| "max_steps": 1256, | |
| "num_input_tokens_seen": 0, | |
| "num_train_epochs": 1, | |
| "save_steps": 100, | |
| "stateful_callbacks": { | |
| "TrainerControl": { | |
| "args": { | |
| "should_epoch_stop": false, | |
| "should_evaluate": false, | |
| "should_log": false, | |
| "should_save": true, | |
| "should_training_stop": false | |
| }, | |
| "attributes": {} | |
| } | |
| }, | |
| "total_flos": 0.0, | |
| "train_batch_size": 8, | |
| "trial_name": null, | |
| "trial_params": null | |
| } | |