{ "best_metric": null, "best_model_checkpoint": null, "epoch": 1.0, "eval_steps": 500, "global_step": 1875, "is_hyper_param_search": false, "is_local_process_zero": true, "is_world_process_zero": true, "log_history": [ { "epoch": 0.0, "learning_rate": 2.6595744680851065e-08, "logits/chosen": 0.04955034703016281, "logits/rejected": 0.2081322818994522, "logps/chosen": -399.2755126953125, "logps/rejected": -356.2502136230469, "loss": 1.8388, "rewards/accuracies": 0.0, "rewards/chosen": 0.0, "rewards/margins": 0.0, "rewards/rejected": 0.0, "step": 1 }, { "epoch": 0.01, "learning_rate": 2.6595744680851066e-07, "logits/chosen": 0.12051750719547272, "logits/rejected": 0.17576032876968384, "logps/chosen": -352.8992919921875, "logps/rejected": -348.6365966796875, "loss": 2.1871, "rewards/accuracies": 0.3888888955116272, "rewards/chosen": 3.012115485034883e-05, "rewards/margins": -1.0702900908654556e-05, "rewards/rejected": 4.0824070310918614e-05, "step": 10 }, { "epoch": 0.01, "learning_rate": 5.319148936170213e-07, "logits/chosen": 0.12970615923404694, "logits/rejected": 0.25619959831237793, "logps/chosen": -330.56390380859375, "logps/rejected": -314.9967346191406, "loss": 2.1076, "rewards/accuracies": 0.48750001192092896, "rewards/chosen": -1.1562701729417313e-05, "rewards/margins": 1.4026712960912846e-05, "rewards/rejected": -2.558941559982486e-05, "step": 20 }, { "epoch": 0.02, "learning_rate": 7.97872340425532e-07, "logits/chosen": 0.0827835202217102, "logits/rejected": 0.23312167823314667, "logps/chosen": -356.8667907714844, "logps/rejected": -348.5129089355469, "loss": 2.0534, "rewards/accuracies": 0.4749999940395355, "rewards/chosen": 8.046765287872404e-05, "rewards/margins": 0.00017265856149606407, "rewards/rejected": -9.21908940654248e-05, "step": 30 }, { "epoch": 0.02, "learning_rate": 1.0638297872340427e-06, "logits/chosen": 0.15581126511096954, "logits/rejected": 0.2709190249443054, "logps/chosen": -379.3042907714844, "logps/rejected": -344.07720947265625, "loss": 2.1496, "rewards/accuracies": 0.41874998807907104, "rewards/chosen": -7.353671389864758e-05, "rewards/margins": -1.741977030178532e-05, "rewards/rejected": -5.6116947234841064e-05, "step": 40 }, { "epoch": 0.03, "learning_rate": 1.3297872340425533e-06, "logits/chosen": 0.14546141028404236, "logits/rejected": 0.1901441365480423, "logps/chosen": -344.20733642578125, "logps/rejected": -325.073974609375, "loss": 2.2903, "rewards/accuracies": 0.4749999940395355, "rewards/chosen": -2.2368712961906567e-05, "rewards/margins": 4.947663546772674e-05, "rewards/rejected": -7.184535206761211e-05, "step": 50 }, { "epoch": 0.03, "learning_rate": 1.595744680851064e-06, "logits/chosen": 0.09698285907506943, "logits/rejected": 0.1907365322113037, "logps/chosen": -347.95751953125, "logps/rejected": -366.4272155761719, "loss": 2.2621, "rewards/accuracies": 0.5062500238418579, "rewards/chosen": -0.00013693059736397117, "rewards/margins": 4.145975617575459e-05, "rewards/rejected": -0.00017839032807387412, "step": 60 }, { "epoch": 0.04, "learning_rate": 1.8617021276595745e-06, "logits/chosen": 0.09873981028795242, "logits/rejected": 0.1650334745645523, "logps/chosen": -343.1393127441406, "logps/rejected": -323.4422912597656, "loss": 1.9323, "rewards/accuracies": 0.625, "rewards/chosen": 0.00018913080566562712, "rewards/margins": 0.0006557141896337271, "rewards/rejected": -0.0004665833548642695, "step": 70 }, { "epoch": 0.04, "learning_rate": 2.1276595744680853e-06, "logits/chosen": 0.04767695814371109, "logits/rejected": 0.15447475016117096, "logps/chosen": -330.0970764160156, "logps/rejected": -345.8455810546875, "loss": 2.2122, "rewards/accuracies": 0.512499988079071, "rewards/chosen": -0.00011976109817624092, "rewards/margins": 0.00030935637187212706, "rewards/rejected": -0.0004291174409445375, "step": 80 }, { "epoch": 0.05, "learning_rate": 2.393617021276596e-06, "logits/chosen": 0.07230930775403976, "logits/rejected": 0.15430572628974915, "logps/chosen": -316.2431335449219, "logps/rejected": -347.070556640625, "loss": 2.0968, "rewards/accuracies": 0.612500011920929, "rewards/chosen": 0.00011777224426623434, "rewards/margins": 0.00068097427720204, "rewards/rejected": -0.0005632020765915513, "step": 90 }, { "epoch": 0.05, "learning_rate": 2.6595744680851065e-06, "logits/chosen": 0.16333770751953125, "logits/rejected": 0.23844099044799805, "logps/chosen": -365.7076110839844, "logps/rejected": -356.7385559082031, "loss": 2.0913, "rewards/accuracies": 0.612500011920929, "rewards/chosen": 0.00023379763297270983, "rewards/margins": 0.001117090112529695, "rewards/rejected": -0.0008832925814203918, "step": 100 }, { "epoch": 0.06, "learning_rate": 2.9255319148936174e-06, "logits/chosen": 0.15442657470703125, "logits/rejected": 0.2177698314189911, "logps/chosen": -325.3774719238281, "logps/rejected": -331.230224609375, "loss": 2.194, "rewards/accuracies": 0.6812499761581421, "rewards/chosen": 0.0007118202047422528, "rewards/margins": 0.002311364747583866, "rewards/rejected": -0.0015995446592569351, "step": 110 }, { "epoch": 0.06, "learning_rate": 3.191489361702128e-06, "logits/chosen": 0.1042538657784462, "logits/rejected": 0.22217731177806854, "logps/chosen": -385.71258544921875, "logps/rejected": -346.8166198730469, "loss": 2.1461, "rewards/accuracies": 0.675000011920929, "rewards/chosen": 0.00048388572758994997, "rewards/margins": 0.0024064378812909126, "rewards/rejected": -0.001922552241012454, "step": 120 }, { "epoch": 0.07, "learning_rate": 3.457446808510639e-06, "logits/chosen": 0.11709433794021606, "logits/rejected": 0.1892680823802948, "logps/chosen": -366.34527587890625, "logps/rejected": -335.19305419921875, "loss": 2.0647, "rewards/accuracies": 0.643750011920929, "rewards/chosen": 0.0013805570779368281, "rewards/margins": 0.0054556322284042835, "rewards/rejected": -0.004075074568390846, "step": 130 }, { "epoch": 0.07, "learning_rate": 3.723404255319149e-06, "logits/chosen": 0.08869902789592743, "logits/rejected": 0.1678289771080017, "logps/chosen": -340.19000244140625, "logps/rejected": -347.8067932128906, "loss": 2.1245, "rewards/accuracies": 0.71875, "rewards/chosen": 0.00034795800456777215, "rewards/margins": 0.007005759980529547, "rewards/rejected": -0.006657801568508148, "step": 140 }, { "epoch": 0.08, "learning_rate": 3.98936170212766e-06, "logits/chosen": 0.09667147696018219, "logits/rejected": 0.17873503267765045, "logps/chosen": -348.5378723144531, "logps/rejected": -354.625732421875, "loss": 2.0674, "rewards/accuracies": 0.6937500238418579, "rewards/chosen": -0.002576244994997978, "rewards/margins": 0.012314345687627792, "rewards/rejected": -0.014890591613948345, "step": 150 }, { "epoch": 0.09, "learning_rate": 4.255319148936171e-06, "logits/chosen": 0.11646904796361923, "logits/rejected": 0.19791707396507263, "logps/chosen": -371.9278869628906, "logps/rejected": -352.6597595214844, "loss": 2.0031, "rewards/accuracies": 0.675000011920929, "rewards/chosen": -0.017293656244874, "rewards/margins": 0.017039310187101364, "rewards/rejected": -0.034332968294620514, "step": 160 }, { "epoch": 0.09, "learning_rate": 4.521276595744681e-06, "logits/chosen": 0.0863865464925766, "logits/rejected": 0.13875712454319, "logps/chosen": -359.6827697753906, "logps/rejected": -397.03314208984375, "loss": 2.0383, "rewards/accuracies": 0.71875, "rewards/chosen": -0.04616696387529373, "rewards/margins": 0.033675678074359894, "rewards/rejected": -0.07984264194965363, "step": 170 }, { "epoch": 0.1, "learning_rate": 4.787234042553192e-06, "logits/chosen": 0.1687321960926056, "logits/rejected": 0.18632039427757263, "logps/chosen": -518.4354248046875, "logps/rejected": -581.8291625976562, "loss": 1.965, "rewards/accuracies": 0.637499988079071, "rewards/chosen": -0.18014833331108093, "rewards/margins": 0.08009239286184311, "rewards/rejected": -0.26024070382118225, "step": 180 }, { "epoch": 0.1, "learning_rate": 4.999982660399688e-06, "logits/chosen": 0.1435881406068802, "logits/rejected": 0.23430481553077698, "logps/chosen": -702.39453125, "logps/rejected": -783.1107177734375, "loss": 1.9302, "rewards/accuracies": 0.6187499761581421, "rewards/chosen": -0.3382980227470398, "rewards/margins": 0.09362809360027313, "rewards/rejected": -0.4319261610507965, "step": 190 }, { "epoch": 0.11, "learning_rate": 4.99937579964398e-06, "logits/chosen": 0.208383709192276, "logits/rejected": 0.2906993627548218, "logps/chosen": -713.2571411132812, "logps/rejected": -865.4801635742188, "loss": 1.9144, "rewards/accuracies": 0.643750011920929, "rewards/chosen": -0.37990179657936096, "rewards/margins": 0.1499054878950119, "rewards/rejected": -0.5298072695732117, "step": 200 }, { "epoch": 0.11, "learning_rate": 4.9979021993870645e-06, "logits/chosen": 0.3532663583755493, "logits/rejected": 0.40125733613967896, "logps/chosen": -827.6688232421875, "logps/rejected": -1008.0562744140625, "loss": 1.9168, "rewards/accuracies": 0.6187499761581421, "rewards/chosen": -0.48167118430137634, "rewards/margins": 0.19706857204437256, "rewards/rejected": -0.6787397861480713, "step": 210 }, { "epoch": 0.12, "learning_rate": 4.995562370647553e-06, "logits/chosen": 0.26290273666381836, "logits/rejected": 0.30596408247947693, "logps/chosen": -668.184326171875, "logps/rejected": -875.458984375, "loss": 1.9675, "rewards/accuracies": 0.65625, "rewards/chosen": -0.3495655059814453, "rewards/margins": 0.21041527390480042, "rewards/rejected": -0.5599807500839233, "step": 220 }, { "epoch": 0.12, "learning_rate": 4.992357124836838e-06, "logits/chosen": 0.20304854214191437, "logits/rejected": 0.3136758804321289, "logps/chosen": -769.69775390625, "logps/rejected": -896.6029052734375, "loss": 1.9428, "rewards/accuracies": 0.699999988079071, "rewards/chosen": -0.387773334980011, "rewards/margins": 0.15167225897312164, "rewards/rejected": -0.5394456386566162, "step": 230 }, { "epoch": 0.13, "learning_rate": 4.9882875734777044e-06, "logits/chosen": 0.1910410374403, "logits/rejected": 0.2551349699497223, "logps/chosen": -708.0855712890625, "logps/rejected": -942.3931884765625, "loss": 1.8036, "rewards/accuracies": 0.637499988079071, "rewards/chosen": -0.38504528999328613, "rewards/margins": 0.18461327254772186, "rewards/rejected": -0.5696585774421692, "step": 240 }, { "epoch": 0.13, "learning_rate": 4.983355127818882e-06, "logits/chosen": 0.21396322548389435, "logits/rejected": 0.2892169654369354, "logps/chosen": -857.2903442382812, "logps/rejected": -1077.1361083984375, "loss": 1.6775, "rewards/accuracies": 0.6499999761581421, "rewards/chosen": -0.48091378808021545, "rewards/margins": 0.24324023723602295, "rewards/rejected": -0.7241540551185608, "step": 250 }, { "epoch": 0.14, "learning_rate": 4.977561498345639e-06, "logits/chosen": 0.24105820059776306, "logits/rejected": 0.32143664360046387, "logps/chosen": -767.349365234375, "logps/rejected": -1007.8854370117188, "loss": 1.7354, "rewards/accuracies": 0.6875, "rewards/chosen": -0.4250938296318054, "rewards/margins": 0.24705836176872253, "rewards/rejected": -0.6721522212028503, "step": 260 }, { "epoch": 0.14, "learning_rate": 4.970908694186624e-06, "logits/chosen": 0.24902021884918213, "logits/rejected": 0.2764545977115631, "logps/chosen": -909.8703002929688, "logps/rejected": -1201.24560546875, "loss": 1.7645, "rewards/accuracies": 0.6812499761581421, "rewards/chosen": -0.5648880004882812, "rewards/margins": 0.3094554543495178, "rewards/rejected": -0.8743433952331543, "step": 270 }, { "epoch": 0.15, "learning_rate": 4.9633990224171305e-06, "logits/chosen": 0.22477073967456818, "logits/rejected": 0.32122597098350525, "logps/chosen": -854.8441162109375, "logps/rejected": -1083.2144775390625, "loss": 1.9272, "rewards/accuracies": 0.6812499761581421, "rewards/chosen": -0.492588609457016, "rewards/margins": 0.24561519920825958, "rewards/rejected": -0.7382038235664368, "step": 280 }, { "epoch": 0.15, "learning_rate": 4.955035087259046e-06, "logits/chosen": 0.24095304310321808, "logits/rejected": 0.27735045552253723, "logps/chosen": -786.0255737304688, "logps/rejected": -907.18603515625, "loss": 1.9192, "rewards/accuracies": 0.6000000238418579, "rewards/chosen": -0.43741363286972046, "rewards/margins": 0.15155784785747528, "rewards/rejected": -0.5889714956283569, "step": 290 }, { "epoch": 0.16, "learning_rate": 4.945819789177756e-06, "logits/chosen": 0.28113454580307007, "logits/rejected": 0.328742116689682, "logps/chosen": -699.2544555664062, "logps/rejected": -963.7498168945312, "loss": 1.6742, "rewards/accuracies": 0.71875, "rewards/chosen": -0.35792165994644165, "rewards/margins": 0.29106318950653076, "rewards/rejected": -0.6489847898483276, "step": 300 }, { "epoch": 0.17, "learning_rate": 4.935756323876306e-06, "logits/chosen": 0.25907808542251587, "logits/rejected": 0.32955312728881836, "logps/chosen": -888.01416015625, "logps/rejected": -1171.78369140625, "loss": 1.6716, "rewards/accuracies": 0.7437499761581421, "rewards/chosen": -0.4973801076412201, "rewards/margins": 0.3079432547092438, "rewards/rejected": -0.8053233027458191, "step": 310 }, { "epoch": 0.17, "learning_rate": 4.924848181187199e-06, "logits/chosen": 0.2684496343135834, "logits/rejected": 0.32716041803359985, "logps/chosen": -976.4813232421875, "logps/rejected": -1259.734619140625, "loss": 1.7699, "rewards/accuracies": 0.706250011920929, "rewards/chosen": -0.6323069334030151, "rewards/margins": 0.2792579233646393, "rewards/rejected": -0.911564826965332, "step": 320 }, { "epoch": 0.18, "learning_rate": 4.913099143862173e-06, "logits/chosen": 0.23882272839546204, "logits/rejected": 0.3363894820213318, "logps/chosen": -949.0950927734375, "logps/rejected": -1274.9371337890625, "loss": 1.5789, "rewards/accuracies": 0.6937500238418579, "rewards/chosen": -0.5801984667778015, "rewards/margins": 0.33727020025253296, "rewards/rejected": -0.9174687266349792, "step": 330 }, { "epoch": 0.18, "learning_rate": 4.900513286260416e-06, "logits/chosen": 0.21043291687965393, "logits/rejected": 0.2558518052101135, "logps/chosen": -877.2346801757812, "logps/rejected": -1073.109130859375, "loss": 2.0442, "rewards/accuracies": 0.6312500238418579, "rewards/chosen": -0.5416908860206604, "rewards/margins": 0.2031005322933197, "rewards/rejected": -0.7447915077209473, "step": 340 }, { "epoch": 0.19, "learning_rate": 4.887094972935645e-06, "logits/chosen": 0.16980823874473572, "logits/rejected": 0.25876349210739136, "logps/chosen": -807.2410888671875, "logps/rejected": -1025.34619140625, "loss": 1.7629, "rewards/accuracies": 0.6312500238418579, "rewards/chosen": -0.47985905408859253, "rewards/margins": 0.2095312625169754, "rewards/rejected": -0.6893904209136963, "step": 350 }, { "epoch": 0.19, "learning_rate": 4.87284885712256e-06, "logits/chosen": 0.18493933975696564, "logits/rejected": 0.2487083226442337, "logps/chosen": -851.6658935546875, "logps/rejected": -1111.9356689453125, "loss": 1.6052, "rewards/accuracies": 0.6625000238418579, "rewards/chosen": -0.5041731595993042, "rewards/margins": 0.2590993344783783, "rewards/rejected": -0.7632725834846497, "step": 360 }, { "epoch": 0.2, "learning_rate": 4.857779879123181e-06, "logits/chosen": 0.21949057281017303, "logits/rejected": 0.2645294964313507, "logps/chosen": -1002.091796875, "logps/rejected": -1264.646240234375, "loss": 1.8152, "rewards/accuracies": 0.668749988079071, "rewards/chosen": -0.6552623510360718, "rewards/margins": 0.2699270248413086, "rewards/rejected": -0.9251893758773804, "step": 370 }, { "epoch": 0.2, "learning_rate": 4.841893264593643e-06, "logits/chosen": 0.1550827920436859, "logits/rejected": 0.21283721923828125, "logps/chosen": -910.0081176757812, "logps/rejected": -1235.340087890625, "loss": 1.7966, "rewards/accuracies": 0.643750011920929, "rewards/chosen": -0.6146690249443054, "rewards/margins": 0.31371524930000305, "rewards/rejected": -0.9283843040466309, "step": 380 }, { "epoch": 0.21, "learning_rate": 4.825194522732023e-06, "logits/chosen": 0.16921932995319366, "logits/rejected": 0.2049584686756134, "logps/chosen": -959.2523193359375, "logps/rejected": -1241.6702880859375, "loss": 1.6467, "rewards/accuracies": 0.699999988079071, "rewards/chosen": -0.6084691286087036, "rewards/margins": 0.2845255434513092, "rewards/rejected": -0.8929947018623352, "step": 390 }, { "epoch": 0.21, "learning_rate": 4.807689444367853e-06, "logits/chosen": 0.11238515377044678, "logits/rejected": 0.18375565111637115, "logps/chosen": -928.6314697265625, "logps/rejected": -1184.4490966796875, "loss": 1.7385, "rewards/accuracies": 0.637499988079071, "rewards/chosen": -0.5591214895248413, "rewards/margins": 0.2713908851146698, "rewards/rejected": -0.8305123448371887, "step": 400 }, { "epoch": 0.22, "learning_rate": 4.78938409995396e-06, "logits/chosen": 0.14083296060562134, "logits/rejected": 0.22474069893360138, "logps/chosen": -847.0357666015625, "logps/rejected": -1209.1551513671875, "loss": 1.5509, "rewards/accuracies": 0.6875, "rewards/chosen": -0.524152398109436, "rewards/margins": 0.3589867353439331, "rewards/rejected": -0.8831392526626587, "step": 410 }, { "epoch": 0.22, "learning_rate": 4.770284837461342e-06, "logits/chosen": 0.12362311035394669, "logits/rejected": 0.2302417755126953, "logps/chosen": -902.5638427734375, "logps/rejected": -1300.2840576171875, "loss": 1.5579, "rewards/accuracies": 0.6875, "rewards/chosen": -0.5615973472595215, "rewards/margins": 0.3917149603366852, "rewards/rejected": -0.9533122777938843, "step": 420 }, { "epoch": 0.23, "learning_rate": 4.7503982801778015e-06, "logits/chosen": 0.13658492267131805, "logits/rejected": 0.2200813740491867, "logps/chosen": -907.9083251953125, "logps/rejected": -1213.299072265625, "loss": 1.6695, "rewards/accuracies": 0.6937500238418579, "rewards/chosen": -0.5479084253311157, "rewards/margins": 0.3318006098270416, "rewards/rejected": -0.8797089457511902, "step": 430 }, { "epoch": 0.23, "learning_rate": 4.729731324411104e-06, "logits/chosen": 0.13933658599853516, "logits/rejected": 0.19316565990447998, "logps/chosen": -839.1355590820312, "logps/rejected": -1168.003173828125, "loss": 1.6066, "rewards/accuracies": 0.675000011920929, "rewards/chosen": -0.5049761533737183, "rewards/margins": 0.344684898853302, "rewards/rejected": -0.8496610522270203, "step": 440 }, { "epoch": 0.24, "learning_rate": 4.7082911370974645e-06, "logits/chosen": 0.22927124798297882, "logits/rejected": 0.23591044545173645, "logps/chosen": -970.3570556640625, "logps/rejected": -1318.6837158203125, "loss": 1.5185, "rewards/accuracies": 0.699999988079071, "rewards/chosen": -0.615644097328186, "rewards/margins": 0.3570120334625244, "rewards/rejected": -0.9726561307907104, "step": 450 }, { "epoch": 0.25, "learning_rate": 4.68608515331618e-06, "logits/chosen": 0.17804929614067078, "logits/rejected": 0.23750662803649902, "logps/chosen": -1088.924560546875, "logps/rejected": -1432.3790283203125, "loss": 1.5923, "rewards/accuracies": 0.65625, "rewards/chosen": -0.7223563194274902, "rewards/margins": 0.37143898010253906, "rewards/rejected": -1.0937952995300293, "step": 460 }, { "epoch": 0.25, "learning_rate": 4.663121073711269e-06, "logits/chosen": 0.14603152871131897, "logits/rejected": 0.23797519505023956, "logps/chosen": -1071.876220703125, "logps/rejected": -1489.7244873046875, "loss": 1.7778, "rewards/accuracies": 0.6499999761581421, "rewards/chosen": -0.7220622301101685, "rewards/margins": 0.40577608346939087, "rewards/rejected": -1.127838373184204, "step": 470 }, { "epoch": 0.26, "learning_rate": 4.63940686182103e-06, "logits/chosen": 0.10619086027145386, "logits/rejected": 0.14224204421043396, "logps/chosen": -1099.4365234375, "logps/rejected": -1425.9031982421875, "loss": 1.7194, "rewards/accuracies": 0.6875, "rewards/chosen": -0.7509239912033081, "rewards/margins": 0.32126182317733765, "rewards/rejected": -1.072185754776001, "step": 480 }, { "epoch": 0.26, "learning_rate": 4.614950741316425e-06, "logits/chosen": 0.10838212817907333, "logits/rejected": 0.1664058417081833, "logps/chosen": -1114.0084228515625, "logps/rejected": -1451.7799072265625, "loss": 1.6984, "rewards/accuracies": 0.7250000238418579, "rewards/chosen": -0.7566073536872864, "rewards/margins": 0.3575124740600586, "rewards/rejected": -1.1141197681427002, "step": 490 }, { "epoch": 0.27, "learning_rate": 4.589761193149254e-06, "logits/chosen": 0.09154035151004791, "logits/rejected": 0.08705167472362518, "logps/chosen": -1139.4503173828125, "logps/rejected": -1456.401611328125, "loss": 1.7525, "rewards/accuracies": 0.668749988079071, "rewards/chosen": -0.8078290820121765, "rewards/margins": 0.33122915029525757, "rewards/rejected": -1.139058232307434, "step": 500 }, { "epoch": 0.27, "learning_rate": 4.563846952611112e-06, "logits/chosen": 0.04763927310705185, "logits/rejected": 0.160281240940094, "logps/chosen": -1115.03369140625, "logps/rejected": -1348.77880859375, "loss": 1.7972, "rewards/accuracies": 0.675000011920929, "rewards/chosen": -0.7022587656974792, "rewards/margins": 0.31208425760269165, "rewards/rejected": -1.014343023300171, "step": 510 }, { "epoch": 0.28, "learning_rate": 4.537217006304141e-06, "logits/chosen": 0.05074567720293999, "logits/rejected": 0.1087862104177475, "logps/chosen": -982.4554443359375, "logps/rejected": -1275.624267578125, "loss": 1.6134, "rewards/accuracies": 0.737500011920929, "rewards/chosen": -0.6048920750617981, "rewards/margins": 0.3106018006801605, "rewards/rejected": -0.9154938459396362, "step": 520 }, { "epoch": 0.28, "learning_rate": 4.50988058902464e-06, "logits/chosen": 0.01388006191700697, "logits/rejected": 0.0734701007604599, "logps/chosen": -849.9498291015625, "logps/rejected": -1094.3758544921875, "loss": 1.8119, "rewards/accuracies": 0.6625000238418579, "rewards/chosen": -0.5126731991767883, "rewards/margins": 0.261741042137146, "rewards/rejected": -0.7744141817092896, "step": 530 }, { "epoch": 0.29, "learning_rate": 4.481847180560593e-06, "logits/chosen": 0.02280263975262642, "logits/rejected": 0.06258614361286163, "logps/chosen": -819.1052856445312, "logps/rejected": -1066.9857177734375, "loss": 1.7565, "rewards/accuracies": 0.6499999761581421, "rewards/chosen": -0.5206565856933594, "rewards/margins": 0.2521199584007263, "rewards/rejected": -0.7727766633033752, "step": 540 }, { "epoch": 0.29, "learning_rate": 4.453126502404253e-06, "logits/chosen": 0.0006181815406307578, "logits/rejected": 0.02327551506459713, "logps/chosen": -941.8448486328125, "logps/rejected": -1197.061767578125, "loss": 1.7268, "rewards/accuracies": 0.675000011920929, "rewards/chosen": -0.5985831022262573, "rewards/margins": 0.26690778136253357, "rewards/rejected": -0.8654907941818237, "step": 550 }, { "epoch": 0.3, "learning_rate": 4.423728514380892e-06, "logits/chosen": 0.00485605001449585, "logits/rejected": 0.03726247698068619, "logps/chosen": -1022.1149291992188, "logps/rejected": -1314.630859375, "loss": 1.8572, "rewards/accuracies": 0.612500011920929, "rewards/chosen": -0.6928538084030151, "rewards/margins": 0.2553819417953491, "rewards/rejected": -0.9482356905937195, "step": 560 }, { "epoch": 0.3, "learning_rate": 4.393663411194918e-06, "logits/chosen": 0.038806699216365814, "logits/rejected": 0.05322469025850296, "logps/chosen": -993.5315551757812, "logps/rejected": -1312.275634765625, "loss": 1.6143, "rewards/accuracies": 0.668749988079071, "rewards/chosen": -0.6640539765357971, "rewards/margins": 0.3003658652305603, "rewards/rejected": -0.9644198417663574, "step": 570 }, { "epoch": 0.31, "learning_rate": 4.362941618894523e-06, "logits/chosen": 0.01965520530939102, "logits/rejected": 0.033857885748147964, "logps/chosen": -962.2711181640625, "logps/rejected": -1249.142578125, "loss": 1.767, "rewards/accuracies": 0.65625, "rewards/chosen": -0.6267088651657104, "rewards/margins": 0.29039520025253296, "rewards/rejected": -0.9171040654182434, "step": 580 }, { "epoch": 0.31, "learning_rate": 4.331573791256116e-06, "logits/chosen": -0.02834545448422432, "logits/rejected": 0.037625543773174286, "logps/chosen": -1008.5965576171875, "logps/rejected": -1362.098876953125, "loss": 1.5613, "rewards/accuracies": 0.6812499761581421, "rewards/chosen": -0.6537022590637207, "rewards/margins": 0.3879777789115906, "rewards/rejected": -1.0416799783706665, "step": 590 }, { "epoch": 0.32, "learning_rate": 4.299570806089786e-06, "logits/chosen": -0.03007788583636284, "logits/rejected": 0.0712488442659378, "logps/chosen": -982.5880737304688, "logps/rejected": -1292.6614990234375, "loss": 1.6944, "rewards/accuracies": 0.675000011920929, "rewards/chosen": -0.6564881801605225, "rewards/margins": 0.3164929747581482, "rewards/rejected": -0.9729811549186707, "step": 600 }, { "epoch": 0.33, "learning_rate": 4.266943761467057e-06, "logits/chosen": 0.03833829611539841, "logits/rejected": 0.06410752981901169, "logps/chosen": -991.5280151367188, "logps/rejected": -1269.9459228515625, "loss": 1.6837, "rewards/accuracies": 0.6499999761581421, "rewards/chosen": -0.6482813954353333, "rewards/margins": 0.28088536858558655, "rewards/rejected": -0.9291667938232422, "step": 610 }, { "epoch": 0.33, "learning_rate": 4.233703971872287e-06, "logits/chosen": 0.01419871486723423, "logits/rejected": 0.041936878114938736, "logps/chosen": -955.8319091796875, "logps/rejected": -1251.8397216796875, "loss": 1.6607, "rewards/accuracies": 0.6812499761581421, "rewards/chosen": -0.6094011664390564, "rewards/margins": 0.32103410363197327, "rewards/rejected": -0.9304353594779968, "step": 620 }, { "epoch": 0.34, "learning_rate": 4.1998629642789925e-06, "logits/chosen": 0.0568360760807991, "logits/rejected": 0.1395053267478943, "logps/chosen": -1048.60009765625, "logps/rejected": -1351.2703857421875, "loss": 1.7727, "rewards/accuracies": 0.612500011920929, "rewards/chosen": -0.6901832222938538, "rewards/margins": 0.2976410686969757, "rewards/rejected": -0.9878243207931519, "step": 630 }, { "epoch": 0.34, "learning_rate": 4.165432474152505e-06, "logits/chosen": 0.032792214304208755, "logits/rejected": 0.019641969352960587, "logps/chosen": -961.2864990234375, "logps/rejected": -1392.465576171875, "loss": 1.4681, "rewards/accuracies": 0.7437499761581421, "rewards/chosen": -0.6157466173171997, "rewards/margins": 0.42596155405044556, "rewards/rejected": -1.04170823097229, "step": 640 }, { "epoch": 0.35, "learning_rate": 4.130424441380308e-06, "logits/chosen": 0.0214821957051754, "logits/rejected": 0.11312129348516464, "logps/chosen": -1139.0208740234375, "logps/rejected": -1529.8564453125, "loss": 1.5676, "rewards/accuracies": 0.731249988079071, "rewards/chosen": -0.7614965438842773, "rewards/margins": 0.40181460976600647, "rewards/rejected": -1.163311243057251, "step": 650 }, { "epoch": 0.35, "learning_rate": 4.09485100613151e-06, "logits/chosen": 0.03265173360705376, "logits/rejected": 0.08288411051034927, "logps/chosen": -1008.8547973632812, "logps/rejected": -1373.1070556640625, "loss": 1.6128, "rewards/accuracies": 0.643750011920929, "rewards/chosen": -0.6687760949134827, "rewards/margins": 0.3712404668331146, "rewards/rejected": -1.040016531944275, "step": 660 }, { "epoch": 0.36, "learning_rate": 4.058724504646834e-06, "logits/chosen": 0.008680765517055988, "logits/rejected": 0.03280487284064293, "logps/chosen": -1010.1678466796875, "logps/rejected": -1343.5008544921875, "loss": 1.6108, "rewards/accuracies": 0.6937500238418579, "rewards/chosen": -0.6555579304695129, "rewards/margins": 0.34102123975753784, "rewards/rejected": -0.9965791702270508, "step": 670 }, { "epoch": 0.36, "learning_rate": 4.022057464960632e-06, "logits/chosen": 0.051326651126146317, "logits/rejected": 0.10143965482711792, "logps/chosen": -1015.6981201171875, "logps/rejected": -1286.204345703125, "loss": 1.8244, "rewards/accuracies": 0.6625000238418579, "rewards/chosen": -0.6295019388198853, "rewards/margins": 0.3051254153251648, "rewards/rejected": -0.93462735414505, "step": 680 }, { "epoch": 0.37, "learning_rate": 3.984862602556383e-06, "logits/chosen": 0.06714431941509247, "logits/rejected": 0.14711423218250275, "logps/chosen": -878.5494384765625, "logps/rejected": -1243.7890625, "loss": 1.5624, "rewards/accuracies": 0.737500011920929, "rewards/chosen": -0.5232223272323608, "rewards/margins": 0.37018781900405884, "rewards/rejected": -0.8934100866317749, "step": 690 }, { "epoch": 0.37, "learning_rate": 3.947152815957187e-06, "logits/chosen": 0.024934569373726845, "logits/rejected": 0.09314241260290146, "logps/chosen": -954.0012817382812, "logps/rejected": -1265.03564453125, "loss": 1.6638, "rewards/accuracies": 0.731249988079071, "rewards/chosen": -0.6106816530227661, "rewards/margins": 0.3257291316986084, "rewards/rejected": -0.9364109039306641, "step": 700 }, { "epoch": 0.38, "learning_rate": 3.908941182252785e-06, "logits/chosen": 0.029830629006028175, "logits/rejected": 0.10770799964666367, "logps/chosen": -949.3072509765625, "logps/rejected": -1249.8748779296875, "loss": 1.7333, "rewards/accuracies": 0.668749988079071, "rewards/chosen": -0.6108871102333069, "rewards/margins": 0.3122674822807312, "rewards/rejected": -0.9231545329093933, "step": 710 }, { "epoch": 0.38, "learning_rate": 3.8702409525646535e-06, "logits/chosen": 0.060981638729572296, "logits/rejected": 0.06661652028560638, "logps/chosen": -897.6103515625, "logps/rejected": -1229.4617919921875, "loss": 1.6608, "rewards/accuracies": 0.65625, "rewards/chosen": -0.5749759674072266, "rewards/margins": 0.29930251836776733, "rewards/rejected": -0.8742784261703491, "step": 720 }, { "epoch": 0.39, "learning_rate": 3.8310655474507495e-06, "logits/chosen": 0.03684331104159355, "logits/rejected": 0.14619871973991394, "logps/chosen": -923.5501098632812, "logps/rejected": -1209.08740234375, "loss": 1.6177, "rewards/accuracies": 0.65625, "rewards/chosen": -0.5860185623168945, "rewards/margins": 0.3128363788127899, "rewards/rejected": -0.8988549113273621, "step": 730 }, { "epoch": 0.39, "learning_rate": 3.7914285522515002e-06, "logits/chosen": 0.011234849691390991, "logits/rejected": 0.10097019374370575, "logps/chosen": -936.65234375, "logps/rejected": -1304.6485595703125, "loss": 1.5484, "rewards/accuracies": 0.731249988079071, "rewards/chosen": -0.5933856964111328, "rewards/margins": 0.3895036578178406, "rewards/rejected": -0.9828893542289734, "step": 740 }, { "epoch": 0.4, "learning_rate": 3.751343712378639e-06, "logits/chosen": 0.05703802779316902, "logits/rejected": 0.13415174186229706, "logps/chosen": -1007.4900512695312, "logps/rejected": -1316.6981201171875, "loss": 1.5569, "rewards/accuracies": 0.6875, "rewards/chosen": -0.6273565888404846, "rewards/margins": 0.3487706184387207, "rewards/rejected": -0.9761272668838501, "step": 750 }, { "epoch": 0.41, "learning_rate": 3.710824928548546e-06, "logits/chosen": 0.11401792615652084, "logits/rejected": 0.149090975522995, "logps/chosen": -1011.75634765625, "logps/rejected": -1267.4716796875, "loss": 1.991, "rewards/accuracies": 0.6187499761581421, "rewards/chosen": -0.662542462348938, "rewards/margins": 0.27919334173202515, "rewards/rejected": -0.9417358636856079, "step": 760 }, { "epoch": 0.41, "learning_rate": 3.6698862519617225e-06, "logits/chosen": 0.04727781563997269, "logits/rejected": 0.0998949259519577, "logps/chosen": -959.3302001953125, "logps/rejected": -1265.2984619140625, "loss": 1.6325, "rewards/accuracies": 0.65625, "rewards/chosen": -0.6212436556816101, "rewards/margins": 0.3175702691078186, "rewards/rejected": -0.9388139843940735, "step": 770 }, { "epoch": 0.42, "learning_rate": 3.6285418794300793e-06, "logits/chosen": 0.08479107916355133, "logits/rejected": 0.1416613757610321, "logps/chosen": -1000.9298706054688, "logps/rejected": -1267.0145263671875, "loss": 1.7375, "rewards/accuracies": 0.637499988079071, "rewards/chosen": -0.6357444524765015, "rewards/margins": 0.310316264629364, "rewards/rejected": -0.9460607767105103, "step": 780 }, { "epoch": 0.42, "learning_rate": 3.5868061484537365e-06, "logits/chosen": 0.02577986940741539, "logits/rejected": 0.08596567064523697, "logps/chosen": -1112.874755859375, "logps/rejected": -1473.1839599609375, "loss": 1.6638, "rewards/accuracies": 0.668749988079071, "rewards/chosen": -0.7674813270568848, "rewards/margins": 0.35035058856010437, "rewards/rejected": -1.1178319454193115, "step": 790 }, { "epoch": 0.43, "learning_rate": 3.5446935322490285e-06, "logits/chosen": 0.03142818063497543, "logits/rejected": 0.0737256333231926, "logps/chosen": -1049.059326171875, "logps/rejected": -1381.332763671875, "loss": 1.6325, "rewards/accuracies": 0.71875, "rewards/chosen": -0.7112890481948853, "rewards/margins": 0.3419298231601715, "rewards/rejected": -1.0532188415527344, "step": 800 }, { "epoch": 0.43, "learning_rate": 3.502218634729447e-06, "logits/chosen": 0.04486529529094696, "logits/rejected": 0.07387622445821762, "logps/chosen": -1063.0550537109375, "logps/rejected": -1385.9937744140625, "loss": 1.7277, "rewards/accuracies": 0.65625, "rewards/chosen": -0.7465726137161255, "rewards/margins": 0.30831706523895264, "rewards/rejected": -1.0548895597457886, "step": 810 }, { "epoch": 0.44, "learning_rate": 3.459396185441265e-06, "logits/chosen": 0.042615022510290146, "logits/rejected": 0.08985388278961182, "logps/chosen": -1015.6598510742188, "logps/rejected": -1335.8912353515625, "loss": 1.5891, "rewards/accuracies": 0.71875, "rewards/chosen": -0.6490973234176636, "rewards/margins": 0.33889901638031006, "rewards/rejected": -0.9879962801933289, "step": 820 }, { "epoch": 0.44, "learning_rate": 3.4162410344555834e-06, "logits/chosen": 0.07760035991668701, "logits/rejected": 0.09005734324455261, "logps/chosen": -1110.9671630859375, "logps/rejected": -1389.163330078125, "loss": 1.7173, "rewards/accuracies": 0.6875, "rewards/chosen": -0.7506981492042542, "rewards/margins": 0.29421886801719666, "rewards/rejected": -1.0449169874191284, "step": 830 }, { "epoch": 0.45, "learning_rate": 3.3727681472185937e-06, "logits/chosen": 0.052268415689468384, "logits/rejected": 0.10122676193714142, "logps/chosen": -903.8248901367188, "logps/rejected": -1204.15576171875, "loss": 1.5496, "rewards/accuracies": 0.6812499761581421, "rewards/chosen": -0.5848682522773743, "rewards/margins": 0.31761133670806885, "rewards/rejected": -0.9024796485900879, "step": 840 }, { "epoch": 0.45, "learning_rate": 3.3289925993618217e-06, "logits/chosen": 0.06663919985294342, "logits/rejected": 0.1188969761133194, "logps/chosen": -1047.30029296875, "logps/rejected": -1304.7379150390625, "loss": 1.7105, "rewards/accuracies": 0.668749988079071, "rewards/chosen": -0.671362578868866, "rewards/margins": 0.3004400134086609, "rewards/rejected": -0.9718025326728821, "step": 850 }, { "epoch": 0.46, "learning_rate": 3.2849295714741643e-06, "logits/chosen": 0.0677184984087944, "logits/rejected": 0.11335499584674835, "logps/chosen": -1036.0316162109375, "logps/rejected": -1401.0706787109375, "loss": 1.622, "rewards/accuracies": 0.6937500238418579, "rewards/chosen": -0.6989107131958008, "rewards/margins": 0.36011943221092224, "rewards/rejected": -1.0590301752090454, "step": 860 }, { "epoch": 0.46, "learning_rate": 3.2405943438375287e-06, "logits/chosen": 0.08610890805721283, "logits/rejected": 0.1343824565410614, "logps/chosen": -1109.2958984375, "logps/rejected": -1425.269287109375, "loss": 1.6286, "rewards/accuracies": 0.6812499761581421, "rewards/chosen": -0.753276526927948, "rewards/margins": 0.3505406975746155, "rewards/rejected": -1.1038172245025635, "step": 870 }, { "epoch": 0.47, "learning_rate": 3.1960022911279036e-06, "logits/chosen": 0.04142437130212784, "logits/rejected": 0.10113891214132309, "logps/chosen": -1057.3348388671875, "logps/rejected": -1386.047607421875, "loss": 1.754, "rewards/accuracies": 0.699999988079071, "rewards/chosen": -0.7235895395278931, "rewards/margins": 0.334214448928833, "rewards/rejected": -1.0578038692474365, "step": 880 }, { "epoch": 0.47, "learning_rate": 3.1511688770836844e-06, "logits/chosen": 0.062126994132995605, "logits/rejected": 0.10421881824731827, "logps/chosen": -1068.828857421875, "logps/rejected": -1316.99951171875, "loss": 1.6456, "rewards/accuracies": 0.6625000238418579, "rewards/chosen": -0.732558012008667, "rewards/margins": 0.2871066927909851, "rewards/rejected": -1.0196647644042969, "step": 890 }, { "epoch": 0.48, "learning_rate": 3.1061096491431307e-06, "logits/chosen": 0.062398504465818405, "logits/rejected": 0.14607712626457214, "logps/chosen": -1060.228271484375, "logps/rejected": -1450.5191650390625, "loss": 1.6495, "rewards/accuracies": 0.7124999761581421, "rewards/chosen": -0.7299556732177734, "rewards/margins": 0.3911103308200836, "rewards/rejected": -1.1210659742355347, "step": 900 }, { "epoch": 0.49, "learning_rate": 3.0608402330527796e-06, "logits/chosen": 0.10377037525177002, "logits/rejected": 0.1702093631029129, "logps/chosen": -1070.2183837890625, "logps/rejected": -1444.0865478515625, "loss": 1.4815, "rewards/accuracies": 0.7749999761581421, "rewards/chosen": -0.6973433494567871, "rewards/margins": 0.40493422746658325, "rewards/rejected": -1.1022775173187256, "step": 910 }, { "epoch": 0.49, "learning_rate": 3.0153763274487176e-06, "logits/chosen": 0.10599883645772934, "logits/rejected": 0.14606773853302002, "logps/chosen": -1007.1170043945312, "logps/rejected": -1397.21826171875, "loss": 1.7218, "rewards/accuracies": 0.706250011920929, "rewards/chosen": -0.7144777178764343, "rewards/margins": 0.3858681619167328, "rewards/rejected": -1.1003458499908447, "step": 920 }, { "epoch": 0.5, "learning_rate": 2.9697336984125683e-06, "logits/chosen": 0.10908961296081543, "logits/rejected": 0.1530420482158661, "logps/chosen": -1157.73828125, "logps/rejected": -1576.6483154296875, "loss": 1.5431, "rewards/accuracies": 0.668749988079071, "rewards/chosen": -0.8187190890312195, "rewards/margins": 0.3967840075492859, "rewards/rejected": -1.2155030965805054, "step": 930 }, { "epoch": 0.5, "learning_rate": 2.923928174004094e-06, "logits/chosen": 0.06097055599093437, "logits/rejected": 0.11391390860080719, "logps/chosen": -1108.5380859375, "logps/rejected": -1444.47607421875, "loss": 1.6738, "rewards/accuracies": 0.65625, "rewards/chosen": -0.7912001609802246, "rewards/margins": 0.3262966573238373, "rewards/rejected": -1.1174967288970947, "step": 940 }, { "epoch": 0.51, "learning_rate": 2.8779756387723036e-06, "logits/chosen": 0.06467770040035248, "logits/rejected": 0.0973706915974617, "logps/chosen": -1154.4818115234375, "logps/rejected": -1414.0888671875, "loss": 1.4953, "rewards/accuracies": 0.668749988079071, "rewards/chosen": -0.7806675434112549, "rewards/margins": 0.309781551361084, "rewards/rejected": -1.0904490947723389, "step": 950 }, { "epoch": 0.51, "learning_rate": 2.831892028246968e-06, "logits/chosen": 0.07558836042881012, "logits/rejected": 0.13848301768302917, "logps/chosen": -1124.1239013671875, "logps/rejected": -1415.198486328125, "loss": 1.6304, "rewards/accuracies": 0.6625000238418579, "rewards/chosen": -0.7694803476333618, "rewards/margins": 0.3106168210506439, "rewards/rejected": -1.0800971984863281, "step": 960 }, { "epoch": 0.52, "learning_rate": 2.7856933234124617e-06, "logits/chosen": 0.09707184135913849, "logits/rejected": 0.15487423539161682, "logps/chosen": -1033.051513671875, "logps/rejected": -1383.0096435546875, "loss": 1.5955, "rewards/accuracies": 0.6812499761581421, "rewards/chosen": -0.7070351243019104, "rewards/margins": 0.32586950063705444, "rewards/rejected": -1.0329046249389648, "step": 970 }, { "epoch": 0.52, "learning_rate": 2.7393955451658387e-06, "logits/chosen": 0.17485928535461426, "logits/rejected": 0.20479004085063934, "logps/chosen": -1019.5245361328125, "logps/rejected": -1405.92578125, "loss": 1.5161, "rewards/accuracies": 0.7562500238418579, "rewards/chosen": -0.6749985814094543, "rewards/margins": 0.39169296622276306, "rewards/rejected": -1.066691517829895, "step": 980 }, { "epoch": 0.53, "learning_rate": 2.6930147487610667e-06, "logits/chosen": 0.1469564139842987, "logits/rejected": 0.18740007281303406, "logps/chosen": -1112.45654296875, "logps/rejected": -1390.9603271484375, "loss": 1.8518, "rewards/accuracies": 0.6812499761581421, "rewards/chosen": -0.7527428269386292, "rewards/margins": 0.29854851961135864, "rewards/rejected": -1.0512912273406982, "step": 990 }, { "epoch": 0.53, "learning_rate": 2.6465670182413487e-06, "logits/chosen": 0.1311412751674652, "logits/rejected": 0.16103684902191162, "logps/chosen": -1034.7266845703125, "logps/rejected": -1487.0198974609375, "loss": 1.4757, "rewards/accuracies": 0.731249988079071, "rewards/chosen": -0.6916247010231018, "rewards/margins": 0.43866676092147827, "rewards/rejected": -1.13029146194458, "step": 1000 }, { "epoch": 0.54, "learning_rate": 2.6000684608614594e-06, "logits/chosen": 0.1081945076584816, "logits/rejected": 0.2089473456144333, "logps/chosen": -1155.034423828125, "logps/rejected": -1513.755615234375, "loss": 1.6896, "rewards/accuracies": 0.6187499761581421, "rewards/chosen": -0.815504252910614, "rewards/margins": 0.3612630367279053, "rewards/rejected": -1.1767674684524536, "step": 1010 }, { "epoch": 0.54, "learning_rate": 2.5535352015020338e-06, "logits/chosen": 0.1341986358165741, "logits/rejected": 0.18716619908809662, "logps/chosen": -1160.8070068359375, "logps/rejected": -1478.892333984375, "loss": 1.566, "rewards/accuracies": 0.6937500238418579, "rewards/chosen": -0.7914396524429321, "rewards/margins": 0.33516114950180054, "rewards/rejected": -1.126600742340088, "step": 1020 }, { "epoch": 0.55, "learning_rate": 2.506983377077741e-06, "logits/chosen": 0.11889129877090454, "logits/rejected": 0.172836571931839, "logps/chosen": -1112.3515625, "logps/rejected": -1390.6231689453125, "loss": 1.7204, "rewards/accuracies": 0.637499988079071, "rewards/chosen": -0.7569676637649536, "rewards/margins": 0.2798970341682434, "rewards/rejected": -1.0368645191192627, "step": 1030 }, { "epoch": 0.55, "learning_rate": 2.460429130941289e-06, "logits/chosen": 0.1303168535232544, "logits/rejected": 0.23098449409008026, "logps/chosen": -1031.8233642578125, "logps/rejected": -1380.739013671875, "loss": 1.4098, "rewards/accuracies": 0.75, "rewards/chosen": -0.6430400609970093, "rewards/margins": 0.3773672878742218, "rewards/rejected": -1.0204073190689087, "step": 1040 }, { "epoch": 0.56, "learning_rate": 2.413888607285192e-06, "logits/chosen": 0.08564915508031845, "logits/rejected": 0.12453781068325043, "logps/chosen": -985.0271606445312, "logps/rejected": -1363.299072265625, "loss": 1.5626, "rewards/accuracies": 0.7437499761581421, "rewards/chosen": -0.6599403619766235, "rewards/margins": 0.3573141098022461, "rewards/rejected": -1.0172545909881592, "step": 1050 }, { "epoch": 0.57, "learning_rate": 2.367377945543249e-06, "logits/chosen": 0.12176636606454849, "logits/rejected": 0.2213442027568817, "logps/chosen": -1076.0185546875, "logps/rejected": -1498.5506591796875, "loss": 1.5116, "rewards/accuracies": 0.7124999761581421, "rewards/chosen": -0.7448247671127319, "rewards/margins": 0.4244067668914795, "rewards/rejected": -1.169231653213501, "step": 1060 }, { "epoch": 0.57, "learning_rate": 2.320913274793676e-06, "logits/chosen": 0.1362072080373764, "logits/rejected": 0.1790778636932373, "logps/chosen": -1171.556396484375, "logps/rejected": -1544.1282958984375, "loss": 1.6752, "rewards/accuracies": 0.6937500238418579, "rewards/chosen": -0.8081449270248413, "rewards/margins": 0.386355459690094, "rewards/rejected": -1.1945003271102905, "step": 1070 }, { "epoch": 0.58, "learning_rate": 2.27451070816582e-06, "logits/chosen": 0.13579389452934265, "logits/rejected": 0.1608073115348816, "logps/chosen": -1076.015380859375, "logps/rejected": -1350.9400634765625, "loss": 1.7586, "rewards/accuracies": 0.5874999761581421, "rewards/chosen": -0.7380760312080383, "rewards/margins": 0.29431313276290894, "rewards/rejected": -1.0323892831802368, "step": 1080 }, { "epoch": 0.58, "learning_rate": 2.228186337252414e-06, "logits/chosen": 0.09430348873138428, "logits/rejected": 0.20111441612243652, "logps/chosen": -1102.0145263671875, "logps/rejected": -1460.5335693359375, "loss": 1.6547, "rewards/accuracies": 0.706250011920929, "rewards/chosen": -0.7317436337471008, "rewards/margins": 0.37700966000556946, "rewards/rejected": -1.1087532043457031, "step": 1090 }, { "epoch": 0.59, "learning_rate": 2.1819562265292946e-06, "logits/chosen": 0.16240547597408295, "logits/rejected": 0.1689942628145218, "logps/chosen": -983.9060668945312, "logps/rejected": -1212.6190185546875, "loss": 1.6005, "rewards/accuracies": 0.6625000238418579, "rewards/chosen": -0.6035498380661011, "rewards/margins": 0.29034045338630676, "rewards/rejected": -0.8938902616500854, "step": 1100 }, { "epoch": 0.59, "learning_rate": 2.1358364077845236e-06, "logits/chosen": 0.13844513893127441, "logits/rejected": 0.2087705433368683, "logps/chosen": -1064.875732421875, "logps/rejected": -1291.0726318359375, "loss": 1.7957, "rewards/accuracies": 0.668749988079071, "rewards/chosen": -0.6955076456069946, "rewards/margins": 0.26428431272506714, "rewards/rejected": -0.9597919583320618, "step": 1110 }, { "epoch": 0.6, "learning_rate": 2.089842874558849e-06, "logits/chosen": 0.15783284604549408, "logits/rejected": 0.22821958363056183, "logps/chosen": -973.8444213867188, "logps/rejected": -1261.2291259765625, "loss": 1.5954, "rewards/accuracies": 0.7124999761581421, "rewards/chosen": -0.6483608484268188, "rewards/margins": 0.29802364110946655, "rewards/rejected": -0.946384608745575, "step": 1120 }, { "epoch": 0.6, "learning_rate": 2.0439915765994242e-06, "logits/chosen": 0.13555693626403809, "logits/rejected": 0.15965650975704193, "logps/chosen": -1126.0108642578125, "logps/rejected": -1440.069580078125, "loss": 1.5556, "rewards/accuracies": 0.6937500238418579, "rewards/chosen": -0.7768179774284363, "rewards/margins": 0.32900911569595337, "rewards/rejected": -1.1058270931243896, "step": 1130 }, { "epoch": 0.61, "learning_rate": 1.9982984143287186e-06, "logits/chosen": 0.15475307404994965, "logits/rejected": 0.1996763050556183, "logps/chosen": -1288.257568359375, "logps/rejected": -1648.780517578125, "loss": 1.4784, "rewards/accuracies": 0.6937500238418579, "rewards/chosen": -0.8938125371932983, "rewards/margins": 0.37623515725135803, "rewards/rejected": -1.270047903060913, "step": 1140 }, { "epoch": 0.61, "learning_rate": 1.95277923333053e-06, "logits/chosen": 0.07992354780435562, "logits/rejected": 0.14772897958755493, "logps/chosen": -1287.0286865234375, "logps/rejected": -1581.1400146484375, "loss": 1.759, "rewards/accuracies": 0.6937500238418579, "rewards/chosen": -0.8965250849723816, "rewards/margins": 0.3494908809661865, "rewards/rejected": -1.2460159063339233, "step": 1150 }, { "epoch": 0.62, "learning_rate": 1.9074498188550156e-06, "logits/chosen": 0.09774446487426758, "logits/rejected": 0.14773234724998474, "logps/chosen": -1122.9798583984375, "logps/rejected": -1415.415283203125, "loss": 1.7595, "rewards/accuracies": 0.699999988079071, "rewards/chosen": -0.7342059016227722, "rewards/margins": 0.3350808918476105, "rewards/rejected": -1.069286823272705, "step": 1160 }, { "epoch": 0.62, "learning_rate": 1.862325890344643e-06, "logits/chosen": 0.08668424934148788, "logits/rejected": 0.1557636559009552, "logps/chosen": -1046.437744140625, "logps/rejected": -1386.939208984375, "loss": 1.5183, "rewards/accuracies": 0.6875, "rewards/chosen": -0.7016348242759705, "rewards/margins": 0.36825114488601685, "rewards/rejected": -1.0698859691619873, "step": 1170 }, { "epoch": 0.63, "learning_rate": 1.817423095982972e-06, "logits/chosen": 0.12340223789215088, "logits/rejected": 0.14640815556049347, "logps/chosen": -974.4490966796875, "logps/rejected": -1238.3133544921875, "loss": 1.6995, "rewards/accuracies": 0.668749988079071, "rewards/chosen": -0.6591961979866028, "rewards/margins": 0.2718682587146759, "rewards/rejected": -0.9310644268989563, "step": 1180 }, { "epoch": 0.63, "learning_rate": 1.7727570072681293e-06, "logits/chosen": 0.14854778349399567, "logits/rejected": 0.17698471248149872, "logps/chosen": -1057.446533203125, "logps/rejected": -1373.8194580078125, "loss": 1.6968, "rewards/accuracies": 0.6625000238418579, "rewards/chosen": -0.7261327505111694, "rewards/margins": 0.3126031458377838, "rewards/rejected": -1.0387358665466309, "step": 1190 }, { "epoch": 0.64, "learning_rate": 1.7283431136128961e-06, "logits/chosen": 0.14250560104846954, "logits/rejected": 0.17423418164253235, "logps/chosen": -1109.4017333984375, "logps/rejected": -1430.1903076171875, "loss": 1.5583, "rewards/accuracies": 0.706250011920929, "rewards/chosen": -0.7613256573677063, "rewards/margins": 0.3207666873931885, "rewards/rejected": -1.08209228515625, "step": 1200 }, { "epoch": 0.65, "learning_rate": 1.6841968169732478e-06, "logits/chosen": 0.10577497631311417, "logits/rejected": 0.17070509493350983, "logps/chosen": -1129.2164306640625, "logps/rejected": -1423.0538330078125, "loss": 1.7291, "rewards/accuracies": 0.6625000238418579, "rewards/chosen": -0.7770806550979614, "rewards/margins": 0.32531023025512695, "rewards/rejected": -1.1023907661437988, "step": 1210 }, { "epoch": 0.65, "learning_rate": 1.6403334265072284e-06, "logits/chosen": 0.1173873171210289, "logits/rejected": 0.18929217755794525, "logps/chosen": -1162.355224609375, "logps/rejected": -1572.89306640625, "loss": 1.5668, "rewards/accuracies": 0.71875, "rewards/chosen": -0.8342976570129395, "rewards/margins": 0.41481128334999084, "rewards/rejected": -1.2491087913513184, "step": 1220 }, { "epoch": 0.66, "learning_rate": 1.5967681532660066e-06, "logits/chosen": 0.08649718761444092, "logits/rejected": 0.12614522874355316, "logps/chosen": -1182.95361328125, "logps/rejected": -1542.8983154296875, "loss": 1.552, "rewards/accuracies": 0.71875, "rewards/chosen": -0.8389142751693726, "rewards/margins": 0.3838485777378082, "rewards/rejected": -1.2227627038955688, "step": 1230 }, { "epoch": 0.66, "learning_rate": 1.5535161049189463e-06, "logits/chosen": 0.048624537885189056, "logits/rejected": 0.12681320309638977, "logps/chosen": -1181.8487548828125, "logps/rejected": -1500.6976318359375, "loss": 1.6678, "rewards/accuracies": 0.675000011920929, "rewards/chosen": -0.8252663612365723, "rewards/margins": 0.32674065232276917, "rewards/rejected": -1.1520068645477295, "step": 1240 }, { "epoch": 0.67, "learning_rate": 1.5105922805145356e-06, "logits/chosen": 0.0684380978345871, "logits/rejected": 0.12469562143087387, "logps/chosen": -1170.2078857421875, "logps/rejected": -1520.978759765625, "loss": 1.6999, "rewards/accuracies": 0.6937500238418579, "rewards/chosen": -0.77997887134552, "rewards/margins": 0.3841548562049866, "rewards/rejected": -1.1641336679458618, "step": 1250 }, { "epoch": 0.67, "learning_rate": 1.4680115652789823e-06, "logits/chosen": 0.08953334391117096, "logits/rejected": 0.12195037305355072, "logps/chosen": -1053.8450927734375, "logps/rejected": -1404.131591796875, "loss": 1.5207, "rewards/accuracies": 0.6625000238418579, "rewards/chosen": -0.7199850082397461, "rewards/margins": 0.3669469952583313, "rewards/rejected": -1.0869319438934326, "step": 1260 }, { "epoch": 0.68, "learning_rate": 1.4257887254542767e-06, "logits/chosen": 0.06104928255081177, "logits/rejected": 0.09251236915588379, "logps/chosen": -1130.771728515625, "logps/rejected": -1455.777587890625, "loss": 1.7221, "rewards/accuracies": 0.6875, "rewards/chosen": -0.7961911559104919, "rewards/margins": 0.32651156187057495, "rewards/rejected": -1.122702717781067, "step": 1270 }, { "epoch": 0.68, "learning_rate": 1.3839384031775227e-06, "logits/chosen": 0.09284323453903198, "logits/rejected": 0.12470339238643646, "logps/chosen": -1184.2957763671875, "logps/rejected": -1491.9906005859375, "loss": 1.7259, "rewards/accuracies": 0.606249988079071, "rewards/chosen": -0.876724362373352, "rewards/margins": 0.3096561133861542, "rewards/rejected": -1.186380386352539, "step": 1280 }, { "epoch": 0.69, "learning_rate": 1.342475111403298e-06, "logits/chosen": 0.07502352446317673, "logits/rejected": 0.16246311366558075, "logps/chosen": -1262.9688720703125, "logps/rejected": -1597.161376953125, "loss": 1.6929, "rewards/accuracies": 0.6499999761581421, "rewards/chosen": -0.899106502532959, "rewards/margins": 0.3475545048713684, "rewards/rejected": -1.2466610670089722, "step": 1290 }, { "epoch": 0.69, "learning_rate": 1.3014132288708209e-06, "logits/chosen": 0.12563182413578033, "logits/rejected": 0.18176163733005524, "logps/chosen": -1079.9150390625, "logps/rejected": -1426.569580078125, "loss": 1.5658, "rewards/accuracies": 0.6499999761581421, "rewards/chosen": -0.7534309029579163, "rewards/margins": 0.33269041776657104, "rewards/rejected": -1.0861213207244873, "step": 1300 }, { "epoch": 0.7, "learning_rate": 1.2607669951176549e-06, "logits/chosen": 0.09636719524860382, "logits/rejected": 0.12595590949058533, "logps/chosen": -1216.9656982421875, "logps/rejected": -1592.846923828125, "loss": 1.5262, "rewards/accuracies": 0.7437499761581421, "rewards/chosen": -0.8419526219367981, "rewards/margins": 0.4114845395088196, "rewards/rejected": -1.2534370422363281, "step": 1310 }, { "epoch": 0.7, "learning_rate": 1.2205505055416891e-06, "logits/chosen": 0.0674719512462616, "logits/rejected": 0.13288016617298126, "logps/chosen": -1207.83203125, "logps/rejected": -1462.1517333984375, "loss": 1.7114, "rewards/accuracies": 0.637499988079071, "rewards/chosen": -0.864331841468811, "rewards/margins": 0.288027286529541, "rewards/rejected": -1.1523590087890625, "step": 1320 }, { "epoch": 0.71, "learning_rate": 1.1807777065131002e-06, "logits/chosen": 0.07271184772253036, "logits/rejected": 0.13265785574913025, "logps/chosen": -1237.49267578125, "logps/rejected": -1638.8544921875, "loss": 1.5222, "rewards/accuracies": 0.6937500238418579, "rewards/chosen": -0.8673235774040222, "rewards/margins": 0.3971632122993469, "rewards/rejected": -1.2644867897033691, "step": 1330 }, { "epoch": 0.71, "learning_rate": 1.1414623905380012e-06, "logits/chosen": 0.07448406517505646, "logits/rejected": 0.14864052832126617, "logps/chosen": -1119.6168212890625, "logps/rejected": -1548.5118408203125, "loss": 1.4952, "rewards/accuracies": 0.706250011920929, "rewards/chosen": -0.7575954794883728, "rewards/margins": 0.4239155352115631, "rewards/rejected": -1.1815111637115479, "step": 1340 }, { "epoch": 0.72, "learning_rate": 1.1026181914754388e-06, "logits/chosen": 0.09954806417226791, "logits/rejected": 0.16613879799842834, "logps/chosen": -1136.9033203125, "logps/rejected": -1430.72900390625, "loss": 1.6896, "rewards/accuracies": 0.6499999761581421, "rewards/chosen": -0.7874740362167358, "rewards/margins": 0.2991321086883545, "rewards/rejected": -1.0866060256958008, "step": 1350 }, { "epoch": 0.73, "learning_rate": 1.0642585798094136e-06, "logits/chosen": 0.07558660209178925, "logits/rejected": 0.1294403225183487, "logps/chosen": -1061.685302734375, "logps/rejected": -1429.032470703125, "loss": 1.5992, "rewards/accuracies": 0.6937500238418579, "rewards/chosen": -0.7224678993225098, "rewards/margins": 0.3703506588935852, "rewards/rejected": -1.0928184986114502, "step": 1360 }, { "epoch": 0.73, "learning_rate": 1.0263968579775522e-06, "logits/chosen": 0.06903640925884247, "logits/rejected": 0.14249339699745178, "logps/chosen": -1082.8212890625, "logps/rejected": -1504.9525146484375, "loss": 1.2627, "rewards/accuracies": 0.7562500238418579, "rewards/chosen": -0.7245824337005615, "rewards/margins": 0.4476887583732605, "rewards/rejected": -1.1722711324691772, "step": 1370 }, { "epoch": 0.74, "learning_rate": 9.89046155758058e-07, "logits/chosen": 0.08827939629554749, "logits/rejected": 0.15384915471076965, "logps/chosen": -1257.3944091796875, "logps/rejected": -1638.11083984375, "loss": 1.6438, "rewards/accuracies": 0.706250011920929, "rewards/chosen": -0.8749507665634155, "rewards/margins": 0.39814838767051697, "rewards/rejected": -1.2730991840362549, "step": 1380 }, { "epoch": 0.74, "learning_rate": 9.52219425716534e-07, "logits/chosen": 0.07912211120128632, "logits/rejected": 0.10791107267141342, "logps/chosen": -1195.5802001953125, "logps/rejected": -1549.964111328125, "loss": 1.6688, "rewards/accuracies": 0.6625000238418579, "rewards/chosen": -0.8793632388114929, "rewards/margins": 0.35304269194602966, "rewards/rejected": -1.2324059009552002, "step": 1390 }, { "epoch": 0.75, "learning_rate": 9.15929438714262e-07, "logits/chosen": 0.10573384910821915, "logits/rejected": 0.1559595763683319, "logps/chosen": -1081.0791015625, "logps/rejected": -1481.296142578125, "loss": 1.4667, "rewards/accuracies": 0.668749988079071, "rewards/chosen": -0.7587048411369324, "rewards/margins": 0.4136192798614502, "rewards/rejected": -1.1723239421844482, "step": 1400 }, { "epoch": 0.75, "learning_rate": 8.801887794794911e-07, "logits/chosen": 0.05759074538946152, "logits/rejected": 0.13223622739315033, "logps/chosen": -1182.5169677734375, "logps/rejected": -1658.713134765625, "loss": 1.3708, "rewards/accuracies": 0.706250011920929, "rewards/chosen": -0.8255613446235657, "rewards/margins": 0.4847756028175354, "rewards/rejected": -1.3103368282318115, "step": 1410 }, { "epoch": 0.76, "learning_rate": 8.450098422432787e-07, "logits/chosen": 0.08058818429708481, "logits/rejected": 0.17761366069316864, "logps/chosen": -1177.451416015625, "logps/rejected": -1569.4161376953125, "loss": 1.7648, "rewards/accuracies": 0.6937500238418579, "rewards/chosen": -0.8251779675483704, "rewards/margins": 0.39350980520248413, "rewards/rejected": -1.218687653541565, "step": 1420 }, { "epoch": 0.76, "learning_rate": 8.104048264413858e-07, "logits/chosen": 0.08303747326135635, "logits/rejected": 0.157932847738266, "logps/chosen": -1107.2733154296875, "logps/rejected": -1565.2877197265625, "loss": 1.3706, "rewards/accuracies": 0.7437499761581421, "rewards/chosen": -0.769390881061554, "rewards/margins": 0.4508587419986725, "rewards/rejected": -1.2202496528625488, "step": 1430 }, { "epoch": 0.77, "learning_rate": 7.763857324837321e-07, "logits/chosen": 0.0844537764787674, "logits/rejected": 0.13607878983020782, "logps/chosen": -1097.9794921875, "logps/rejected": -1437.67138671875, "loss": 1.6685, "rewards/accuracies": 0.6625000238418579, "rewards/chosen": -0.7822902798652649, "rewards/margins": 0.364152729511261, "rewards/rejected": -1.1464428901672363, "step": 1440 }, { "epoch": 0.77, "learning_rate": 7.429643575928605e-07, "logits/chosen": 0.057485781610012054, "logits/rejected": 0.14504484832286835, "logps/chosen": -1145.53515625, "logps/rejected": -1544.49462890625, "loss": 1.6211, "rewards/accuracies": 0.737500011920929, "rewards/chosen": -0.7827796936035156, "rewards/margins": 0.4238261580467224, "rewards/rejected": -1.2066057920455933, "step": 1450 }, { "epoch": 0.78, "learning_rate": 7.101522917128709e-07, "logits/chosen": 0.09009334444999695, "logits/rejected": 0.12808983027935028, "logps/chosen": -1061.626708984375, "logps/rejected": -1464.641357421875, "loss": 1.4222, "rewards/accuracies": 0.7250000238418579, "rewards/chosen": -0.7373768091201782, "rewards/margins": 0.39346086978912354, "rewards/rejected": -1.1308376789093018, "step": 1460 }, { "epoch": 0.78, "learning_rate": 6.779609134902312e-07, "logits/chosen": 0.11229290068149567, "logits/rejected": 0.16109147667884827, "logps/chosen": -1014.5134887695312, "logps/rejected": -1326.9337158203125, "loss": 1.724, "rewards/accuracies": 0.643750011920929, "rewards/chosen": -0.6920040845870972, "rewards/margins": 0.3285384774208069, "rewards/rejected": -1.0205425024032593, "step": 1470 }, { "epoch": 0.79, "learning_rate": 6.464013863278629e-07, "logits/chosen": 0.12503568828105927, "logits/rejected": 0.19441263377666473, "logps/chosen": -1199.6285400390625, "logps/rejected": -1496.771240234375, "loss": 1.7378, "rewards/accuracies": 0.6499999761581421, "rewards/chosen": -0.8214676976203918, "rewards/margins": 0.3142891228199005, "rewards/rejected": -1.1357567310333252, "step": 1480 }, { "epoch": 0.79, "learning_rate": 6.154846545138696e-07, "logits/chosen": 0.09698246419429779, "logits/rejected": 0.17293763160705566, "logps/chosen": -1231.1141357421875, "logps/rejected": -1501.1031494140625, "loss": 1.9488, "rewards/accuracies": 0.625, "rewards/chosen": -0.8393710851669312, "rewards/margins": 0.2999555170536041, "rewards/rejected": -1.1393266916275024, "step": 1490 }, { "epoch": 0.8, "learning_rate": 5.852214394262515e-07, "logits/chosen": 0.09356184303760529, "logits/rejected": 0.12046798318624496, "logps/chosen": -1084.208740234375, "logps/rejected": -1468.5865478515625, "loss": 1.6395, "rewards/accuracies": 0.6812499761581421, "rewards/chosen": -0.7786747217178345, "rewards/margins": 0.34865671396255493, "rewards/rejected": -1.1273314952850342, "step": 1500 }, { "epoch": 0.81, "learning_rate": 5.556222358149191e-07, "logits/chosen": 0.06989214569330215, "logits/rejected": 0.12371733039617538, "logps/chosen": -1111.44775390625, "logps/rejected": -1397.4815673828125, "loss": 1.6382, "rewards/accuracies": 0.6625000238418579, "rewards/chosen": -0.7657614946365356, "rewards/margins": 0.3277229070663452, "rewards/rejected": -1.0934845209121704, "step": 1510 }, { "epoch": 0.81, "learning_rate": 5.266973081622992e-07, "logits/chosen": 0.09623502939939499, "logits/rejected": 0.12523935735225677, "logps/chosen": -1027.00634765625, "logps/rejected": -1464.6611328125, "loss": 1.5786, "rewards/accuracies": 0.6875, "rewards/chosen": -0.7143763899803162, "rewards/margins": 0.40375128388404846, "rewards/rejected": -1.118127703666687, "step": 1520 }, { "epoch": 0.82, "learning_rate": 4.984566871237942e-07, "logits/chosen": 0.12962420284748077, "logits/rejected": 0.18393470346927643, "logps/chosen": -1017.5735473632812, "logps/rejected": -1400.67138671875, "loss": 1.5976, "rewards/accuracies": 0.668749988079071, "rewards/chosen": -0.7108270525932312, "rewards/margins": 0.38168221712112427, "rewards/rejected": -1.092509150505066, "step": 1530 }, { "epoch": 0.82, "learning_rate": 4.709101660493251e-07, "logits/chosen": 0.11588941514492035, "logits/rejected": 0.199117973446846, "logps/chosen": -1026.5185546875, "logps/rejected": -1404.314208984375, "loss": 1.5534, "rewards/accuracies": 0.71875, "rewards/chosen": -0.6769729852676392, "rewards/margins": 0.3804163932800293, "rewards/rejected": -1.0573893785476685, "step": 1540 }, { "epoch": 0.83, "learning_rate": 4.440672975871743e-07, "logits/chosen": 0.08243189007043839, "logits/rejected": 0.17241016030311584, "logps/chosen": -1097.091552734375, "logps/rejected": -1497.203857421875, "loss": 1.362, "rewards/accuracies": 0.7437499761581421, "rewards/chosen": -0.722274661064148, "rewards/margins": 0.4388486444950104, "rewards/rejected": -1.1611233949661255, "step": 1550 }, { "epoch": 0.83, "learning_rate": 4.1793739037129134e-07, "logits/chosen": 0.08427461236715317, "logits/rejected": 0.15939496457576752, "logps/chosen": -1080.250732421875, "logps/rejected": -1431.294677734375, "loss": 1.5649, "rewards/accuracies": 0.7124999761581421, "rewards/chosen": -0.7504817247390747, "rewards/margins": 0.3592514097690582, "rewards/rejected": -1.1097332239151, "step": 1560 }, { "epoch": 0.84, "learning_rate": 3.9252950579322405e-07, "logits/chosen": 0.09509806334972382, "logits/rejected": 0.14530691504478455, "logps/chosen": -1005.5162353515625, "logps/rejected": -1375.9266357421875, "loss": 1.5294, "rewards/accuracies": 0.6937500238418579, "rewards/chosen": -0.6889379620552063, "rewards/margins": 0.3749217092990875, "rewards/rejected": -1.0638595819473267, "step": 1570 }, { "epoch": 0.84, "learning_rate": 3.6785245485978864e-07, "logits/chosen": 0.13486482203006744, "logits/rejected": 0.12031221389770508, "logps/chosen": -1113.1337890625, "logps/rejected": -1436.71337890625, "loss": 1.7315, "rewards/accuracies": 0.6312500238418579, "rewards/chosen": -0.7942808866500854, "rewards/margins": 0.31816208362579346, "rewards/rejected": -1.112442970275879, "step": 1580 }, { "epoch": 0.85, "learning_rate": 3.43914795137566e-07, "logits/chosen": 0.12698054313659668, "logits/rejected": 0.20286865532398224, "logps/chosen": -1038.547607421875, "logps/rejected": -1364.681396484375, "loss": 1.6174, "rewards/accuracies": 0.6625000238418579, "rewards/chosen": -0.7222835421562195, "rewards/margins": 0.33720117807388306, "rewards/rejected": -1.0594847202301025, "step": 1590 }, { "epoch": 0.85, "learning_rate": 3.207248277852901e-07, "logits/chosen": 0.11406983435153961, "logits/rejected": 0.19401147961616516, "logps/chosen": -1013.9378051757812, "logps/rejected": -1410.285888671875, "loss": 1.5172, "rewards/accuracies": 0.606249988079071, "rewards/chosen": -0.6757684946060181, "rewards/margins": 0.38184699416160583, "rewards/rejected": -1.0576155185699463, "step": 1600 }, { "epoch": 0.86, "learning_rate": 2.9829059467515074e-07, "logits/chosen": 0.11214902251958847, "logits/rejected": 0.13649222254753113, "logps/chosen": -1164.2906494140625, "logps/rejected": -1486.5689697265625, "loss": 1.6461, "rewards/accuracies": 0.6625000238418579, "rewards/chosen": -0.8062076568603516, "rewards/margins": 0.3245384097099304, "rewards/rejected": -1.1307461261749268, "step": 1610 }, { "epoch": 0.86, "learning_rate": 2.766198756040153e-07, "logits/chosen": 0.11051913350820541, "logits/rejected": 0.1855718195438385, "logps/chosen": -1100.14501953125, "logps/rejected": -1472.6260986328125, "loss": 1.5808, "rewards/accuracies": 0.706250011920929, "rewards/chosen": -0.7349221706390381, "rewards/margins": 0.3916444778442383, "rewards/rejected": -1.1265666484832764, "step": 1620 }, { "epoch": 0.87, "learning_rate": 2.5572018559553155e-07, "logits/chosen": 0.1027497872710228, "logits/rejected": 0.13441568613052368, "logps/chosen": -1062.0927734375, "logps/rejected": -1416.841064453125, "loss": 1.5722, "rewards/accuracies": 0.675000011920929, "rewards/chosen": -0.7326242327690125, "rewards/margins": 0.36529573798179626, "rewards/rejected": -1.0979200601577759, "step": 1630 }, { "epoch": 0.87, "learning_rate": 2.3559877229404864e-07, "logits/chosen": 0.0654052123427391, "logits/rejected": 0.14604170620441437, "logps/chosen": -1041.9859619140625, "logps/rejected": -1462.3514404296875, "loss": 1.3094, "rewards/accuracies": 0.737500011920929, "rewards/chosen": -0.6855828166007996, "rewards/margins": 0.435748428106308, "rewards/rejected": -1.1213312149047852, "step": 1640 }, { "epoch": 0.88, "learning_rate": 2.1626261345126576e-07, "logits/chosen": 0.10176499933004379, "logits/rejected": 0.12414362281560898, "logps/chosen": -1152.2626953125, "logps/rejected": -1480.9588623046875, "loss": 1.5617, "rewards/accuracies": 0.6875, "rewards/chosen": -0.779994547367096, "rewards/margins": 0.3541329503059387, "rewards/rejected": -1.1341274976730347, "step": 1650 }, { "epoch": 0.89, "learning_rate": 1.9771841450646505e-07, "logits/chosen": 0.09231498837471008, "logits/rejected": 0.15306654572486877, "logps/chosen": -1075.6600341796875, "logps/rejected": -1482.9368896484375, "loss": 1.4986, "rewards/accuracies": 0.75, "rewards/chosen": -0.7160147428512573, "rewards/margins": 0.4125767648220062, "rewards/rejected": -1.1285914182662964, "step": 1660 }, { "epoch": 0.89, "learning_rate": 1.7997260626118758e-07, "logits/chosen": 0.07871339470148087, "logits/rejected": 0.1630566418170929, "logps/chosen": -1022.990234375, "logps/rejected": -1364.323974609375, "loss": 1.532, "rewards/accuracies": 0.706250011920929, "rewards/chosen": -0.6643663048744202, "rewards/margins": 0.3671983480453491, "rewards/rejected": -1.031564474105835, "step": 1670 }, { "epoch": 0.9, "learning_rate": 1.6303134264914365e-07, "logits/chosen": 0.05614648386836052, "logits/rejected": 0.13106751441955566, "logps/chosen": -1096.6705322265625, "logps/rejected": -1407.255615234375, "loss": 1.5868, "rewards/accuracies": 0.6812499761581421, "rewards/chosen": -0.7051296234130859, "rewards/margins": 0.3592469096183777, "rewards/rejected": -1.0643765926361084, "step": 1680 }, { "epoch": 0.9, "learning_rate": 1.469004986021355e-07, "logits/chosen": 0.08326787501573563, "logits/rejected": 0.14079895615577698, "logps/chosen": -1091.39697265625, "logps/rejected": -1426.65283203125, "loss": 1.7203, "rewards/accuracies": 0.668749988079071, "rewards/chosen": -0.73790043592453, "rewards/margins": 0.3496142029762268, "rewards/rejected": -1.0875146389007568, "step": 1690 }, { "epoch": 0.91, "learning_rate": 1.315856680127367e-07, "logits/chosen": 0.07813490927219391, "logits/rejected": 0.18866008520126343, "logps/chosen": -1085.9488525390625, "logps/rejected": -1426.083251953125, "loss": 1.7138, "rewards/accuracies": 0.637499988079071, "rewards/chosen": -0.743714451789856, "rewards/margins": 0.342837393283844, "rewards/rejected": -1.0865519046783447, "step": 1700 }, { "epoch": 0.91, "learning_rate": 1.1709216179442817e-07, "logits/chosen": 0.07439470291137695, "logits/rejected": 0.14408041536808014, "logps/chosen": -1104.112548828125, "logps/rejected": -1494.4288330078125, "loss": 1.3662, "rewards/accuracies": 0.75, "rewards/chosen": -0.7067037224769592, "rewards/margins": 0.4223453104496002, "rewards/rejected": -1.1290490627288818, "step": 1710 }, { "epoch": 0.92, "learning_rate": 1.0342500603986421e-07, "logits/chosen": 0.06722331047058105, "logits/rejected": 0.10560252517461777, "logps/chosen": -989.7840576171875, "logps/rejected": -1307.6107177734375, "loss": 1.7934, "rewards/accuracies": 0.699999988079071, "rewards/chosen": -0.6622638702392578, "rewards/margins": 0.3219539523124695, "rewards/rejected": -0.9842178225517273, "step": 1720 }, { "epoch": 0.92, "learning_rate": 9.058894027791643e-08, "logits/chosen": 0.08770893514156342, "logits/rejected": 0.15539699792861938, "logps/chosen": -1029.530517578125, "logps/rejected": -1314.664306640625, "loss": 1.8332, "rewards/accuracies": 0.6187499761581421, "rewards/chosen": -0.6983517408370972, "rewards/margins": 0.2923336625099182, "rewards/rejected": -0.9906853437423706, "step": 1730 }, { "epoch": 0.93, "learning_rate": 7.858841583008592e-08, "logits/chosen": 0.1337372064590454, "logits/rejected": 0.17292381823062897, "logps/chosen": -989.8046875, "logps/rejected": -1368.048095703125, "loss": 1.5287, "rewards/accuracies": 0.6875, "rewards/chosen": -0.6523457765579224, "rewards/margins": 0.3857383131980896, "rewards/rejected": -1.0380841493606567, "step": 1740 }, { "epoch": 0.93, "learning_rate": 6.742759426686313e-08, "logits/chosen": 0.12590888142585754, "logits/rejected": 0.17510025203227997, "logps/chosen": -1068.725830078125, "logps/rejected": -1355.000244140625, "loss": 1.5962, "rewards/accuracies": 0.706250011920929, "rewards/chosen": -0.6910558938980103, "rewards/margins": 0.33794966340065, "rewards/rejected": -1.029005527496338, "step": 1750 }, { "epoch": 0.94, "learning_rate": 5.7110345964571104e-08, "logits/chosen": 0.09558887034654617, "logits/rejected": 0.15812750160694122, "logps/chosen": -1157.002685546875, "logps/rejected": -1459.370849609375, "loss": 1.5588, "rewards/accuracies": 0.699999988079071, "rewards/chosen": -0.7800796627998352, "rewards/margins": 0.33304229378700256, "rewards/rejected": -1.1131219863891602, "step": 1760 }, { "epoch": 0.94, "learning_rate": 4.764024876318357e-08, "logits/chosen": 0.08710858970880508, "logits/rejected": 0.16179919242858887, "logps/chosen": -1023.4801025390625, "logps/rejected": -1355.501953125, "loss": 1.711, "rewards/accuracies": 0.6499999761581421, "rewards/chosen": -0.7188773155212402, "rewards/margins": 0.3240003287792206, "rewards/rejected": -1.0428775548934937, "step": 1770 }, { "epoch": 0.95, "learning_rate": 3.902058672559633e-08, "logits/chosen": 0.13087372481822968, "logits/rejected": 0.18574543297290802, "logps/chosen": -1036.978759765625, "logps/rejected": -1390.5789794921875, "loss": 1.5651, "rewards/accuracies": 0.6875, "rewards/chosen": -0.6995924711227417, "rewards/margins": 0.35320958495140076, "rewards/rejected": -1.0528020858764648, "step": 1780 }, { "epoch": 0.95, "learning_rate": 3.125434899876933e-08, "logits/chosen": 0.09127840399742126, "logits/rejected": 0.18651551008224487, "logps/chosen": -1081.856201171875, "logps/rejected": -1476.410888671875, "loss": 1.5536, "rewards/accuracies": 0.6875, "rewards/chosen": -0.7147141098976135, "rewards/margins": 0.4145401120185852, "rewards/rejected": -1.1292542219161987, "step": 1790 }, { "epoch": 0.96, "learning_rate": 2.4344228777145873e-08, "logits/chosen": 0.0788322165608406, "logits/rejected": 0.1238178238272667, "logps/chosen": -986.0138549804688, "logps/rejected": -1424.9691162109375, "loss": 1.6491, "rewards/accuracies": 0.6875, "rewards/chosen": -0.6681248545646667, "rewards/margins": 0.41846686601638794, "rewards/rejected": -1.0865918397903442, "step": 1800 }, { "epoch": 0.97, "learning_rate": 1.829262236869772e-08, "logits/chosen": 0.11619732528924942, "logits/rejected": 0.17705723643302917, "logps/chosen": -1101.0535888671875, "logps/rejected": -1451.33544921875, "loss": 1.5375, "rewards/accuracies": 0.6937500238418579, "rewards/chosen": -0.7447971105575562, "rewards/margins": 0.3711633086204529, "rewards/rejected": -1.1159603595733643, "step": 1810 }, { "epoch": 0.97, "learning_rate": 1.3101628363929586e-08, "logits/chosen": 0.07590712606906891, "logits/rejected": 0.10692886263132095, "logps/chosen": -1034.286376953125, "logps/rejected": -1416.898681640625, "loss": 1.4333, "rewards/accuracies": 0.6937500238418579, "rewards/chosen": -0.6867724657058716, "rewards/margins": 0.39223819971084595, "rewards/rejected": -1.0790107250213623, "step": 1820 }, { "epoch": 0.98, "learning_rate": 8.773046908123195e-09, "logits/chosen": 0.11580286175012589, "logits/rejected": 0.17976327240467072, "logps/chosen": -1056.7957763671875, "logps/rejected": -1410.120361328125, "loss": 1.5549, "rewards/accuracies": 0.7250000238418579, "rewards/chosen": -0.6937663555145264, "rewards/margins": 0.37733033299446106, "rewards/rejected": -1.071096658706665, "step": 1830 }, { "epoch": 0.98, "learning_rate": 5.308379077080817e-09, "logits/chosen": 0.08766238391399384, "logits/rejected": 0.15721826255321503, "logps/chosen": -1196.5023193359375, "logps/rejected": -1440.4573974609375, "loss": 1.7298, "rewards/accuracies": 0.668749988079071, "rewards/chosen": -0.7777877449989319, "rewards/margins": 0.31406041979789734, "rewards/rejected": -1.0918481349945068, "step": 1840 }, { "epoch": 0.99, "learning_rate": 2.7088263565760996e-09, "logits/chosen": 0.05079632252454758, "logits/rejected": 0.1428813636302948, "logps/chosen": -1076.935302734375, "logps/rejected": -1446.3316650390625, "loss": 1.482, "rewards/accuracies": 0.7124999761581421, "rewards/chosen": -0.7342230081558228, "rewards/margins": 0.37562787532806396, "rewards/rejected": -1.1098508834838867, "step": 1850 }, { "epoch": 0.99, "learning_rate": 9.752902257023633e-10, "logits/chosen": 0.07703147828578949, "logits/rejected": 0.13958509266376495, "logps/chosen": -1000.0032348632812, "logps/rejected": -1295.458984375, "loss": 1.8327, "rewards/accuracies": 0.6000000238418579, "rewards/chosen": -0.6804636716842651, "rewards/margins": 0.2884565591812134, "rewards/rejected": -0.9689203500747681, "step": 1860 }, { "epoch": 1.0, "learning_rate": 1.083718442532189e-10, "logits/chosen": 0.09295358508825302, "logits/rejected": 0.16223040223121643, "logps/chosen": -1160.269287109375, "logps/rejected": -1494.4976806640625, "loss": 1.5841, "rewards/accuracies": 0.643750011920929, "rewards/chosen": -0.7781329154968262, "rewards/margins": 0.3674696981906891, "rewards/rejected": -1.1456025838851929, "step": 1870 }, { "epoch": 1.0, "step": 1875, "total_flos": 0.0, "train_loss": 1.579099040222168, "train_runtime": 16673.7788, "train_samples_per_second": 1.799, "train_steps_per_second": 0.112 } ], "logging_steps": 10, "max_steps": 1875, "num_input_tokens_seen": 0, "num_train_epochs": 1, "save_steps": 100, "total_flos": 0.0, "train_batch_size": 4, "trial_name": null, "trial_params": null }