diff --git "a/checkpoint-7000/trainer_state.json" "b/checkpoint-7000/trainer_state.json" new file mode 100644--- /dev/null +++ "b/checkpoint-7000/trainer_state.json" @@ -0,0 +1,9821 @@ +{ + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.7683653028182542, + "eval_steps": 500, + "global_step": 7000, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.0, + "learning_rate": 9.999970269475589e-06, + "logits/chosen": -2.748720645904541, + "logits/rejected": -3.5176024436950684, + "logps/chosen": -165.87062072753906, + "logps/rejected": -23.556869506835938, + "loss": 1.5949, + "rewards/accuracies": 0.612500011920929, + "rewards/chosen": 11.860421180725098, + "rewards/margins": 2.147804021835327, + "rewards/rejected": 9.712616920471191, + "step": 10 + }, + { + "epoch": 0.0, + "learning_rate": 9.999881078255916e-06, + "logits/chosen": -3.261986255645752, + "logits/rejected": -4.041715145111084, + "logps/chosen": -153.11898803710938, + "logps/rejected": -24.845947265625, + "loss": 1.6208, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": 12.161478042602539, + "rewards/margins": 2.482882499694824, + "rewards/rejected": 9.678595542907715, + "step": 20 + }, + { + "epoch": 0.0, + "learning_rate": 9.99973242740166e-06, + "logits/chosen": -2.687460422515869, + "logits/rejected": -3.837062358856201, + "logps/chosen": -148.68341064453125, + "logps/rejected": -34.57600402832031, + "loss": 1.497, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": 12.840588569641113, + "rewards/margins": 2.2458953857421875, + "rewards/rejected": 10.59469223022461, + "step": 30 + }, + { + "epoch": 0.0, + "learning_rate": 9.99954780509173e-06, + "logits/chosen": -2.9765782356262207, + "logits/rejected": -4.164349555969238, + "logps/chosen": -201.6632537841797, + "logps/rejected": -29.7652645111084, + "loss": 1.8816, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": 12.654102325439453, + "rewards/margins": 2.569594383239746, + "rewards/rejected": 10.084507942199707, + "step": 40 + }, + { + "epoch": 0.01, + "learning_rate": 9.999286186386431e-06, + "logits/chosen": -3.153574228286743, + "logits/rejected": -3.934051036834717, + "logps/chosen": -138.03778076171875, + "logps/rejected": -30.519153594970703, + "loss": 1.9103, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": 11.428102493286133, + "rewards/margins": 1.7032209634780884, + "rewards/rejected": 9.724882125854492, + "step": 50 + }, + { + "epoch": 0.01, + "learning_rate": 9.998965115121132e-06, + "logits/chosen": -3.201958417892456, + "logits/rejected": -4.274596691131592, + "logps/chosen": -166.93260192871094, + "logps/rejected": -35.59614944458008, + "loss": 2.0164, + "rewards/accuracies": 0.5625, + "rewards/chosen": 11.013410568237305, + "rewards/margins": 1.4899368286132812, + "rewards/rejected": 9.523473739624023, + "step": 60 + }, + { + "epoch": 0.01, + "learning_rate": 9.998584595114074e-06, + "logits/chosen": -3.6508994102478027, + "logits/rejected": -4.988117694854736, + "logps/chosen": -192.28738403320312, + "logps/rejected": -39.88438415527344, + "loss": 0.7081, + "rewards/accuracies": 0.7875000238418579, + "rewards/chosen": 12.313508987426758, + "rewards/margins": 4.361058712005615, + "rewards/rejected": 7.952448844909668, + "step": 70 + }, + { + "epoch": 0.01, + "learning_rate": 9.998144630890485e-06, + "logits/chosen": -3.2785956859588623, + "logits/rejected": -4.609368801116943, + "logps/chosen": -177.5666961669922, + "logps/rejected": -48.05603790283203, + "loss": 1.1081, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": 12.321931838989258, + "rewards/margins": 4.14510440826416, + "rewards/rejected": 8.176828384399414, + "step": 80 + }, + { + "epoch": 0.01, + "learning_rate": 9.997645227682513e-06, + "logits/chosen": -3.4411368370056152, + "logits/rejected": -4.750131607055664, + "logps/chosen": -163.21450805664062, + "logps/rejected": -53.2952880859375, + "loss": 1.1417, + "rewards/accuracies": 0.737500011920929, + "rewards/chosen": 11.179890632629395, + "rewards/margins": 3.490216016769409, + "rewards/rejected": 7.689675331115723, + "step": 90 + }, + { + "epoch": 0.01, + "learning_rate": 9.997086391429161e-06, + "logits/chosen": -3.429443836212158, + "logits/rejected": -4.5238776206970215, + "logps/chosen": -176.43289184570312, + "logps/rejected": -65.92268371582031, + "loss": 0.9445, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": 10.390297889709473, + "rewards/margins": 3.6286094188690186, + "rewards/rejected": 6.761689186096191, + "step": 100 + }, + { + "epoch": 0.01, + "learning_rate": 9.99646812877623e-06, + "logits/chosen": -3.4572105407714844, + "logits/rejected": -4.460841655731201, + "logps/chosen": -170.5079803466797, + "logps/rejected": -68.17359161376953, + "loss": 0.6313, + "rewards/accuracies": 0.8125, + "rewards/chosen": 10.23057746887207, + "rewards/margins": 4.633530616760254, + "rewards/rejected": 5.5970458984375, + "step": 110 + }, + { + "epoch": 0.01, + "learning_rate": 9.995790447076232e-06, + "logits/chosen": -3.5251846313476562, + "logits/rejected": -4.237984657287598, + "logps/chosen": -163.4308624267578, + "logps/rejected": -69.18915557861328, + "loss": 0.8116, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": 10.12580394744873, + "rewards/margins": 4.935164451599121, + "rewards/rejected": 5.190640449523926, + "step": 120 + }, + { + "epoch": 0.01, + "learning_rate": 9.995053354388295e-06, + "logits/chosen": -3.2857518196105957, + "logits/rejected": -4.0882792472839355, + "logps/chosen": -211.8911590576172, + "logps/rejected": -76.1227035522461, + "loss": 0.1832, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 10.317492485046387, + "rewards/margins": 5.812389373779297, + "rewards/rejected": 4.505103588104248, + "step": 130 + }, + { + "epoch": 0.02, + "learning_rate": 9.994256859478082e-06, + "logits/chosen": -3.0444934368133545, + "logits/rejected": -3.676633358001709, + "logps/chosen": -229.4088134765625, + "logps/rejected": -82.24313354492188, + "loss": 0.3931, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 10.056417465209961, + "rewards/margins": 5.8769307136535645, + "rewards/rejected": 4.17948579788208, + "step": 140 + }, + { + "epoch": 0.02, + "learning_rate": 9.993489232972927e-06, + "logits/chosen": -3.0874276161193848, + "logits/rejected": -3.6212425231933594, + "logps/chosen": -163.163330078125, + "logps/rejected": -76.16889953613281, + "loss": 0.7483, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": 9.392515182495117, + "rewards/margins": 5.074815273284912, + "rewards/rejected": 4.3177008628845215, + "step": 150 + }, + { + "epoch": 0.02, + "learning_rate": 9.992579900519751e-06, + "logits/chosen": -2.9442169666290283, + "logits/rejected": -3.488318681716919, + "logps/chosen": -202.21347045898438, + "logps/rejected": -84.56549072265625, + "loss": 0.3173, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 9.955198287963867, + "rewards/margins": 6.719505310058594, + "rewards/rejected": 3.2356934547424316, + "step": 160 + }, + { + "epoch": 0.02, + "learning_rate": 9.99161119525913e-06, + "logits/chosen": -2.819887161254883, + "logits/rejected": -3.2742202281951904, + "logps/chosen": -212.2724151611328, + "logps/rejected": -90.73429870605469, + "loss": 0.3464, + "rewards/accuracies": 0.875, + "rewards/chosen": 9.708670616149902, + "rewards/margins": 5.832002639770508, + "rewards/rejected": 3.8766684532165527, + "step": 170 + }, + { + "epoch": 0.02, + "learning_rate": 9.990688606280938e-06, + "logits/chosen": -2.7344448566436768, + "logits/rejected": -2.890084743499756, + "logps/chosen": -184.69085693359375, + "logps/rejected": -89.32521057128906, + "loss": 0.6789, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 9.630158424377441, + "rewards/margins": 5.920450687408447, + "rewards/rejected": 3.7097079753875732, + "step": 180 + }, + { + "epoch": 0.02, + "learning_rate": 9.989607125007361e-06, + "logits/chosen": -2.5483996868133545, + "logits/rejected": -2.572554588317871, + "logps/chosen": -172.837158203125, + "logps/rejected": -92.3455581665039, + "loss": 0.4471, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": 8.161615371704102, + "rewards/margins": 4.883408069610596, + "rewards/rejected": 3.2782065868377686, + "step": 190 + }, + { + "epoch": 0.02, + "learning_rate": 9.98846630627921e-06, + "logits/chosen": -2.7279348373413086, + "logits/rejected": -2.8250045776367188, + "logps/chosen": -216.2030487060547, + "logps/rejected": -94.36392211914062, + "loss": 0.1686, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 9.2470703125, + "rewards/margins": 6.067578315734863, + "rewards/rejected": 3.1794912815093994, + "step": 200 + }, + { + "epoch": 0.02, + "learning_rate": 9.98726616366334e-06, + "logits/chosen": -2.476562261581421, + "logits/rejected": -2.579338550567627, + "logps/chosen": -169.55120849609375, + "logps/rejected": -101.41313171386719, + "loss": 0.3145, + "rewards/accuracies": 0.875, + "rewards/chosen": 8.830245018005371, + "rewards/margins": 5.8404154777526855, + "rewards/rejected": 2.9898295402526855, + "step": 210 + }, + { + "epoch": 0.02, + "learning_rate": 9.986006711432099e-06, + "logits/chosen": -2.6179986000061035, + "logits/rejected": -2.938459873199463, + "logps/chosen": -239.5026397705078, + "logps/rejected": -93.572998046875, + "loss": 0.136, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 10.406990051269531, + "rewards/margins": 7.794189453125, + "rewards/rejected": 2.612799882888794, + "step": 220 + }, + { + "epoch": 0.03, + "learning_rate": 9.984687964563159e-06, + "logits/chosen": -2.4739646911621094, + "logits/rejected": -2.5754473209381104, + "logps/chosen": -178.31881713867188, + "logps/rejected": -95.78547668457031, + "loss": 0.2827, + "rewards/accuracies": 0.9624999761581421, + "rewards/chosen": 8.740915298461914, + "rewards/margins": 6.064324855804443, + "rewards/rejected": 2.676591396331787, + "step": 230 + }, + { + "epoch": 0.03, + "learning_rate": 9.983309938739332e-06, + "logits/chosen": -2.6331839561462402, + "logits/rejected": -2.8605971336364746, + "logps/chosen": -229.9063720703125, + "logps/rejected": -90.2779541015625, + "loss": 0.0817, + "rewards/accuracies": 0.9750000238418579, + "rewards/chosen": 10.207468032836914, + "rewards/margins": 7.982739448547363, + "rewards/rejected": 2.2247281074523926, + "step": 240 + }, + { + "epoch": 0.03, + "learning_rate": 9.98187265034839e-06, + "logits/chosen": -2.3521595001220703, + "logits/rejected": -2.5626819133758545, + "logps/chosen": -190.8962860107422, + "logps/rejected": -104.4652099609375, + "loss": 0.2826, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 8.663321495056152, + "rewards/margins": 6.1150994300842285, + "rewards/rejected": 2.5482218265533447, + "step": 250 + }, + { + "epoch": 0.03, + "learning_rate": 9.980376116482871e-06, + "logits/chosen": -2.4073798656463623, + "logits/rejected": -2.531911849975586, + "logps/chosen": -204.7512664794922, + "logps/rejected": -108.78132629394531, + "loss": 0.2884, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 9.377767562866211, + "rewards/margins": 6.906778812408447, + "rewards/rejected": 2.470989227294922, + "step": 260 + }, + { + "epoch": 0.03, + "learning_rate": 9.978820354939866e-06, + "logits/chosen": -2.2742762565612793, + "logits/rejected": -2.1533379554748535, + "logps/chosen": -189.8150177001953, + "logps/rejected": -112.7627944946289, + "loss": 0.4643, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 9.204684257507324, + "rewards/margins": 6.9484758377075195, + "rewards/rejected": 2.2562096118927, + "step": 270 + }, + { + "epoch": 0.03, + "learning_rate": 9.97720538422082e-06, + "logits/chosen": -2.1974568367004395, + "logits/rejected": -2.233710765838623, + "logps/chosen": -198.45361328125, + "logps/rejected": -111.978515625, + "loss": 0.1247, + "rewards/accuracies": 0.9750000238418579, + "rewards/chosen": 10.231064796447754, + "rewards/margins": 8.133554458618164, + "rewards/rejected": 2.0975089073181152, + "step": 280 + }, + { + "epoch": 0.03, + "learning_rate": 9.975531223531301e-06, + "logits/chosen": -2.2490475177764893, + "logits/rejected": -2.30739164352417, + "logps/chosen": -187.04519653320312, + "logps/rejected": -109.82148742675781, + "loss": 0.1268, + "rewards/accuracies": 0.9375, + "rewards/chosen": 8.633919715881348, + "rewards/margins": 6.911574363708496, + "rewards/rejected": 1.7223446369171143, + "step": 290 + }, + { + "epoch": 0.03, + "learning_rate": 9.973797892780783e-06, + "logits/chosen": -2.035712718963623, + "logits/rejected": -1.961692452430725, + "logps/chosen": -179.4069366455078, + "logps/rejected": -116.109130859375, + "loss": 0.0938, + "rewards/accuracies": 0.9750000238418579, + "rewards/chosen": 9.934889793395996, + "rewards/margins": 8.173246383666992, + "rewards/rejected": 1.7616443634033203, + "step": 300 + }, + { + "epoch": 0.03, + "learning_rate": 9.972005412582397e-06, + "logits/chosen": -2.2436442375183105, + "logits/rejected": -1.8246514797210693, + "logps/chosen": -201.26980590820312, + "logps/rejected": -117.66944885253906, + "loss": 0.2073, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 8.626791954040527, + "rewards/margins": 7.419760227203369, + "rewards/rejected": 1.207031488418579, + "step": 310 + }, + { + "epoch": 0.04, + "learning_rate": 9.970153804252692e-06, + "logits/chosen": -1.8984334468841553, + "logits/rejected": -1.6126270294189453, + "logps/chosen": -185.87290954589844, + "logps/rejected": -117.4137954711914, + "loss": 0.278, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 8.371435165405273, + "rewards/margins": 7.151828765869141, + "rewards/rejected": 1.2196067571640015, + "step": 320 + }, + { + "epoch": 0.04, + "learning_rate": 9.968243089811384e-06, + "logits/chosen": -1.771867036819458, + "logits/rejected": -1.5103999376296997, + "logps/chosen": -192.2825469970703, + "logps/rejected": -118.695556640625, + "loss": 0.0729, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": 8.544290542602539, + "rewards/margins": 7.602395057678223, + "rewards/rejected": 0.9418962597846985, + "step": 330 + }, + { + "epoch": 0.04, + "learning_rate": 9.96627329198109e-06, + "logits/chosen": -1.924703598022461, + "logits/rejected": -1.3986445665359497, + "logps/chosen": -203.59628295898438, + "logps/rejected": -128.64163208007812, + "loss": 0.2888, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 8.449407577514648, + "rewards/margins": 7.650280952453613, + "rewards/rejected": 0.7991273999214172, + "step": 340 + }, + { + "epoch": 0.04, + "learning_rate": 9.96424443418706e-06, + "logits/chosen": -1.8718311786651611, + "logits/rejected": -1.552980661392212, + "logps/chosen": -181.55087280273438, + "logps/rejected": -120.4705581665039, + "loss": 0.0762, + "rewards/accuracies": 0.9750000238418579, + "rewards/chosen": 8.385054588317871, + "rewards/margins": 7.822892665863037, + "rewards/rejected": 0.5621613264083862, + "step": 350 + }, + { + "epoch": 0.04, + "learning_rate": 9.962156540556895e-06, + "logits/chosen": -1.7886946201324463, + "logits/rejected": -1.5114587545394897, + "logps/chosen": -183.34066772460938, + "logps/rejected": -117.0723648071289, + "loss": 0.1247, + "rewards/accuracies": 0.9624999761581421, + "rewards/chosen": 7.962329864501953, + "rewards/margins": 7.366267204284668, + "rewards/rejected": 0.5960620045661926, + "step": 360 + }, + { + "epoch": 0.04, + "learning_rate": 9.960009635920264e-06, + "logits/chosen": -1.788691520690918, + "logits/rejected": -1.3814653158187866, + "logps/chosen": -197.45651245117188, + "logps/rejected": -127.6612319946289, + "loss": 0.076, + "rewards/accuracies": 0.9750000238418579, + "rewards/chosen": 9.019963264465332, + "rewards/margins": 8.529821395874023, + "rewards/rejected": 0.49014148116111755, + "step": 370 + }, + { + "epoch": 0.04, + "learning_rate": 9.95780374580861e-06, + "logits/chosen": -1.5638988018035889, + "logits/rejected": -1.1953327655792236, + "logps/chosen": -189.52597045898438, + "logps/rejected": -122.5257797241211, + "loss": 0.2198, + "rewards/accuracies": 0.9375, + "rewards/chosen": 7.37133264541626, + "rewards/margins": 6.876678466796875, + "rewards/rejected": 0.4946538507938385, + "step": 380 + }, + { + "epoch": 0.04, + "learning_rate": 9.955538896454835e-06, + "logits/chosen": -1.5664819478988647, + "logits/rejected": -0.962997317314148, + "logps/chosen": -204.67657470703125, + "logps/rejected": -129.93968200683594, + "loss": 0.0446, + "rewards/accuracies": 0.9624999761581421, + "rewards/chosen": 7.449831485748291, + "rewards/margins": 7.785650730133057, + "rewards/rejected": -0.33581945300102234, + "step": 390 + }, + { + "epoch": 0.04, + "learning_rate": 9.95321511479301e-06, + "logits/chosen": -1.7048896551132202, + "logits/rejected": -1.1939642429351807, + "logps/chosen": -223.2345428466797, + "logps/rejected": -130.3861846923828, + "loss": 0.1745, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": 8.562383651733398, + "rewards/margins": 8.441825866699219, + "rewards/rejected": 0.12055821716785431, + "step": 400 + }, + { + "epoch": 0.05, + "learning_rate": 9.950832428458025e-06, + "logits/chosen": -1.2762507200241089, + "logits/rejected": -0.9500716924667358, + "logps/chosen": -195.8381805419922, + "logps/rejected": -124.5715560913086, + "loss": 0.1329, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": 7.099652290344238, + "rewards/margins": 7.511727809906006, + "rewards/rejected": -0.41207608580589294, + "step": 410 + }, + { + "epoch": 0.05, + "learning_rate": 9.948390865785295e-06, + "logits/chosen": -1.151587963104248, + "logits/rejected": -0.925134539604187, + "logps/chosen": -199.50967407226562, + "logps/rejected": -137.6339111328125, + "loss": 0.1983, + "rewards/accuracies": 0.9750000238418579, + "rewards/chosen": 7.476855278015137, + "rewards/margins": 7.712057590484619, + "rewards/rejected": -0.23520107567310333, + "step": 420 + }, + { + "epoch": 0.05, + "learning_rate": 9.94589045581039e-06, + "logits/chosen": -1.0321991443634033, + "logits/rejected": -0.8457671999931335, + "logps/chosen": -206.13467407226562, + "logps/rejected": -130.86544799804688, + "loss": 0.2554, + "rewards/accuracies": 0.9375, + "rewards/chosen": 7.223177433013916, + "rewards/margins": 7.446470737457275, + "rewards/rejected": -0.2232942134141922, + "step": 430 + }, + { + "epoch": 0.05, + "learning_rate": 9.943331228268711e-06, + "logits/chosen": -1.1853946447372437, + "logits/rejected": -0.7296536564826965, + "logps/chosen": -217.77304077148438, + "logps/rejected": -127.19853210449219, + "loss": 0.1477, + "rewards/accuracies": 0.9624999761581421, + "rewards/chosen": 7.660037040710449, + "rewards/margins": 8.213974952697754, + "rewards/rejected": -0.5539380311965942, + "step": 440 + }, + { + "epoch": 0.05, + "learning_rate": 9.94071321359513e-06, + "logits/chosen": -0.9670246243476868, + "logits/rejected": -0.314952552318573, + "logps/chosen": -225.93533325195312, + "logps/rejected": -154.71826171875, + "loss": 0.172, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": 7.689838409423828, + "rewards/margins": 8.839930534362793, + "rewards/rejected": -1.1500914096832275, + "step": 450 + }, + { + "epoch": 0.05, + "learning_rate": 9.938036442923624e-06, + "logits/chosen": -0.9485570788383484, + "logits/rejected": -0.33327311277389526, + "logps/chosen": -225.99398803710938, + "logps/rejected": -142.39173889160156, + "loss": 0.0579, + "rewards/accuracies": 0.9750000238418579, + "rewards/chosen": 7.543333530426025, + "rewards/margins": 8.586603164672852, + "rewards/rejected": -1.0432696342468262, + "step": 460 + }, + { + "epoch": 0.05, + "learning_rate": 9.935300948086915e-06, + "logits/chosen": -0.8986530303955078, + "logits/rejected": -0.400226354598999, + "logps/chosen": -209.4599609375, + "logps/rejected": -130.23941040039062, + "loss": 0.1114, + "rewards/accuracies": 0.9750000238418579, + "rewards/chosen": 7.468968391418457, + "rewards/margins": 8.008504867553711, + "rewards/rejected": -0.5395378470420837, + "step": 470 + }, + { + "epoch": 0.05, + "learning_rate": 9.932506761616078e-06, + "logits/chosen": -0.7118775248527527, + "logits/rejected": -0.03559774532914162, + "logps/chosen": -193.45834350585938, + "logps/rejected": -131.99844360351562, + "loss": 0.0508, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 7.474764823913574, + "rewards/margins": 8.308823585510254, + "rewards/rejected": -0.8340586423873901, + "step": 480 + }, + { + "epoch": 0.05, + "learning_rate": 9.929653916740168e-06, + "logits/chosen": -1.1100711822509766, + "logits/rejected": -0.40355220437049866, + "logps/chosen": -242.05697631835938, + "logps/rejected": -139.188232421875, + "loss": 0.1127, + "rewards/accuracies": 0.9624999761581421, + "rewards/chosen": 8.726511001586914, + "rewards/margins": 9.716850280761719, + "rewards/rejected": -0.9903383255004883, + "step": 490 + }, + { + "epoch": 0.05, + "learning_rate": 9.92674244738581e-06, + "logits/chosen": -0.8486649394035339, + "logits/rejected": 0.05283903330564499, + "logps/chosen": -155.493408203125, + "logps/rejected": -141.4785919189453, + "loss": 0.0901, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": 7.535587310791016, + "rewards/margins": 8.244181632995605, + "rewards/rejected": -0.7085943818092346, + "step": 500 + }, + { + "epoch": 0.06, + "learning_rate": 9.923772388176812e-06, + "logits/chosen": -0.907477080821991, + "logits/rejected": -0.171137273311615, + "logps/chosen": -236.19198608398438, + "logps/rejected": -133.06643676757812, + "loss": 0.0178, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 8.850728034973145, + "rewards/margins": 10.053266525268555, + "rewards/rejected": -1.202539086341858, + "step": 510 + }, + { + "epoch": 0.06, + "learning_rate": 9.92074377443374e-06, + "logits/chosen": -1.0009920597076416, + "logits/rejected": -0.20153608918190002, + "logps/chosen": -178.60147094726562, + "logps/rejected": -143.53695678710938, + "loss": 0.1976, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": 7.2967023849487305, + "rewards/margins": 8.339073181152344, + "rewards/rejected": -1.0423698425292969, + "step": 520 + }, + { + "epoch": 0.06, + "learning_rate": 9.917656642173505e-06, + "logits/chosen": -1.0008546113967896, + "logits/rejected": -0.01453827042132616, + "logps/chosen": -206.38461303710938, + "logps/rejected": -144.1584014892578, + "loss": 0.1575, + "rewards/accuracies": 0.9750000238418579, + "rewards/chosen": 8.916428565979004, + "rewards/margins": 9.663307189941406, + "rewards/rejected": -0.7468785047531128, + "step": 530 + }, + { + "epoch": 0.06, + "learning_rate": 9.91451102810893e-06, + "logits/chosen": -1.326416015625, + "logits/rejected": -0.38943323493003845, + "logps/chosen": -220.2677764892578, + "logps/rejected": -130.0069122314453, + "loss": 0.054, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 8.959368705749512, + "rewards/margins": 9.750789642333984, + "rewards/rejected": -0.7914207577705383, + "step": 540 + }, + { + "epoch": 0.06, + "learning_rate": 9.911306969648318e-06, + "logits/chosen": -0.932703971862793, + "logits/rejected": -0.0587722584605217, + "logps/chosen": -205.8779296875, + "logps/rejected": -135.040283203125, + "loss": 0.1965, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 8.557300567626953, + "rewards/margins": 9.26187515258789, + "rewards/rejected": -0.7045742869377136, + "step": 550 + }, + { + "epoch": 0.06, + "learning_rate": 9.908044504895004e-06, + "logits/chosen": -0.9326754808425903, + "logits/rejected": -0.19441956281661987, + "logps/chosen": -195.15328979492188, + "logps/rejected": -131.31869506835938, + "loss": 0.0943, + "rewards/accuracies": 0.9624999761581421, + "rewards/chosen": 8.366579055786133, + "rewards/margins": 9.570085525512695, + "rewards/rejected": -1.203505277633667, + "step": 560 + }, + { + "epoch": 0.06, + "learning_rate": 9.904723672646903e-06, + "logits/chosen": -0.9648317098617554, + "logits/rejected": -0.06580810248851776, + "logps/chosen": -211.77633666992188, + "logps/rejected": -136.65756225585938, + "loss": 0.2817, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 8.721487045288086, + "rewards/margins": 9.342923164367676, + "rewards/rejected": -0.6214349865913391, + "step": 570 + }, + { + "epoch": 0.06, + "learning_rate": 9.901344512396049e-06, + "logits/chosen": -1.1457253694534302, + "logits/rejected": -0.2043023556470871, + "logps/chosen": -210.77450561523438, + "logps/rejected": -128.11520385742188, + "loss": 0.1145, + "rewards/accuracies": 0.9750000238418579, + "rewards/chosen": 8.162195205688477, + "rewards/margins": 9.146950721740723, + "rewards/rejected": -0.9847553968429565, + "step": 580 + }, + { + "epoch": 0.06, + "learning_rate": 9.897907064328125e-06, + "logits/chosen": -0.8582237362861633, + "logits/rejected": -0.14161944389343262, + "logps/chosen": -202.4379425048828, + "logps/rejected": -136.9176483154297, + "loss": 0.2544, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": 8.603880882263184, + "rewards/margins": 9.390780448913574, + "rewards/rejected": -0.7868987321853638, + "step": 590 + }, + { + "epoch": 0.07, + "learning_rate": 9.894411369321984e-06, + "logits/chosen": -0.9913557767868042, + "logits/rejected": -0.0038151517510414124, + "logps/chosen": -182.00146484375, + "logps/rejected": -135.45352172851562, + "loss": 0.0794, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": 7.4787445068359375, + "rewards/margins": 8.599211692810059, + "rewards/rejected": -1.120466947555542, + "step": 600 + }, + { + "epoch": 0.07, + "learning_rate": 9.890857468949164e-06, + "logits/chosen": -0.9070969820022583, + "logits/rejected": 0.03064725361764431, + "logps/chosen": -223.1108856201172, + "logps/rejected": -145.6663055419922, + "loss": 0.0421, + "rewards/accuracies": 0.9750000238418579, + "rewards/chosen": 8.449647903442383, + "rewards/margins": 10.343244552612305, + "rewards/rejected": -1.8935970067977905, + "step": 610 + }, + { + "epoch": 0.07, + "learning_rate": 9.887245405473393e-06, + "logits/chosen": -0.7473093867301941, + "logits/rejected": 0.2375013381242752, + "logps/chosen": -218.11428833007812, + "logps/rejected": -137.17213439941406, + "loss": 0.0778, + "rewards/accuracies": 0.9624999761581421, + "rewards/chosen": 7.949044704437256, + "rewards/margins": 8.823598861694336, + "rewards/rejected": -0.8745536804199219, + "step": 620 + }, + { + "epoch": 0.07, + "learning_rate": 9.88357522185009e-06, + "logits/chosen": -0.913166344165802, + "logits/rejected": -0.011747682467103004, + "logps/chosen": -220.34603881835938, + "logps/rejected": -141.53561401367188, + "loss": 0.0287, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 8.446084976196289, + "rewards/margins": 9.644007682800293, + "rewards/rejected": -1.1979233026504517, + "step": 630 + }, + { + "epoch": 0.07, + "learning_rate": 9.879846961725846e-06, + "logits/chosen": -0.8051003217697144, + "logits/rejected": 0.030248349532485008, + "logps/chosen": -199.5961456298828, + "logps/rejected": -155.56056213378906, + "loss": 0.0765, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": 8.313097953796387, + "rewards/margins": 9.846205711364746, + "rewards/rejected": -1.5331077575683594, + "step": 640 + }, + { + "epoch": 0.07, + "learning_rate": 9.876060669437913e-06, + "logits/chosen": -0.616194486618042, + "logits/rejected": 0.2745420038700104, + "logps/chosen": -215.4296417236328, + "logps/rejected": -142.76492309570312, + "loss": 0.0848, + "rewards/accuracies": 0.9375, + "rewards/chosen": 7.84356164932251, + "rewards/margins": 9.544157981872559, + "rewards/rejected": -1.7005951404571533, + "step": 650 + }, + { + "epoch": 0.07, + "learning_rate": 9.872216390013675e-06, + "logits/chosen": -0.7901605367660522, + "logits/rejected": 0.41072148084640503, + "logps/chosen": -185.0421142578125, + "logps/rejected": -144.97398376464844, + "loss": 0.1577, + "rewards/accuracies": 0.9750000238418579, + "rewards/chosen": 7.704103946685791, + "rewards/margins": 9.240386009216309, + "rewards/rejected": -1.5362818241119385, + "step": 660 + }, + { + "epoch": 0.07, + "learning_rate": 9.868314169170108e-06, + "logits/chosen": -0.5289187431335449, + "logits/rejected": 0.397865355014801, + "logps/chosen": -219.9363555908203, + "logps/rejected": -142.4893798828125, + "loss": 0.1996, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 7.231544494628906, + "rewards/margins": 8.826276779174805, + "rewards/rejected": -1.5947315692901611, + "step": 670 + }, + { + "epoch": 0.07, + "learning_rate": 9.86435405331324e-06, + "logits/chosen": -0.3374841809272766, + "logits/rejected": 0.4616989195346832, + "logps/chosen": -197.3474578857422, + "logps/rejected": -141.45425415039062, + "loss": 0.0638, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 7.858680725097656, + "rewards/margins": 9.586578369140625, + "rewards/rejected": -1.7278966903686523, + "step": 680 + }, + { + "epoch": 0.08, + "learning_rate": 9.860336089537601e-06, + "logits/chosen": -0.4742531180381775, + "logits/rejected": 0.3263815939426422, + "logps/chosen": -197.79933166503906, + "logps/rejected": -145.15786743164062, + "loss": 0.1294, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": 7.016458988189697, + "rewards/margins": 8.728564262390137, + "rewards/rejected": -1.7121059894561768, + "step": 690 + }, + { + "epoch": 0.08, + "learning_rate": 9.856260325625657e-06, + "logits/chosen": -0.6684874892234802, + "logits/rejected": 0.5160067677497864, + "logps/chosen": -215.1957550048828, + "logps/rejected": -157.3702392578125, + "loss": 0.0588, + "rewards/accuracies": 0.9624999761581421, + "rewards/chosen": 6.882597923278809, + "rewards/margins": 9.006999969482422, + "rewards/rejected": -2.1244006156921387, + "step": 700 + }, + { + "epoch": 0.08, + "learning_rate": 9.85212681004725e-06, + "logits/chosen": -0.7230426073074341, + "logits/rejected": 0.48965024948120117, + "logps/chosen": -238.30581665039062, + "logps/rejected": -151.21774291992188, + "loss": 0.0211, + "rewards/accuracies": 1.0, + "rewards/chosen": 8.455827713012695, + "rewards/margins": 10.841313362121582, + "rewards/rejected": -2.385484457015991, + "step": 710 + }, + { + "epoch": 0.08, + "learning_rate": 9.847935591959011e-06, + "logits/chosen": -0.2026168555021286, + "logits/rejected": 0.6747661828994751, + "logps/chosen": -206.3802490234375, + "logps/rejected": -141.45103454589844, + "loss": 0.0742, + "rewards/accuracies": 0.9624999761581421, + "rewards/chosen": 7.532872676849365, + "rewards/margins": 9.587210655212402, + "rewards/rejected": -2.0543372631073, + "step": 720 + }, + { + "epoch": 0.08, + "learning_rate": 9.84368672120379e-06, + "logits/chosen": -0.3312041461467743, + "logits/rejected": 0.7338079214096069, + "logps/chosen": -200.46630859375, + "logps/rejected": -147.34877014160156, + "loss": 0.1475, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": 7.3677077293396, + "rewards/margins": 9.639891624450684, + "rewards/rejected": -2.272183656692505, + "step": 730 + }, + { + "epoch": 0.08, + "learning_rate": 9.839380248310043e-06, + "logits/chosen": -0.16066689789295197, + "logits/rejected": 0.8108816146850586, + "logps/chosen": -175.85104370117188, + "logps/rejected": -157.10450744628906, + "loss": 0.0537, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 7.471951961517334, + "rewards/margins": 9.829510688781738, + "rewards/rejected": -2.3575589656829834, + "step": 740 + }, + { + "epoch": 0.08, + "learning_rate": 9.835016224491251e-06, + "logits/chosen": -0.3715950846672058, + "logits/rejected": 0.7917531728744507, + "logps/chosen": -183.4439697265625, + "logps/rejected": -141.3053436279297, + "loss": 0.0628, + "rewards/accuracies": 0.9750000238418579, + "rewards/chosen": 7.702794551849365, + "rewards/margins": 9.382607460021973, + "rewards/rejected": -1.6798127889633179, + "step": 750 + }, + { + "epoch": 0.08, + "learning_rate": 9.830594701645304e-06, + "logits/chosen": -0.3017290234565735, + "logits/rejected": 0.6022787094116211, + "logps/chosen": -198.8234100341797, + "logps/rejected": -144.2804412841797, + "loss": 0.0508, + "rewards/accuracies": 0.9750000238418579, + "rewards/chosen": 7.64525842666626, + "rewards/margins": 9.416332244873047, + "rewards/rejected": -1.7710736989974976, + "step": 760 + }, + { + "epoch": 0.08, + "learning_rate": 9.826115732353877e-06, + "logits/chosen": -0.36430418491363525, + "logits/rejected": 0.5193418860435486, + "logps/chosen": -256.91583251953125, + "logps/rejected": -153.57138061523438, + "loss": 0.1406, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": 7.545983791351318, + "rewards/margins": 10.080999374389648, + "rewards/rejected": -2.535015821456909, + "step": 770 + }, + { + "epoch": 0.09, + "learning_rate": 9.82157936988181e-06, + "logits/chosen": -0.4545150697231293, + "logits/rejected": 0.928754985332489, + "logps/chosen": -220.9812469482422, + "logps/rejected": -150.5502166748047, + "loss": 0.0332, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 7.531121730804443, + "rewards/margins": 9.915712356567383, + "rewards/rejected": -2.3845906257629395, + "step": 780 + }, + { + "epoch": 0.09, + "learning_rate": 9.816985668176482e-06, + "logits/chosen": -0.7030404806137085, + "logits/rejected": 0.5952990055084229, + "logps/chosen": -231.7360382080078, + "logps/rejected": -148.20033264160156, + "loss": 0.152, + "rewards/accuracies": 0.9750000238418579, + "rewards/chosen": 8.060033798217773, + "rewards/margins": 9.900753021240234, + "rewards/rejected": -1.8407185077667236, + "step": 790 + }, + { + "epoch": 0.09, + "learning_rate": 9.812334681867152e-06, + "logits/chosen": -0.7694862484931946, + "logits/rejected": 0.30579477548599243, + "logps/chosen": -182.74615478515625, + "logps/rejected": -146.8044891357422, + "loss": 0.106, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": 7.797210693359375, + "rewards/margins": 9.230527877807617, + "rewards/rejected": -1.4333173036575317, + "step": 800 + }, + { + "epoch": 0.09, + "learning_rate": 9.807626466264328e-06, + "logits/chosen": -0.6776883006095886, + "logits/rejected": 0.4075050354003906, + "logps/chosen": -188.23257446289062, + "logps/rejected": -147.1197052001953, + "loss": 0.0302, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 8.79939079284668, + "rewards/margins": 10.872949600219727, + "rewards/rejected": -2.073558807373047, + "step": 810 + }, + { + "epoch": 0.09, + "learning_rate": 9.802861077359098e-06, + "logits/chosen": -0.6511849164962769, + "logits/rejected": 0.4911740720272064, + "logps/chosen": -197.52383422851562, + "logps/rejected": -153.37123107910156, + "loss": 0.1162, + "rewards/accuracies": 0.9750000238418579, + "rewards/chosen": 7.9149274826049805, + "rewards/margins": 10.323246955871582, + "rewards/rejected": -2.4083173274993896, + "step": 820 + }, + { + "epoch": 0.09, + "learning_rate": 9.798038571822463e-06, + "logits/chosen": -0.8261006474494934, + "logits/rejected": 0.3168261647224426, + "logps/chosen": -200.90821838378906, + "logps/rejected": -147.97134399414062, + "loss": 0.0839, + "rewards/accuracies": 0.9375, + "rewards/chosen": 7.613385200500488, + "rewards/margins": 9.651713371276855, + "rewards/rejected": -2.038329601287842, + "step": 830 + }, + { + "epoch": 0.09, + "learning_rate": 9.793159007004675e-06, + "logits/chosen": -0.9683536291122437, + "logits/rejected": -0.2409909963607788, + "logps/chosen": -231.24169921875, + "logps/rejected": -154.0245361328125, + "loss": 0.0287, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 8.987038612365723, + "rewards/margins": 10.66617202758789, + "rewards/rejected": -1.6791340112686157, + "step": 840 + }, + { + "epoch": 0.09, + "learning_rate": 9.78822244093454e-06, + "logits/chosen": -0.7381896376609802, + "logits/rejected": 0.2965128421783447, + "logps/chosen": -174.1750946044922, + "logps/rejected": -151.99537658691406, + "loss": 0.0919, + "rewards/accuracies": 0.9624999761581421, + "rewards/chosen": 7.649115085601807, + "rewards/margins": 9.841694831848145, + "rewards/rejected": -2.192580223083496, + "step": 850 + }, + { + "epoch": 0.09, + "learning_rate": 9.783228932318736e-06, + "logits/chosen": -0.6059221625328064, + "logits/rejected": 0.08314159512519836, + "logps/chosen": -210.4627227783203, + "logps/rejected": -152.2406768798828, + "loss": 0.0535, + "rewards/accuracies": 0.9750000238418579, + "rewards/chosen": 7.937192440032959, + "rewards/margins": 9.917118072509766, + "rewards/rejected": -1.979925513267517, + "step": 860 + }, + { + "epoch": 0.1, + "learning_rate": 9.778178540541116e-06, + "logits/chosen": -0.730563759803772, + "logits/rejected": 0.34863799810409546, + "logps/chosen": -200.68209838867188, + "logps/rejected": -151.97323608398438, + "loss": 0.0397, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 7.986664772033691, + "rewards/margins": 10.529966354370117, + "rewards/rejected": -2.543302059173584, + "step": 870 + }, + { + "epoch": 0.1, + "learning_rate": 9.773071325662e-06, + "logits/chosen": -0.8053274154663086, + "logits/rejected": 0.12549547851085663, + "logps/chosen": -192.92202758789062, + "logps/rejected": -146.79693603515625, + "loss": 0.1526, + "rewards/accuracies": 0.9624999761581421, + "rewards/chosen": 8.616433143615723, + "rewards/margins": 10.316951751708984, + "rewards/rejected": -1.7005198001861572, + "step": 880 + }, + { + "epoch": 0.1, + "learning_rate": 9.767907348417458e-06, + "logits/chosen": -0.6939282417297363, + "logits/rejected": 0.5801184773445129, + "logps/chosen": -237.7701873779297, + "logps/rejected": -150.30313110351562, + "loss": 0.2363, + "rewards/accuracies": 0.9375, + "rewards/chosen": 8.603517532348633, + "rewards/margins": 10.700518608093262, + "rewards/rejected": -2.0970005989074707, + "step": 890 + }, + { + "epoch": 0.1, + "learning_rate": 9.762686670218588e-06, + "logits/chosen": -0.5126532912254333, + "logits/rejected": 0.5091619491577148, + "logps/chosen": -213.9130401611328, + "logps/rejected": -155.66085815429688, + "loss": 0.0143, + "rewards/accuracies": 1.0, + "rewards/chosen": 8.648975372314453, + "rewards/margins": 11.388581275939941, + "rewards/rejected": -2.739605665206909, + "step": 900 + }, + { + "epoch": 0.1, + "learning_rate": 9.757409353150793e-06, + "logits/chosen": -0.4479140341281891, + "logits/rejected": 0.5997704267501831, + "logps/chosen": -206.91928100585938, + "logps/rejected": -153.65902709960938, + "loss": 0.0504, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 6.941058158874512, + "rewards/margins": 9.638825416564941, + "rewards/rejected": -2.6977667808532715, + "step": 910 + }, + { + "epoch": 0.1, + "learning_rate": 9.752075459973034e-06, + "logits/chosen": -0.48702773451805115, + "logits/rejected": 0.5097901225090027, + "logps/chosen": -253.9662322998047, + "logps/rejected": -161.4595947265625, + "loss": 0.0841, + "rewards/accuracies": 0.9750000238418579, + "rewards/chosen": 8.159878730773926, + "rewards/margins": 10.821348190307617, + "rewards/rejected": -2.661470890045166, + "step": 920 + }, + { + "epoch": 0.1, + "learning_rate": 9.746685054117086e-06, + "logits/chosen": -0.4953214228153229, + "logits/rejected": 0.5581221580505371, + "logps/chosen": -248.16482543945312, + "logps/rejected": -150.33505249023438, + "loss": 0.0869, + "rewards/accuracies": 0.9750000238418579, + "rewards/chosen": 7.8457465171813965, + "rewards/margins": 10.702252388000488, + "rewards/rejected": -2.856506109237671, + "step": 930 + }, + { + "epoch": 0.1, + "learning_rate": 9.741238199686789e-06, + "logits/chosen": -0.5145618915557861, + "logits/rejected": 0.7150206565856934, + "logps/chosen": -236.22683715820312, + "logps/rejected": -149.36941528320312, + "loss": 0.0362, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 7.276123046875, + "rewards/margins": 10.051583290100098, + "rewards/rejected": -2.775459051132202, + "step": 940 + }, + { + "epoch": 0.1, + "learning_rate": 9.735734961457277e-06, + "logits/chosen": -0.4139340817928314, + "logits/rejected": 0.7916821241378784, + "logps/chosen": -179.39382934570312, + "logps/rejected": -153.31869506835938, + "loss": 0.0393, + "rewards/accuracies": 0.9750000238418579, + "rewards/chosen": 7.780765533447266, + "rewards/margins": 10.50986099243164, + "rewards/rejected": -2.729094982147217, + "step": 950 + }, + { + "epoch": 0.11, + "learning_rate": 9.730175404874212e-06, + "logits/chosen": -0.3699759542942047, + "logits/rejected": 0.7061504125595093, + "logps/chosen": -173.6160888671875, + "logps/rejected": -157.6312713623047, + "loss": 0.15, + "rewards/accuracies": 0.9375, + "rewards/chosen": 8.363532066345215, + "rewards/margins": 10.816950798034668, + "rewards/rejected": -2.4534192085266113, + "step": 960 + }, + { + "epoch": 0.11, + "learning_rate": 9.724559596053009e-06, + "logits/chosen": -0.6346268057823181, + "logits/rejected": 0.48736244440078735, + "logps/chosen": -197.9464111328125, + "logps/rejected": -163.78814697265625, + "loss": 0.0861, + "rewards/accuracies": 0.9624999761581421, + "rewards/chosen": 8.08003044128418, + "rewards/margins": 10.763525009155273, + "rewards/rejected": -2.683495044708252, + "step": 970 + }, + { + "epoch": 0.11, + "learning_rate": 9.718887601778044e-06, + "logits/chosen": -0.4699893891811371, + "logits/rejected": 0.5934313535690308, + "logps/chosen": -172.5706787109375, + "logps/rejected": -147.60012817382812, + "loss": 0.0598, + "rewards/accuracies": 0.9750000238418579, + "rewards/chosen": 7.218860626220703, + "rewards/margins": 9.611445426940918, + "rewards/rejected": -2.3925843238830566, + "step": 980 + }, + { + "epoch": 0.11, + "learning_rate": 9.713159489501863e-06, + "logits/chosen": -0.7212098836898804, + "logits/rejected": 0.6830633282661438, + "logps/chosen": -195.26705932617188, + "logps/rejected": -144.74806213378906, + "loss": 0.0301, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 7.268791198730469, + "rewards/margins": 9.785104751586914, + "rewards/rejected": -2.5163135528564453, + "step": 990 + }, + { + "epoch": 0.11, + "learning_rate": 9.707956263849591e-06, + "logits/chosen": -0.653510570526123, + "logits/rejected": 0.6825904250144958, + "logps/chosen": -214.06729125976562, + "logps/rejected": -151.3599395751953, + "loss": 0.1073, + "rewards/accuracies": 0.9624999761581421, + "rewards/chosen": 8.91938591003418, + "rewards/margins": 11.606144905090332, + "rewards/rejected": -2.6867592334747314, + "step": 1000 + }, + { + "epoch": 0.11, + "learning_rate": 9.702121715592386e-06, + "logits/chosen": -0.6122604608535767, + "logits/rejected": 0.5413939356803894, + "logps/chosen": -245.49929809570312, + "logps/rejected": -154.56021118164062, + "loss": 0.098, + "rewards/accuracies": 0.9624999761581421, + "rewards/chosen": 8.354562759399414, + "rewards/margins": 10.709449768066406, + "rewards/rejected": -2.3548874855041504, + "step": 1010 + }, + { + "epoch": 0.11, + "learning_rate": 9.696231248717397e-06, + "logits/chosen": -0.3631553053855896, + "logits/rejected": 0.6392610669136047, + "logps/chosen": -209.0885467529297, + "logps/rejected": -144.05416870117188, + "loss": 0.053, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 8.57081413269043, + "rewards/margins": 10.535473823547363, + "rewards/rejected": -1.9646602869033813, + "step": 1020 + }, + { + "epoch": 0.11, + "learning_rate": 9.690284933275296e-06, + "logits/chosen": -0.7420902252197266, + "logits/rejected": 0.7464100122451782, + "logps/chosen": -223.1457061767578, + "logps/rejected": -161.8677215576172, + "loss": 0.125, + "rewards/accuracies": 0.9375, + "rewards/chosen": 8.317934036254883, + "rewards/margins": 10.725369453430176, + "rewards/rejected": -2.407435894012451, + "step": 1030 + }, + { + "epoch": 0.11, + "learning_rate": 9.684282839980912e-06, + "logits/chosen": -0.4254356324672699, + "logits/rejected": 0.7181139588356018, + "logps/chosen": -219.45327758789062, + "logps/rejected": -147.68211364746094, + "loss": 0.0078, + "rewards/accuracies": 1.0, + "rewards/chosen": 8.572770118713379, + "rewards/margins": 11.732831954956055, + "rewards/rejected": -3.1600615978240967, + "step": 1040 + }, + { + "epoch": 0.12, + "learning_rate": 9.678225040212398e-06, + "logits/chosen": -0.22140467166900635, + "logits/rejected": 1.044932246208191, + "logps/chosen": -178.0739288330078, + "logps/rejected": -160.81057739257812, + "loss": 0.148, + "rewards/accuracies": 0.9750000238418579, + "rewards/chosen": 7.5312628746032715, + "rewards/margins": 10.080840110778809, + "rewards/rejected": -2.5495781898498535, + "step": 1050 + }, + { + "epoch": 0.12, + "learning_rate": 9.67211160601038e-06, + "logits/chosen": -0.3561934232711792, + "logits/rejected": 0.9342392683029175, + "logps/chosen": -224.42599487304688, + "logps/rejected": -165.5540771484375, + "loss": 0.007, + "rewards/accuracies": 1.0, + "rewards/chosen": 8.821218490600586, + "rewards/margins": 12.073785781860352, + "rewards/rejected": -3.252568483352661, + "step": 1060 + }, + { + "epoch": 0.12, + "learning_rate": 9.665942610077096e-06, + "logits/chosen": -0.2816230058670044, + "logits/rejected": 0.7103188037872314, + "logps/chosen": -226.01358032226562, + "logps/rejected": -147.710205078125, + "loss": 0.1092, + "rewards/accuracies": 0.9375, + "rewards/chosen": 8.991147994995117, + "rewards/margins": 11.593085289001465, + "rewards/rejected": -2.601935863494873, + "step": 1070 + }, + { + "epoch": 0.12, + "learning_rate": 9.659718125775547e-06, + "logits/chosen": -0.5628015995025635, + "logits/rejected": 0.7087093591690063, + "logps/chosen": -212.25149536132812, + "logps/rejected": -164.18304443359375, + "loss": 0.2843, + "rewards/accuracies": 0.9750000238418579, + "rewards/chosen": 8.73010540008545, + "rewards/margins": 11.143708229064941, + "rewards/rejected": -2.413602352142334, + "step": 1080 + }, + { + "epoch": 0.12, + "learning_rate": 9.6534382271286e-06, + "logits/chosen": -0.7271457314491272, + "logits/rejected": 0.3624531626701355, + "logps/chosen": -244.6057891845703, + "logps/rejected": -150.74534606933594, + "loss": 0.0431, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 8.928186416625977, + "rewards/margins": 11.571906089782715, + "rewards/rejected": -2.64371919631958, + "step": 1090 + }, + { + "epoch": 0.12, + "learning_rate": 9.64710298881813e-06, + "logits/chosen": -0.42074766755104065, + "logits/rejected": 0.531056821346283, + "logps/chosen": -178.32199096679688, + "logps/rejected": -145.59683227539062, + "loss": 0.0996, + "rewards/accuracies": 0.9624999761581421, + "rewards/chosen": 8.120710372924805, + "rewards/margins": 10.442843437194824, + "rewards/rejected": -2.3221330642700195, + "step": 1100 + }, + { + "epoch": 0.12, + "learning_rate": 9.640712486184117e-06, + "logits/chosen": -0.7548767924308777, + "logits/rejected": 0.36750084161758423, + "logps/chosen": -216.7880859375, + "logps/rejected": -160.77999877929688, + "loss": 0.1199, + "rewards/accuracies": 0.9624999761581421, + "rewards/chosen": 8.382489204406738, + "rewards/margins": 11.168853759765625, + "rewards/rejected": -2.7863640785217285, + "step": 1110 + }, + { + "epoch": 0.12, + "learning_rate": 9.634266795223761e-06, + "logits/chosen": -0.36768269538879395, + "logits/rejected": 0.8007356524467468, + "logps/chosen": -182.25375366210938, + "logps/rejected": -149.307373046875, + "loss": 0.0616, + "rewards/accuracies": 0.9624999761581421, + "rewards/chosen": 6.98935079574585, + "rewards/margins": 9.277505874633789, + "rewards/rejected": -2.288156032562256, + "step": 1120 + }, + { + "epoch": 0.12, + "learning_rate": 9.627765992590572e-06, + "logits/chosen": -0.679458737373352, + "logits/rejected": 0.520967423915863, + "logps/chosen": -194.57183837890625, + "logps/rejected": -157.29466247558594, + "loss": 0.0976, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 8.470559120178223, + "rewards/margins": 11.556069374084473, + "rewards/rejected": -3.0855116844177246, + "step": 1130 + }, + { + "epoch": 0.13, + "learning_rate": 9.621210155593457e-06, + "logits/chosen": -0.3645058274269104, + "logits/rejected": 0.6819556355476379, + "logps/chosen": -183.23855590820312, + "logps/rejected": -151.49508666992188, + "loss": 0.1729, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": 6.996212005615234, + "rewards/margins": 9.404515266418457, + "rewards/rejected": -2.408303737640381, + "step": 1140 + }, + { + "epoch": 0.13, + "learning_rate": 9.614599362195803e-06, + "logits/chosen": -0.2955774664878845, + "logits/rejected": 0.5863500833511353, + "logps/chosen": -206.7914276123047, + "logps/rejected": -146.72183227539062, + "loss": 0.0833, + "rewards/accuracies": 0.9624999761581421, + "rewards/chosen": 8.1628999710083, + "rewards/margins": 10.706645965576172, + "rewards/rejected": -2.5437464714050293, + "step": 1150 + }, + { + "epoch": 0.13, + "learning_rate": 9.607933691014556e-06, + "logits/chosen": -0.5553183555603027, + "logits/rejected": 0.3617022633552551, + "logps/chosen": -198.2234649658203, + "logps/rejected": -154.90408325195312, + "loss": 0.0427, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 8.177728652954102, + "rewards/margins": 11.280416488647461, + "rewards/rejected": -3.1026864051818848, + "step": 1160 + }, + { + "epoch": 0.13, + "learning_rate": 9.601213221319271e-06, + "logits/chosen": -0.6628633141517639, + "logits/rejected": 0.6921408176422119, + "logps/chosen": -192.33914184570312, + "logps/rejected": -155.47396850585938, + "loss": 0.0349, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 8.054289817810059, + "rewards/margins": 11.028264045715332, + "rewards/rejected": -2.973972797393799, + "step": 1170 + }, + { + "epoch": 0.13, + "learning_rate": 9.594438033031188e-06, + "logits/chosen": -0.510148286819458, + "logits/rejected": 0.6729466319084167, + "logps/chosen": -183.96536254882812, + "logps/rejected": -155.4961395263672, + "loss": 0.0129, + "rewards/accuracies": 1.0, + "rewards/chosen": 8.211958885192871, + "rewards/margins": 11.422294616699219, + "rewards/rejected": -3.210334300994873, + "step": 1180 + }, + { + "epoch": 0.13, + "learning_rate": 9.587608206722265e-06, + "logits/chosen": -0.6839829087257385, + "logits/rejected": 0.4797239303588867, + "logps/chosen": -209.3610076904297, + "logps/rejected": -152.60250854492188, + "loss": 0.0246, + "rewards/accuracies": 1.0, + "rewards/chosen": 7.890170097351074, + "rewards/margins": 10.441574096679688, + "rewards/rejected": -2.5514039993286133, + "step": 1190 + }, + { + "epoch": 0.13, + "learning_rate": 9.580723823614229e-06, + "logits/chosen": -0.32581615447998047, + "logits/rejected": 0.7886501550674438, + "logps/chosen": -187.45545959472656, + "logps/rejected": -161.00302124023438, + "loss": 0.0698, + "rewards/accuracies": 0.9750000238418579, + "rewards/chosen": 7.390974521636963, + "rewards/margins": 10.586621284484863, + "rewards/rejected": -3.1956467628479004, + "step": 1200 + }, + { + "epoch": 0.13, + "learning_rate": 9.573784965577608e-06, + "logits/chosen": -0.4175148606300354, + "logits/rejected": 0.8941513299942017, + "logps/chosen": -184.9542999267578, + "logps/rejected": -170.08277893066406, + "loss": 0.0517, + "rewards/accuracies": 0.9750000238418579, + "rewards/chosen": 6.477563381195068, + "rewards/margins": 9.664083480834961, + "rewards/rejected": -3.186519145965576, + "step": 1210 + }, + { + "epoch": 0.13, + "learning_rate": 9.56679171513076e-06, + "logits/chosen": -0.4166285991668701, + "logits/rejected": 0.7068973779678345, + "logps/chosen": -211.63351440429688, + "logps/rejected": -157.5101776123047, + "loss": 0.0588, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 8.07889461517334, + "rewards/margins": 10.903955459594727, + "rewards/rejected": -2.825061321258545, + "step": 1220 + }, + { + "epoch": 0.14, + "learning_rate": 9.55974415543888e-06, + "logits/chosen": -0.5035102963447571, + "logits/rejected": 0.7554497718811035, + "logps/chosen": -246.5172576904297, + "logps/rejected": -167.3823699951172, + "loss": 0.0217, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 8.890531539916992, + "rewards/margins": 12.393576622009277, + "rewards/rejected": -3.5030460357666016, + "step": 1230 + }, + { + "epoch": 0.14, + "learning_rate": 9.552642370313032e-06, + "logits/chosen": -0.4481489658355713, + "logits/rejected": 0.6294432282447815, + "logps/chosen": -213.8181915283203, + "logps/rejected": -153.36618041992188, + "loss": 0.0668, + "rewards/accuracies": 0.9624999761581421, + "rewards/chosen": 8.723450660705566, + "rewards/margins": 12.148078918457031, + "rewards/rejected": -3.4246277809143066, + "step": 1240 + }, + { + "epoch": 0.14, + "learning_rate": 9.545486444209133e-06, + "logits/chosen": -0.3546793758869171, + "logits/rejected": 0.7414132356643677, + "logps/chosen": -146.98406982421875, + "logps/rejected": -143.73516845703125, + "loss": 0.134, + "rewards/accuracies": 0.9624999761581421, + "rewards/chosen": 7.439810752868652, + "rewards/margins": 9.926045417785645, + "rewards/rejected": -2.4862332344055176, + "step": 1250 + }, + { + "epoch": 0.14, + "learning_rate": 9.538276462226956e-06, + "logits/chosen": -0.34465163946151733, + "logits/rejected": 0.9261838793754578, + "logps/chosen": -185.11166381835938, + "logps/rejected": -161.7854766845703, + "loss": 0.0201, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 7.154252529144287, + "rewards/margins": 10.484378814697266, + "rewards/rejected": -3.3301258087158203, + "step": 1260 + }, + { + "epoch": 0.14, + "learning_rate": 9.531012510109121e-06, + "logits/chosen": -0.5041292905807495, + "logits/rejected": 0.8341606855392456, + "logps/chosen": -223.11752319335938, + "logps/rejected": -168.34884643554688, + "loss": 0.0152, + "rewards/accuracies": 1.0, + "rewards/chosen": 7.197157382965088, + "rewards/margins": 10.75901985168457, + "rewards/rejected": -3.5618622303009033, + "step": 1270 + }, + { + "epoch": 0.14, + "learning_rate": 9.52369467424007e-06, + "logits/chosen": -0.5082358121871948, + "logits/rejected": 0.90031898021698, + "logps/chosen": -217.27194213867188, + "logps/rejected": -157.5386199951172, + "loss": 0.1151, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 7.356767177581787, + "rewards/margins": 10.65047550201416, + "rewards/rejected": -3.293708086013794, + "step": 1280 + }, + { + "epoch": 0.14, + "learning_rate": 9.516323041645041e-06, + "logits/chosen": -0.4632319509983063, + "logits/rejected": 0.6919183135032654, + "logps/chosen": -251.9365997314453, + "logps/rejected": -159.3051300048828, + "loss": 0.0228, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 7.674399375915527, + "rewards/margins": 10.893823623657227, + "rewards/rejected": -3.219423294067383, + "step": 1290 + }, + { + "epoch": 0.14, + "learning_rate": 9.508897699989034e-06, + "logits/chosen": -0.4946981370449066, + "logits/rejected": 0.5167168378829956, + "logps/chosen": -239.37088012695312, + "logps/rejected": -163.68618774414062, + "loss": 0.0437, + "rewards/accuracies": 0.9750000238418579, + "rewards/chosen": 7.406832695007324, + "rewards/margins": 10.362913131713867, + "rewards/rejected": -2.9560797214508057, + "step": 1300 + }, + { + "epoch": 0.14, + "learning_rate": 9.501418737575771e-06, + "logits/chosen": -0.3266671895980835, + "logits/rejected": 0.8748236894607544, + "logps/chosen": -215.59164428710938, + "logps/rejected": -151.0523681640625, + "loss": 0.1203, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 7.814599514007568, + "rewards/margins": 11.216487884521484, + "rewards/rejected": -3.401888608932495, + "step": 1310 + }, + { + "epoch": 0.14, + "learning_rate": 9.493886243346644e-06, + "logits/chosen": -0.4157380163669586, + "logits/rejected": 1.0467342138290405, + "logps/chosen": -251.1937255859375, + "logps/rejected": -169.79441833496094, + "loss": 0.129, + "rewards/accuracies": 0.9624999761581421, + "rewards/chosen": 8.097917556762695, + "rewards/margins": 11.96033763885498, + "rewards/rejected": -3.8624203205108643, + "step": 1320 + }, + { + "epoch": 0.15, + "learning_rate": 9.486300306879652e-06, + "logits/chosen": -0.46027225255966187, + "logits/rejected": 0.9197125434875488, + "logps/chosen": -213.0179901123047, + "logps/rejected": -185.57815551757812, + "loss": 0.1064, + "rewards/accuracies": 0.9750000238418579, + "rewards/chosen": 7.102902889251709, + "rewards/margins": 10.794425964355469, + "rewards/rejected": -3.6915230751037598, + "step": 1330 + }, + { + "epoch": 0.15, + "learning_rate": 9.478661018388343e-06, + "logits/chosen": -0.27332133054733276, + "logits/rejected": 1.0553624629974365, + "logps/chosen": -195.7577667236328, + "logps/rejected": -162.10386657714844, + "loss": 0.028, + "rewards/accuracies": 0.9750000238418579, + "rewards/chosen": 7.459615230560303, + "rewards/margins": 11.042313575744629, + "rewards/rejected": -3.5826964378356934, + "step": 1340 + }, + { + "epoch": 0.15, + "learning_rate": 9.470968468720737e-06, + "logits/chosen": -0.4711340069770813, + "logits/rejected": 0.6478385925292969, + "logps/chosen": -218.00247192382812, + "logps/rejected": -156.74826049804688, + "loss": 0.036, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 8.571695327758789, + "rewards/margins": 11.931859016418457, + "rewards/rejected": -3.360163450241089, + "step": 1350 + }, + { + "epoch": 0.15, + "learning_rate": 9.46322274935825e-06, + "logits/chosen": -0.8137804269790649, + "logits/rejected": 0.6631590127944946, + "logps/chosen": -238.30917358398438, + "logps/rejected": -160.2780303955078, + "loss": 0.0763, + "rewards/accuracies": 0.9750000238418579, + "rewards/chosen": 8.956851959228516, + "rewards/margins": 12.471067428588867, + "rewards/rejected": -3.514216184616089, + "step": 1360 + }, + { + "epoch": 0.15, + "learning_rate": 9.455423952414602e-06, + "logits/chosen": -0.45048221945762634, + "logits/rejected": 0.9729928970336914, + "logps/chosen": -211.9479217529297, + "logps/rejected": -163.4690704345703, + "loss": 0.006, + "rewards/accuracies": 1.0, + "rewards/chosen": 8.597060203552246, + "rewards/margins": 12.271041870117188, + "rewards/rejected": -3.673980236053467, + "step": 1370 + }, + { + "epoch": 0.15, + "learning_rate": 9.44757217063472e-06, + "logits/chosen": -0.6646102666854858, + "logits/rejected": 0.6734617352485657, + "logps/chosen": -186.35360717773438, + "logps/rejected": -152.72317504882812, + "loss": 0.0149, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 8.054737091064453, + "rewards/margins": 11.063182830810547, + "rewards/rejected": -3.0084469318389893, + "step": 1380 + }, + { + "epoch": 0.15, + "learning_rate": 9.439667497393644e-06, + "logits/chosen": -0.509512722492218, + "logits/rejected": 0.4323197901248932, + "logps/chosen": -206.06008911132812, + "logps/rejected": -156.47021484375, + "loss": 0.0029, + "rewards/accuracies": 1.0, + "rewards/chosen": 8.382911682128906, + "rewards/margins": 11.36556339263916, + "rewards/rejected": -2.9826526641845703, + "step": 1390 + }, + { + "epoch": 0.15, + "learning_rate": 9.4317100266954e-06, + "logits/chosen": -0.2360249012708664, + "logits/rejected": 0.7698749303817749, + "logps/chosen": -193.63430786132812, + "logps/rejected": -154.2914276123047, + "loss": 0.3086, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 7.249920845031738, + "rewards/margins": 9.66682243347168, + "rewards/rejected": -2.4169018268585205, + "step": 1400 + }, + { + "epoch": 0.15, + "learning_rate": 9.423699853171903e-06, + "logits/chosen": -0.4919816553592682, + "logits/rejected": 0.6998618841171265, + "logps/chosen": -191.9207305908203, + "logps/rejected": -161.76254272460938, + "loss": 0.0641, + "rewards/accuracies": 0.9624999761581421, + "rewards/chosen": 7.954064846038818, + "rewards/margins": 11.05593490600586, + "rewards/rejected": -3.10187029838562, + "step": 1410 + }, + { + "epoch": 0.16, + "learning_rate": 9.415637072081817e-06, + "logits/chosen": -0.22287793457508087, + "logits/rejected": 0.6352042555809021, + "logps/chosen": -193.3926239013672, + "logps/rejected": -156.7662811279297, + "loss": 0.0086, + "rewards/accuracies": 1.0, + "rewards/chosen": 7.585391998291016, + "rewards/margins": 11.008899688720703, + "rewards/rejected": -3.4235072135925293, + "step": 1420 + }, + { + "epoch": 0.16, + "learning_rate": 9.407521779309426e-06, + "logits/chosen": -0.4121321737766266, + "logits/rejected": 0.8394039869308472, + "logps/chosen": -213.10006713867188, + "logps/rejected": -166.130615234375, + "loss": 0.011, + "rewards/accuracies": 1.0, + "rewards/chosen": 8.29853343963623, + "rewards/margins": 11.414801597595215, + "rewards/rejected": -3.116267442703247, + "step": 1430 + }, + { + "epoch": 0.16, + "learning_rate": 9.39935407136349e-06, + "logits/chosen": -0.41288232803344727, + "logits/rejected": 0.7763572335243225, + "logps/chosen": -232.4348907470703, + "logps/rejected": -164.8203887939453, + "loss": 0.0195, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 9.438920974731445, + "rewards/margins": 12.723814010620117, + "rewards/rejected": -3.28489351272583, + "step": 1440 + }, + { + "epoch": 0.16, + "learning_rate": 9.39113404537611e-06, + "logits/chosen": -0.16888564825057983, + "logits/rejected": 1.0841331481933594, + "logps/chosen": -151.8446502685547, + "logps/rejected": -165.15887451171875, + "loss": 0.1485, + "rewards/accuracies": 0.9750000238418579, + "rewards/chosen": 7.191922187805176, + "rewards/margins": 10.471902847290039, + "rewards/rejected": -3.279982089996338, + "step": 1450 + }, + { + "epoch": 0.16, + "learning_rate": 9.382861799101556e-06, + "logits/chosen": -0.3828301727771759, + "logits/rejected": 0.7569293975830078, + "logps/chosen": -209.0618896484375, + "logps/rejected": -169.80148315429688, + "loss": 0.074, + "rewards/accuracies": 0.9750000238418579, + "rewards/chosen": 8.271611213684082, + "rewards/margins": 11.68904972076416, + "rewards/rejected": -3.4174396991729736, + "step": 1460 + }, + { + "epoch": 0.16, + "learning_rate": 9.37453743091512e-06, + "logits/chosen": -0.4373001158237457, + "logits/rejected": 0.7532221674919128, + "logps/chosen": -214.9142303466797, + "logps/rejected": -157.76080322265625, + "loss": 0.1568, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": 8.783075332641602, + "rewards/margins": 11.846728324890137, + "rewards/rejected": -3.063652276992798, + "step": 1470 + }, + { + "epoch": 0.16, + "learning_rate": 9.36616103981193e-06, + "logits/chosen": -0.7200452089309692, + "logits/rejected": 0.5077813267707825, + "logps/chosen": -231.0227813720703, + "logps/rejected": -163.6404571533203, + "loss": 0.0111, + "rewards/accuracies": 1.0, + "rewards/chosen": 8.481043815612793, + "rewards/margins": 11.727174758911133, + "rewards/rejected": -3.246131420135498, + "step": 1480 + }, + { + "epoch": 0.16, + "learning_rate": 9.357732725405788e-06, + "logits/chosen": -0.5195794701576233, + "logits/rejected": 0.5995023846626282, + "logps/chosen": -223.84408569335938, + "logps/rejected": -167.75845336914062, + "loss": 0.072, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 8.739862442016602, + "rewards/margins": 12.258071899414062, + "rewards/rejected": -3.5182101726531982, + "step": 1490 + }, + { + "epoch": 0.16, + "learning_rate": 9.349252587927977e-06, + "logits/chosen": -0.5706058740615845, + "logits/rejected": 0.7487145662307739, + "logps/chosen": -206.3643798828125, + "logps/rejected": -160.89007568359375, + "loss": 0.0292, + "rewards/accuracies": 0.9750000238418579, + "rewards/chosen": 9.057950973510742, + "rewards/margins": 12.710989952087402, + "rewards/rejected": -3.653038501739502, + "step": 1500 + }, + { + "epoch": 0.17, + "learning_rate": 9.340720728226071e-06, + "logits/chosen": -0.6348392963409424, + "logits/rejected": 0.7459853887557983, + "logps/chosen": -222.1605682373047, + "logps/rejected": -167.15003967285156, + "loss": 0.0187, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 8.992013931274414, + "rewards/margins": 12.756234169006348, + "rewards/rejected": -3.764221668243408, + "step": 1510 + }, + { + "epoch": 0.17, + "learning_rate": 9.332137247762735e-06, + "logits/chosen": -0.5036366581916809, + "logits/rejected": 0.43747663497924805, + "logps/chosen": -206.8778533935547, + "logps/rejected": -163.91195678710938, + "loss": 0.0239, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 7.371187686920166, + "rewards/margins": 10.75023078918457, + "rewards/rejected": -3.379042863845825, + "step": 1520 + }, + { + "epoch": 0.17, + "learning_rate": 9.32350224861452e-06, + "logits/chosen": -0.7491061687469482, + "logits/rejected": 0.40472355484962463, + "logps/chosen": -196.0803985595703, + "logps/rejected": -162.4312744140625, + "loss": 0.005, + "rewards/accuracies": 1.0, + "rewards/chosen": 8.126627922058105, + "rewards/margins": 12.170022964477539, + "rewards/rejected": -4.043394565582275, + "step": 1530 + }, + { + "epoch": 0.17, + "learning_rate": 9.314815833470646e-06, + "logits/chosen": -0.8289221525192261, + "logits/rejected": -0.022221926599740982, + "logps/chosen": -184.82785034179688, + "logps/rejected": -164.69155883789062, + "loss": 0.0055, + "rewards/accuracies": 1.0, + "rewards/chosen": 8.109321594238281, + "rewards/margins": 12.10090446472168, + "rewards/rejected": -3.9915835857391357, + "step": 1540 + }, + { + "epoch": 0.17, + "learning_rate": 9.306078105631783e-06, + "logits/chosen": -0.5130811333656311, + "logits/rejected": 0.783342719078064, + "logps/chosen": -191.61032104492188, + "logps/rejected": -168.2999725341797, + "loss": 0.0233, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 8.567174911499023, + "rewards/margins": 12.22905445098877, + "rewards/rejected": -3.661879301071167, + "step": 1550 + }, + { + "epoch": 0.17, + "learning_rate": 9.297289169008826e-06, + "logits/chosen": -0.6227730512619019, + "logits/rejected": 0.2883726954460144, + "logps/chosen": -199.9819793701172, + "logps/rejected": -153.50430297851562, + "loss": 0.0051, + "rewards/accuracies": 1.0, + "rewards/chosen": 7.932660102844238, + "rewards/margins": 11.378183364868164, + "rewards/rejected": -3.445523500442505, + "step": 1560 + }, + { + "epoch": 0.17, + "learning_rate": 9.288449128121651e-06, + "logits/chosen": -0.26492536067962646, + "logits/rejected": 0.8799971342086792, + "logps/chosen": -183.4518280029297, + "logps/rejected": -151.05831909179688, + "loss": 0.0694, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": 7.4086761474609375, + "rewards/margins": 10.281698226928711, + "rewards/rejected": -2.8730225563049316, + "step": 1570 + }, + { + "epoch": 0.17, + "learning_rate": 9.27955808809788e-06, + "logits/chosen": -0.551371693611145, + "logits/rejected": 0.7983097434043884, + "logps/chosen": -233.4415283203125, + "logps/rejected": -164.3033905029297, + "loss": 0.106, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": 8.844173431396484, + "rewards/margins": 12.37480354309082, + "rewards/rejected": -3.5306293964385986, + "step": 1580 + }, + { + "epoch": 0.17, + "learning_rate": 9.270616154671624e-06, + "logits/chosen": -0.7698153257369995, + "logits/rejected": 0.640004575252533, + "logps/chosen": -186.4699249267578, + "logps/rejected": -168.69979858398438, + "loss": 0.0583, + "rewards/accuracies": 0.9750000238418579, + "rewards/chosen": 7.0685834884643555, + "rewards/margins": 10.25090217590332, + "rewards/rejected": -3.1823184490203857, + "step": 1590 + }, + { + "epoch": 0.18, + "learning_rate": 9.261623434182232e-06, + "logits/chosen": -0.46172839403152466, + "logits/rejected": 0.6960395574569702, + "logps/chosen": -186.02001953125, + "logps/rejected": -162.45809936523438, + "loss": 0.0538, + "rewards/accuracies": 0.9750000238418579, + "rewards/chosen": 7.1736321449279785, + "rewards/margins": 10.610064506530762, + "rewards/rejected": -3.436432361602783, + "step": 1600 + }, + { + "epoch": 0.18, + "learning_rate": 9.252580033573024e-06, + "logits/chosen": -0.18526312708854675, + "logits/rejected": 0.9217432141304016, + "logps/chosen": -166.52264404296875, + "logps/rejected": -156.47720336914062, + "loss": 0.0342, + "rewards/accuracies": 0.9750000238418579, + "rewards/chosen": 6.552803039550781, + "rewards/margins": 10.394461631774902, + "rewards/rejected": -3.841658115386963, + "step": 1610 + }, + { + "epoch": 0.18, + "learning_rate": 9.243486060390014e-06, + "logits/chosen": -0.7024975419044495, + "logits/rejected": 0.3586685061454773, + "logps/chosen": -261.14373779296875, + "logps/rejected": -156.59616088867188, + "loss": 0.0398, + "rewards/accuracies": 0.9750000238418579, + "rewards/chosen": 9.696449279785156, + "rewards/margins": 12.929067611694336, + "rewards/rejected": -3.232619047164917, + "step": 1620 + }, + { + "epoch": 0.18, + "learning_rate": 9.23434162278064e-06, + "logits/chosen": -0.5696882009506226, + "logits/rejected": 0.5719606280326843, + "logps/chosen": -206.2845916748047, + "logps/rejected": -161.68795776367188, + "loss": 0.0043, + "rewards/accuracies": 1.0, + "rewards/chosen": 8.744453430175781, + "rewards/margins": 12.412347793579102, + "rewards/rejected": -3.6678924560546875, + "step": 1630 + }, + { + "epoch": 0.18, + "learning_rate": 9.225146829492472e-06, + "logits/chosen": -0.4481044411659241, + "logits/rejected": 0.750464677810669, + "logps/chosen": -179.038330078125, + "logps/rejected": -166.00767517089844, + "loss": 0.1427, + "rewards/accuracies": 0.9750000238418579, + "rewards/chosen": 7.574297904968262, + "rewards/margins": 11.261307716369629, + "rewards/rejected": -3.6870102882385254, + "step": 1640 + }, + { + "epoch": 0.18, + "learning_rate": 9.215901789871924e-06, + "logits/chosen": -0.9091712236404419, + "logits/rejected": 0.3831535875797272, + "logps/chosen": -206.7153778076172, + "logps/rejected": -156.24366760253906, + "loss": 0.0878, + "rewards/accuracies": 0.9624999761581421, + "rewards/chosen": 8.617593765258789, + "rewards/margins": 11.938032150268555, + "rewards/rejected": -3.3204398155212402, + "step": 1650 + }, + { + "epoch": 0.18, + "learning_rate": 9.206606613862941e-06, + "logits/chosen": -0.6661981344223022, + "logits/rejected": 0.7159786224365234, + "logps/chosen": -199.4313507080078, + "logps/rejected": -173.4095458984375, + "loss": 0.0661, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 8.909134864807129, + "rewards/margins": 12.851089477539062, + "rewards/rejected": -3.941955089569092, + "step": 1660 + }, + { + "epoch": 0.18, + "learning_rate": 9.197261412005708e-06, + "logits/chosen": -0.48630446195602417, + "logits/rejected": 0.5496179461479187, + "logps/chosen": -222.904541015625, + "logps/rejected": -164.3258514404297, + "loss": 0.0076, + "rewards/accuracies": 1.0, + "rewards/chosen": 8.863286018371582, + "rewards/margins": 12.797664642333984, + "rewards/rejected": -3.9343795776367188, + "step": 1670 + }, + { + "epoch": 0.18, + "learning_rate": 9.187866295435328e-06, + "logits/chosen": -0.7716933488845825, + "logits/rejected": 0.49292677640914917, + "logps/chosen": -224.15087890625, + "logps/rejected": -160.72998046875, + "loss": 0.0166, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 8.712728500366211, + "rewards/margins": 12.726627349853516, + "rewards/rejected": -4.013897895812988, + "step": 1680 + }, + { + "epoch": 0.19, + "learning_rate": 9.178421375880495e-06, + "logits/chosen": -0.985678493976593, + "logits/rejected": -0.16567879915237427, + "logps/chosen": -190.69140625, + "logps/rejected": -148.5782928466797, + "loss": 0.0516, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 8.473517417907715, + "rewards/margins": 11.43741226196289, + "rewards/rejected": -2.963894844055176, + "step": 1690 + }, + { + "epoch": 0.19, + "learning_rate": 9.168926765662176e-06, + "logits/chosen": -0.8118952512741089, + "logits/rejected": 0.2844352722167969, + "logps/chosen": -200.7667999267578, + "logps/rejected": -161.06378173828125, + "loss": 0.0476, + "rewards/accuracies": 0.9750000238418579, + "rewards/chosen": 9.1055908203125, + "rewards/margins": 12.5818510055542, + "rewards/rejected": -3.476259231567383, + "step": 1700 + }, + { + "epoch": 0.19, + "learning_rate": 9.159382577692264e-06, + "logits/chosen": -0.827900767326355, + "logits/rejected": 0.0768413171172142, + "logps/chosen": -229.66104125976562, + "logps/rejected": -162.81088256835938, + "loss": 0.0584, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 8.187432289123535, + "rewards/margins": 12.042821884155273, + "rewards/rejected": -3.855389356613159, + "step": 1710 + }, + { + "epoch": 0.19, + "learning_rate": 9.149788925472248e-06, + "logits/chosen": -0.6998910903930664, + "logits/rejected": 0.26318255066871643, + "logps/chosen": -178.77804565429688, + "logps/rejected": -171.1815185546875, + "loss": 0.0809, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": 8.187336921691895, + "rewards/margins": 11.490485191345215, + "rewards/rejected": -3.303149461746216, + "step": 1720 + }, + { + "epoch": 0.19, + "learning_rate": 9.14014592309185e-06, + "logits/chosen": -0.831656277179718, + "logits/rejected": -0.06519290804862976, + "logps/chosen": -185.37979125976562, + "logps/rejected": -156.30807495117188, + "loss": 0.0415, + "rewards/accuracies": 0.9750000238418579, + "rewards/chosen": 7.396572113037109, + "rewards/margins": 10.937728881835938, + "rewards/rejected": -3.5411572456359863, + "step": 1730 + }, + { + "epoch": 0.19, + "learning_rate": 9.130453685227679e-06, + "logits/chosen": -0.8854426145553589, + "logits/rejected": 0.19448325037956238, + "logps/chosen": -204.03329467773438, + "logps/rejected": -172.7010040283203, + "loss": 0.3383, + "rewards/accuracies": 0.9624999761581421, + "rewards/chosen": 8.299615859985352, + "rewards/margins": 11.909491539001465, + "rewards/rejected": -3.609875202178955, + "step": 1740 + }, + { + "epoch": 0.19, + "learning_rate": 9.120712327141858e-06, + "logits/chosen": -0.7734172344207764, + "logits/rejected": 0.3606404662132263, + "logps/chosen": -226.90707397460938, + "logps/rejected": -171.28457641601562, + "loss": 0.0259, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 8.864892959594727, + "rewards/margins": 13.222407341003418, + "rewards/rejected": -4.35751485824585, + "step": 1750 + }, + { + "epoch": 0.19, + "learning_rate": 9.110921964680665e-06, + "logits/chosen": -0.752662181854248, + "logits/rejected": 0.22373339533805847, + "logps/chosen": -205.64590454101562, + "logps/rejected": -160.78933715820312, + "loss": 0.1175, + "rewards/accuracies": 0.9375, + "rewards/chosen": 8.075421333312988, + "rewards/margins": 11.642030715942383, + "rewards/rejected": -3.566610336303711, + "step": 1760 + }, + { + "epoch": 0.19, + "learning_rate": 9.10108271427314e-06, + "logits/chosen": -0.9537338018417358, + "logits/rejected": 0.396420419216156, + "logps/chosen": -221.5945587158203, + "logps/rejected": -171.73233032226562, + "loss": 0.039, + "rewards/accuracies": 0.9750000238418579, + "rewards/chosen": 8.998331069946289, + "rewards/margins": 12.500982284545898, + "rewards/rejected": -3.502650737762451, + "step": 1770 + }, + { + "epoch": 0.2, + "learning_rate": 9.091194692929716e-06, + "logits/chosen": -0.48165398836135864, + "logits/rejected": 0.6215115785598755, + "logps/chosen": -179.63136291503906, + "logps/rejected": -170.05300903320312, + "loss": 0.0076, + "rewards/accuracies": 1.0, + "rewards/chosen": 7.977053642272949, + "rewards/margins": 11.927708625793457, + "rewards/rejected": -3.9506537914276123, + "step": 1780 + }, + { + "epoch": 0.2, + "learning_rate": 9.081258018240815e-06, + "logits/chosen": -0.5007386207580566, + "logits/rejected": 0.5413516759872437, + "logps/chosen": -210.7262725830078, + "logps/rejected": -165.52427673339844, + "loss": 0.0723, + "rewards/accuracies": 0.9750000238418579, + "rewards/chosen": 8.764327049255371, + "rewards/margins": 12.426679611206055, + "rewards/rejected": -3.662353038787842, + "step": 1790 + }, + { + "epoch": 0.2, + "learning_rate": 9.071272808375457e-06, + "logits/chosen": -0.5356274843215942, + "logits/rejected": 0.5369559526443481, + "logps/chosen": -232.5734405517578, + "logps/rejected": -174.54151916503906, + "loss": 0.0791, + "rewards/accuracies": 0.9375, + "rewards/chosen": 7.314566612243652, + "rewards/margins": 11.709010124206543, + "rewards/rejected": -4.394443511962891, + "step": 1800 + }, + { + "epoch": 0.2, + "learning_rate": 9.061239182079851e-06, + "logits/chosen": -0.4780815541744232, + "logits/rejected": 0.8617140054702759, + "logps/chosen": -262.71435546875, + "logps/rejected": -168.20004272460938, + "loss": 0.0126, + "rewards/accuracies": 1.0, + "rewards/chosen": 9.593928337097168, + "rewards/margins": 13.690221786499023, + "rewards/rejected": -4.096295356750488, + "step": 1810 + }, + { + "epoch": 0.2, + "learning_rate": 9.05115725867599e-06, + "logits/chosen": -0.3635478913784027, + "logits/rejected": 0.9611604809761047, + "logps/chosen": -232.6603546142578, + "logps/rejected": -163.62411499023438, + "loss": 0.1665, + "rewards/accuracies": 0.9624999761581421, + "rewards/chosen": 7.188954830169678, + "rewards/margins": 11.151288986206055, + "rewards/rejected": -3.962334156036377, + "step": 1820 + }, + { + "epoch": 0.2, + "learning_rate": 9.041027158060216e-06, + "logits/chosen": -0.3142985701560974, + "logits/rejected": 0.846194863319397, + "logps/chosen": -200.39620971679688, + "logps/rejected": -165.14682006835938, + "loss": 0.0169, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 7.522047519683838, + "rewards/margins": 11.614678382873535, + "rewards/rejected": -4.092629432678223, + "step": 1830 + }, + { + "epoch": 0.2, + "learning_rate": 9.030849000701815e-06, + "logits/chosen": -0.30892837047576904, + "logits/rejected": 0.7143265604972839, + "logps/chosen": -210.937255859375, + "logps/rejected": -162.75869750976562, + "loss": 0.0715, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": 7.18255090713501, + "rewards/margins": 11.248716354370117, + "rewards/rejected": -4.066165447235107, + "step": 1840 + }, + { + "epoch": 0.2, + "learning_rate": 9.020622907641568e-06, + "logits/chosen": -0.8121980428695679, + "logits/rejected": 0.7805534601211548, + "logps/chosen": -219.88967895507812, + "logps/rejected": -177.97035217285156, + "loss": 0.0041, + "rewards/accuracies": 1.0, + "rewards/chosen": 8.750587463378906, + "rewards/margins": 12.908777236938477, + "rewards/rejected": -4.15818977355957, + "step": 1850 + }, + { + "epoch": 0.2, + "learning_rate": 9.01034900049032e-06, + "logits/chosen": -0.7348997592926025, + "logits/rejected": 0.8176345825195312, + "logps/chosen": -229.6995391845703, + "logps/rejected": -180.89663696289062, + "loss": 0.069, + "rewards/accuracies": 0.9375, + "rewards/chosen": 8.388083457946777, + "rewards/margins": 12.502962112426758, + "rewards/rejected": -4.1148786544799805, + "step": 1860 + }, + { + "epoch": 0.21, + "learning_rate": 9.000027401427524e-06, + "logits/chosen": -0.28413471579551697, + "logits/rejected": 1.1145570278167725, + "logps/chosen": -216.53085327148438, + "logps/rejected": -160.30909729003906, + "loss": 0.0177, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 7.6873016357421875, + "rewards/margins": 12.045165061950684, + "rewards/rejected": -4.357863426208496, + "step": 1870 + }, + { + "epoch": 0.21, + "learning_rate": 8.98965823319981e-06, + "logits/chosen": -0.5494770407676697, + "logits/rejected": 0.7957988977432251, + "logps/chosen": -195.81954956054688, + "logps/rejected": -172.00186157226562, + "loss": 0.0415, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 7.2244873046875, + "rewards/margins": 11.97068977355957, + "rewards/rejected": -4.746201992034912, + "step": 1880 + }, + { + "epoch": 0.21, + "learning_rate": 8.979241619119496e-06, + "logits/chosen": -0.652524471282959, + "logits/rejected": 0.37755849957466125, + "logps/chosen": -210.0800323486328, + "logps/rejected": -173.26296997070312, + "loss": 0.0172, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 8.477156639099121, + "rewards/margins": 12.708356857299805, + "rewards/rejected": -4.231201648712158, + "step": 1890 + }, + { + "epoch": 0.21, + "learning_rate": 8.968777683063142e-06, + "logits/chosen": -0.9727234840393066, + "logits/rejected": 0.3874947428703308, + "logps/chosen": -241.3202362060547, + "logps/rejected": -169.428955078125, + "loss": 0.0129, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 8.604389190673828, + "rewards/margins": 13.190521240234375, + "rewards/rejected": -4.586132526397705, + "step": 1900 + }, + { + "epoch": 0.21, + "learning_rate": 8.958266549470072e-06, + "logits/chosen": -0.6917303800582886, + "logits/rejected": 0.503921627998352, + "logps/chosen": -199.63735961914062, + "logps/rejected": -166.35006713867188, + "loss": 0.0064, + "rewards/accuracies": 1.0, + "rewards/chosen": 8.385486602783203, + "rewards/margins": 12.161643981933594, + "rewards/rejected": -3.7761566638946533, + "step": 1910 + }, + { + "epoch": 0.21, + "learning_rate": 8.947708343340891e-06, + "logits/chosen": -0.4662698209285736, + "logits/rejected": 0.6918913125991821, + "logps/chosen": -176.79347229003906, + "logps/rejected": -166.1067352294922, + "loss": 0.0586, + "rewards/accuracies": 0.9624999761581421, + "rewards/chosen": 8.096259117126465, + "rewards/margins": 11.056158065795898, + "rewards/rejected": -2.959900140762329, + "step": 1920 + }, + { + "epoch": 0.21, + "learning_rate": 8.937103190236003e-06, + "logits/chosen": -0.32194486260414124, + "logits/rejected": 0.7147929072380066, + "logps/chosen": -215.06838989257812, + "logps/rejected": -168.64984130859375, + "loss": 0.0501, + "rewards/accuracies": 0.9624999761581421, + "rewards/chosen": 7.570442199707031, + "rewards/margins": 11.79222583770752, + "rewards/rejected": -4.221782684326172, + "step": 1930 + }, + { + "epoch": 0.21, + "learning_rate": 8.92645121627411e-06, + "logits/chosen": -0.20297428965568542, + "logits/rejected": 0.9233037233352661, + "logps/chosen": -241.4458770751953, + "logps/rejected": -169.84255981445312, + "loss": 0.0118, + "rewards/accuracies": 1.0, + "rewards/chosen": 8.750960350036621, + "rewards/margins": 12.908734321594238, + "rewards/rejected": -4.157774925231934, + "step": 1940 + }, + { + "epoch": 0.21, + "learning_rate": 8.915752548130721e-06, + "logits/chosen": -0.6144320964813232, + "logits/rejected": 0.5129267573356628, + "logps/chosen": -235.00973510742188, + "logps/rejected": -155.31793212890625, + "loss": 0.0595, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 8.357634544372559, + "rewards/margins": 12.143450736999512, + "rewards/rejected": -3.785815477371216, + "step": 1950 + }, + { + "epoch": 0.22, + "learning_rate": 8.905007313036644e-06, + "logits/chosen": -0.571103572845459, + "logits/rejected": 0.44340604543685913, + "logps/chosen": -252.47384643554688, + "logps/rejected": -152.4981689453125, + "loss": 0.167, + "rewards/accuracies": 0.9750000238418579, + "rewards/chosen": 8.95770263671875, + "rewards/margins": 12.356184005737305, + "rewards/rejected": -3.3984806537628174, + "step": 1960 + }, + { + "epoch": 0.22, + "learning_rate": 8.894215638776468e-06, + "logits/chosen": -0.4927874505519867, + "logits/rejected": 0.810386061668396, + "logps/chosen": -181.13320922851562, + "logps/rejected": -157.927978515625, + "loss": 0.0427, + "rewards/accuracies": 0.9750000238418579, + "rewards/chosen": 7.887845516204834, + "rewards/margins": 11.767987251281738, + "rewards/rejected": -3.8801403045654297, + "step": 1970 + }, + { + "epoch": 0.22, + "learning_rate": 8.883377653687046e-06, + "logits/chosen": -0.3371856212615967, + "logits/rejected": 0.7843553423881531, + "logps/chosen": -237.75802612304688, + "logps/rejected": -162.60653686523438, + "loss": 0.0209, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 8.312698364257812, + "rewards/margins": 12.682243347167969, + "rewards/rejected": -4.369544982910156, + "step": 1980 + }, + { + "epoch": 0.22, + "learning_rate": 8.87249348665597e-06, + "logits/chosen": -0.3437243103981018, + "logits/rejected": 0.6998990178108215, + "logps/chosen": -202.02743530273438, + "logps/rejected": -166.65127563476562, + "loss": 0.0705, + "rewards/accuracies": 0.9624999761581421, + "rewards/chosen": 7.631216526031494, + "rewards/margins": 11.797039985656738, + "rewards/rejected": -4.165823936462402, + "step": 1990 + }, + { + "epoch": 0.22, + "learning_rate": 8.861563267120041e-06, + "logits/chosen": -0.4705849587917328, + "logits/rejected": 0.5384072661399841, + "logps/chosen": -192.47850036621094, + "logps/rejected": -172.2180938720703, + "loss": 0.0345, + "rewards/accuracies": 0.9750000238418579, + "rewards/chosen": 7.673575401306152, + "rewards/margins": 12.357210159301758, + "rewards/rejected": -4.6836347579956055, + "step": 2000 + }, + { + "epoch": 0.22, + "learning_rate": 8.850587125063717e-06, + "logits/chosen": -0.33058807253837585, + "logits/rejected": 0.971091091632843, + "logps/chosen": -214.54623413085938, + "logps/rejected": -168.44432067871094, + "loss": 0.0881, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 7.962454319000244, + "rewards/margins": 12.074080467224121, + "rewards/rejected": -4.111626625061035, + "step": 2010 + }, + { + "epoch": 0.22, + "learning_rate": 8.839565191017587e-06, + "logits/chosen": -0.2974284887313843, + "logits/rejected": 0.97636878490448, + "logps/chosen": -235.8816680908203, + "logps/rejected": -175.85360717773438, + "loss": 0.0187, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 8.393792152404785, + "rewards/margins": 12.942235946655273, + "rewards/rejected": -4.548444747924805, + "step": 2020 + }, + { + "epoch": 0.22, + "learning_rate": 8.828497596056797e-06, + "logits/chosen": -0.1643526256084442, + "logits/rejected": 0.8214351534843445, + "logps/chosen": -181.15838623046875, + "logps/rejected": -162.52178955078125, + "loss": 0.1844, + "rewards/accuracies": 0.9624999761581421, + "rewards/chosen": 7.370779514312744, + "rewards/margins": 11.121458053588867, + "rewards/rejected": -3.7506790161132812, + "step": 2030 + }, + { + "epoch": 0.22, + "learning_rate": 8.817384471799514e-06, + "logits/chosen": -0.3292265832424164, + "logits/rejected": 1.094087839126587, + "logps/chosen": -208.61074829101562, + "logps/rejected": -167.3018035888672, + "loss": 0.0633, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": 7.627663612365723, + "rewards/margins": 12.10447883605957, + "rewards/rejected": -4.476813316345215, + "step": 2040 + }, + { + "epoch": 0.23, + "learning_rate": 8.806225950405338e-06, + "logits/chosen": -0.27208006381988525, + "logits/rejected": 0.9757376909255981, + "logps/chosen": -216.1806182861328, + "logps/rejected": -167.65142822265625, + "loss": 0.0108, + "rewards/accuracies": 1.0, + "rewards/chosen": 7.379110813140869, + "rewards/margins": 11.649734497070312, + "rewards/rejected": -4.270624160766602, + "step": 2050 + }, + { + "epoch": 0.23, + "learning_rate": 8.79502216457375e-06, + "logits/chosen": -0.47350209951400757, + "logits/rejected": 0.7839899659156799, + "logps/chosen": -212.70156860351562, + "logps/rejected": -170.24105834960938, + "loss": 0.0054, + "rewards/accuracies": 1.0, + "rewards/chosen": 8.414898872375488, + "rewards/margins": 12.99805736541748, + "rewards/rejected": -4.583159446716309, + "step": 2060 + }, + { + "epoch": 0.23, + "learning_rate": 8.783773247542518e-06, + "logits/chosen": -0.690665066242218, + "logits/rejected": 0.4333856701850891, + "logps/chosen": -233.6276397705078, + "logps/rejected": -180.33018493652344, + "loss": 0.0056, + "rewards/accuracies": 1.0, + "rewards/chosen": 9.300657272338867, + "rewards/margins": 13.976263046264648, + "rewards/rejected": -4.675606727600098, + "step": 2070 + }, + { + "epoch": 0.23, + "learning_rate": 8.772479333086122e-06, + "logits/chosen": -0.5696179270744324, + "logits/rejected": 0.5036565661430359, + "logps/chosen": -208.5790252685547, + "logps/rejected": -160.26483154296875, + "loss": 0.0131, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 8.516992568969727, + "rewards/margins": 12.505514144897461, + "rewards/rejected": -3.9885215759277344, + "step": 2080 + }, + { + "epoch": 0.23, + "learning_rate": 8.761140555514167e-06, + "logits/chosen": -0.8938825726509094, + "logits/rejected": 0.0002821937086991966, + "logps/chosen": -224.6837158203125, + "logps/rejected": -163.0015869140625, + "loss": 0.0345, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 9.163957595825195, + "rewards/margins": 13.306193351745605, + "rewards/rejected": -4.14223575592041, + "step": 2090 + }, + { + "epoch": 0.23, + "learning_rate": 8.74975704966977e-06, + "logits/chosen": -0.7335391640663147, + "logits/rejected": 0.13731873035430908, + "logps/chosen": -202.0205078125, + "logps/rejected": -184.16671752929688, + "loss": 0.0502, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 8.3814058303833, + "rewards/margins": 12.815424919128418, + "rewards/rejected": -4.434020042419434, + "step": 2100 + }, + { + "epoch": 0.23, + "learning_rate": 8.738328950927971e-06, + "logits/chosen": -0.6339744329452515, + "logits/rejected": 0.1389215737581253, + "logps/chosen": -214.9847869873047, + "logps/rejected": -174.8680877685547, + "loss": 0.0095, + "rewards/accuracies": 1.0, + "rewards/chosen": 8.587041854858398, + "rewards/margins": 12.97746467590332, + "rewards/rejected": -4.390421390533447, + "step": 2110 + }, + { + "epoch": 0.23, + "learning_rate": 8.726856395194119e-06, + "logits/chosen": -0.8502337336540222, + "logits/rejected": 0.08457791805267334, + "logps/chosen": -240.1321563720703, + "logps/rejected": -180.6337432861328, + "loss": 0.1664, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": 8.162525177001953, + "rewards/margins": 12.321939468383789, + "rewards/rejected": -4.159415245056152, + "step": 2120 + }, + { + "epoch": 0.23, + "learning_rate": 8.715339518902253e-06, + "logits/chosen": -0.9064286947250366, + "logits/rejected": -0.1819077879190445, + "logps/chosen": -198.82833862304688, + "logps/rejected": -167.35934448242188, + "loss": 0.093, + "rewards/accuracies": 0.9624999761581421, + "rewards/chosen": 8.25260066986084, + "rewards/margins": 12.027249336242676, + "rewards/rejected": -3.7746481895446777, + "step": 2130 + }, + { + "epoch": 0.23, + "learning_rate": 8.703778459013479e-06, + "logits/chosen": -0.9572340846061707, + "logits/rejected": -0.146784707903862, + "logps/chosen": -186.52105712890625, + "logps/rejected": -160.72584533691406, + "loss": 0.0249, + "rewards/accuracies": 1.0, + "rewards/chosen": 7.873948574066162, + "rewards/margins": 12.232105255126953, + "rewards/rejected": -4.358155727386475, + "step": 2140 + }, + { + "epoch": 0.24, + "learning_rate": 8.692173353014351e-06, + "logits/chosen": -0.9497688412666321, + "logits/rejected": -0.1295793503522873, + "logps/chosen": -215.9934539794922, + "logps/rejected": -175.27833557128906, + "loss": 0.0175, + "rewards/accuracies": 1.0, + "rewards/chosen": 8.687699317932129, + "rewards/margins": 13.0452299118042, + "rewards/rejected": -4.357530117034912, + "step": 2150 + }, + { + "epoch": 0.24, + "learning_rate": 8.68052433891522e-06, + "logits/chosen": -0.9528650045394897, + "logits/rejected": -0.11717786639928818, + "logps/chosen": -220.6671905517578, + "logps/rejected": -152.87850952148438, + "loss": 0.1651, + "rewards/accuracies": 0.9750000238418579, + "rewards/chosen": 8.587167739868164, + "rewards/margins": 11.920230865478516, + "rewards/rejected": -3.3330631256103516, + "step": 2160 + }, + { + "epoch": 0.24, + "learning_rate": 8.668831555248607e-06, + "logits/chosen": -0.7263249158859253, + "logits/rejected": 0.23681822419166565, + "logps/chosen": -177.57666015625, + "logps/rejected": -185.82980346679688, + "loss": 0.1393, + "rewards/accuracies": 0.9750000238418579, + "rewards/chosen": 7.666592597961426, + "rewards/margins": 11.784173011779785, + "rewards/rejected": -4.117580890655518, + "step": 2170 + }, + { + "epoch": 0.24, + "learning_rate": 8.657095141067546e-06, + "logits/chosen": -0.43287286162376404, + "logits/rejected": 0.7004810571670532, + "logps/chosen": -189.4801025390625, + "logps/rejected": -163.88528442382812, + "loss": 0.0881, + "rewards/accuracies": 0.9624999761581421, + "rewards/chosen": 7.4667816162109375, + "rewards/margins": 11.954621315002441, + "rewards/rejected": -4.487839698791504, + "step": 2180 + }, + { + "epoch": 0.24, + "learning_rate": 8.645315235943939e-06, + "logits/chosen": -0.768271267414093, + "logits/rejected": 0.12874235212802887, + "logps/chosen": -217.19259643554688, + "logps/rejected": -167.7063751220703, + "loss": 0.0842, + "rewards/accuracies": 0.9750000238418579, + "rewards/chosen": 8.23132610321045, + "rewards/margins": 12.512828826904297, + "rewards/rejected": -4.2815022468566895, + "step": 2190 + }, + { + "epoch": 0.24, + "learning_rate": 8.633491979966888e-06, + "logits/chosen": -0.9630998373031616, + "logits/rejected": 0.4160458445549011, + "logps/chosen": -229.47616577148438, + "logps/rejected": -186.0326690673828, + "loss": 0.0697, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 8.416041374206543, + "rewards/margins": 12.873327255249023, + "rewards/rejected": -4.457285404205322, + "step": 2200 + }, + { + "epoch": 0.24, + "learning_rate": 8.621625513741033e-06, + "logits/chosen": -0.5632399320602417, + "logits/rejected": 0.3805318772792816, + "logps/chosen": -228.39486694335938, + "logps/rejected": -169.94186401367188, + "loss": 0.1161, + "rewards/accuracies": 0.9624999761581421, + "rewards/chosen": 7.571549415588379, + "rewards/margins": 12.16765022277832, + "rewards/rejected": -4.596099853515625, + "step": 2210 + }, + { + "epoch": 0.24, + "learning_rate": 8.60971597838488e-06, + "logits/chosen": -0.6236270666122437, + "logits/rejected": 0.4582180976867676, + "logps/chosen": -238.4077606201172, + "logps/rejected": -174.6767578125, + "loss": 0.006, + "rewards/accuracies": 1.0, + "rewards/chosen": 8.159147262573242, + "rewards/margins": 13.25964641571045, + "rewards/rejected": -5.100498676300049, + "step": 2220 + }, + { + "epoch": 0.24, + "learning_rate": 8.59776351552912e-06, + "logits/chosen": -0.395003080368042, + "logits/rejected": 0.7398403882980347, + "logps/chosen": -170.48580932617188, + "logps/rejected": -165.9557342529297, + "loss": 0.0223, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 7.435987949371338, + "rewards/margins": 12.090422630310059, + "rewards/rejected": -4.654435157775879, + "step": 2230 + }, + { + "epoch": 0.25, + "learning_rate": 8.585768267314949e-06, + "logits/chosen": -0.37745702266693115, + "logits/rejected": 0.6807891130447388, + "logps/chosen": -190.44113159179688, + "logps/rejected": -174.80258178710938, + "loss": 0.0156, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 7.024377346038818, + "rewards/margins": 11.530973434448242, + "rewards/rejected": -4.506596565246582, + "step": 2240 + }, + { + "epoch": 0.25, + "learning_rate": 8.573730376392376e-06, + "logits/chosen": -0.5341759920120239, + "logits/rejected": 0.3510711193084717, + "logps/chosen": -202.7803192138672, + "logps/rejected": -181.39561462402344, + "loss": 0.0102, + "rewards/accuracies": 1.0, + "rewards/chosen": 8.461994171142578, + "rewards/margins": 13.099119186401367, + "rewards/rejected": -4.637126922607422, + "step": 2250 + }, + { + "epoch": 0.25, + "learning_rate": 8.561649985918526e-06, + "logits/chosen": -0.5021628141403198, + "logits/rejected": 0.3783276081085205, + "logps/chosen": -190.25570678710938, + "logps/rejected": -164.2488250732422, + "loss": 0.0259, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 7.367849826812744, + "rewards/margins": 11.650557518005371, + "rewards/rejected": -4.282708168029785, + "step": 2260 + }, + { + "epoch": 0.25, + "learning_rate": 8.549527239555933e-06, + "logits/chosen": -0.8880068063735962, + "logits/rejected": 0.04517006874084473, + "logps/chosen": -218.488525390625, + "logps/rejected": -173.66485595703125, + "loss": 0.0597, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 8.142195701599121, + "rewards/margins": 12.357624053955078, + "rewards/rejected": -4.215428352355957, + "step": 2270 + }, + { + "epoch": 0.25, + "learning_rate": 8.537362281470842e-06, + "logits/chosen": -0.8509469032287598, + "logits/rejected": 0.16993197798728943, + "logps/chosen": -225.26181030273438, + "logps/rejected": -165.80532836914062, + "loss": 0.0196, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 7.537097930908203, + "rewards/margins": 12.063167572021484, + "rewards/rejected": -4.526069641113281, + "step": 2280 + }, + { + "epoch": 0.25, + "learning_rate": 8.525155256331485e-06, + "logits/chosen": -0.6800307631492615, + "logits/rejected": 0.25085148215293884, + "logps/chosen": -221.555419921875, + "logps/rejected": -167.46531677246094, + "loss": 0.0208, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 7.689160346984863, + "rewards/margins": 11.93071174621582, + "rewards/rejected": -4.24155330657959, + "step": 2290 + }, + { + "epoch": 0.25, + "learning_rate": 8.512906309306369e-06, + "logits/chosen": -0.4905932545661926, + "logits/rejected": 0.41362684965133667, + "logps/chosen": -190.53590393066406, + "logps/rejected": -179.59628295898438, + "loss": 0.0392, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 8.085442543029785, + "rewards/margins": 13.140932083129883, + "rewards/rejected": -5.055489540100098, + "step": 2300 + }, + { + "epoch": 0.25, + "learning_rate": 8.500615586062537e-06, + "logits/chosen": -0.7107552886009216, + "logits/rejected": 0.2486438751220703, + "logps/chosen": -223.39205932617188, + "logps/rejected": -165.79603576660156, + "loss": 0.0127, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 7.861292839050293, + "rewards/margins": 12.904462814331055, + "rewards/rejected": -5.043169975280762, + "step": 2310 + }, + { + "epoch": 0.25, + "learning_rate": 8.48828323276385e-06, + "logits/chosen": -0.2899216115474701, + "logits/rejected": 0.7743111252784729, + "logps/chosen": -195.27996826171875, + "logps/rejected": -168.7857666015625, + "loss": 0.046, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 8.390793800354004, + "rewards/margins": 12.747232437133789, + "rewards/rejected": -4.356438636779785, + "step": 2320 + }, + { + "epoch": 0.26, + "learning_rate": 8.475909396069243e-06, + "logits/chosen": -0.8413451910018921, + "logits/rejected": 0.5604926347732544, + "logps/chosen": -224.261962890625, + "logps/rejected": -183.05560302734375, + "loss": 0.0574, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 8.465588569641113, + "rewards/margins": 13.44543743133545, + "rewards/rejected": -4.979849815368652, + "step": 2330 + }, + { + "epoch": 0.26, + "learning_rate": 8.463494223130972e-06, + "logits/chosen": -0.8377776145935059, + "logits/rejected": 0.26796481013298035, + "logps/chosen": -233.9268035888672, + "logps/rejected": -167.1359405517578, + "loss": 0.0115, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 8.433236122131348, + "rewards/margins": 12.992330551147461, + "rewards/rejected": -4.559096336364746, + "step": 2340 + }, + { + "epoch": 0.26, + "learning_rate": 8.451037861592882e-06, + "logits/chosen": -0.531271755695343, + "logits/rejected": 0.5945345163345337, + "logps/chosen": -233.77487182617188, + "logps/rejected": -171.58950805664062, + "loss": 0.0076, + "rewards/accuracies": 1.0, + "rewards/chosen": 7.894737243652344, + "rewards/margins": 12.823348999023438, + "rewards/rejected": -4.9286112785339355, + "step": 2350 + }, + { + "epoch": 0.26, + "learning_rate": 8.438540459588635e-06, + "logits/chosen": -0.3315597176551819, + "logits/rejected": 0.8130376935005188, + "logps/chosen": -179.4055938720703, + "logps/rejected": -166.93429565429688, + "loss": 0.1071, + "rewards/accuracies": 0.9624999761581421, + "rewards/chosen": 6.321532726287842, + "rewards/margins": 10.406835556030273, + "rewards/rejected": -4.085303783416748, + "step": 2360 + }, + { + "epoch": 0.26, + "learning_rate": 8.426002165739959e-06, + "logits/chosen": -0.46008366346359253, + "logits/rejected": 0.7065877318382263, + "logps/chosen": -208.07479858398438, + "logps/rejected": -170.52133178710938, + "loss": 0.0107, + "rewards/accuracies": 1.0, + "rewards/chosen": 7.816873073577881, + "rewards/margins": 12.425461769104004, + "rewards/rejected": -4.608588218688965, + "step": 2370 + }, + { + "epoch": 0.26, + "learning_rate": 8.413423129154873e-06, + "logits/chosen": -0.6410793662071228, + "logits/rejected": 0.3276979923248291, + "logps/chosen": -234.51693725585938, + "logps/rejected": -169.89077758789062, + "loss": 0.0061, + "rewards/accuracies": 1.0, + "rewards/chosen": 8.718478202819824, + "rewards/margins": 13.924470901489258, + "rewards/rejected": -5.205992221832275, + "step": 2380 + }, + { + "epoch": 0.26, + "learning_rate": 8.40080349942592e-06, + "logits/chosen": -0.5077573657035828, + "logits/rejected": 0.3929711878299713, + "logps/chosen": -192.05772399902344, + "logps/rejected": -169.24960327148438, + "loss": 0.1019, + "rewards/accuracies": 0.9750000238418579, + "rewards/chosen": 7.895243167877197, + "rewards/margins": 12.026467323303223, + "rewards/rejected": -4.131225109100342, + "step": 2390 + }, + { + "epoch": 0.26, + "learning_rate": 8.388143426628384e-06, + "logits/chosen": -0.6147021651268005, + "logits/rejected": 0.23026438057422638, + "logps/chosen": -228.5024871826172, + "logps/rejected": -170.653076171875, + "loss": 0.0201, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 8.201008796691895, + "rewards/margins": 13.116500854492188, + "rewards/rejected": -4.915493488311768, + "step": 2400 + }, + { + "epoch": 0.26, + "learning_rate": 8.375443061318504e-06, + "logits/chosen": -0.6069657206535339, + "logits/rejected": 0.5055216550827026, + "logps/chosen": -182.94375610351562, + "logps/rejected": -178.98904418945312, + "loss": 0.0367, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 7.235411167144775, + "rewards/margins": 11.979175567626953, + "rewards/rejected": -4.743765830993652, + "step": 2410 + }, + { + "epoch": 0.27, + "learning_rate": 8.36270255453169e-06, + "logits/chosen": -0.6311702728271484, + "logits/rejected": 0.5873549580574036, + "logps/chosen": -216.9287567138672, + "logps/rejected": -169.9967803955078, + "loss": 0.0112, + "rewards/accuracies": 1.0, + "rewards/chosen": 8.0543851852417, + "rewards/margins": 13.188531875610352, + "rewards/rejected": -5.1341471672058105, + "step": 2420 + }, + { + "epoch": 0.27, + "learning_rate": 8.349922057780723e-06, + "logits/chosen": -0.5530081987380981, + "logits/rejected": 0.7694286108016968, + "logps/chosen": -223.22305297851562, + "logps/rejected": -187.7498016357422, + "loss": 0.0015, + "rewards/accuracies": 1.0, + "rewards/chosen": 8.594778060913086, + "rewards/margins": 13.714917182922363, + "rewards/rejected": -5.120139122009277, + "step": 2430 + }, + { + "epoch": 0.27, + "learning_rate": 8.337101723053949e-06, + "logits/chosen": -0.4461049437522888, + "logits/rejected": 0.6591795682907104, + "logps/chosen": -230.85140991210938, + "logps/rejected": -166.4679718017578, + "loss": 0.0101, + "rewards/accuracies": 1.0, + "rewards/chosen": 8.5826416015625, + "rewards/margins": 13.19091510772705, + "rewards/rejected": -4.608273506164551, + "step": 2440 + }, + { + "epoch": 0.27, + "learning_rate": 8.324241702813477e-06, + "logits/chosen": -0.4857296049594879, + "logits/rejected": 0.5430890321731567, + "logps/chosen": -187.19033813476562, + "logps/rejected": -172.521728515625, + "loss": 0.0192, + "rewards/accuracies": 1.0, + "rewards/chosen": 6.923466682434082, + "rewards/margins": 11.60053539276123, + "rewards/rejected": -4.677069187164307, + "step": 2450 + }, + { + "epoch": 0.27, + "learning_rate": 8.311342149993366e-06, + "logits/chosen": -0.9044029116630554, + "logits/rejected": 0.05248773843050003, + "logps/chosen": -217.3992156982422, + "logps/rejected": -167.77630615234375, + "loss": 0.0012, + "rewards/accuracies": 1.0, + "rewards/chosen": 8.785954475402832, + "rewards/margins": 13.33287525177002, + "rewards/rejected": -4.5469207763671875, + "step": 2460 + }, + { + "epoch": 0.27, + "learning_rate": 8.298403217997805e-06, + "logits/chosen": -1.2175676822662354, + "logits/rejected": -0.6908454895019531, + "logps/chosen": -269.8515930175781, + "logps/rejected": -162.03921508789062, + "loss": 0.0145, + "rewards/accuracies": 1.0, + "rewards/chosen": 9.291863441467285, + "rewards/margins": 13.004359245300293, + "rewards/rejected": -3.7124953269958496, + "step": 2470 + }, + { + "epoch": 0.27, + "learning_rate": 8.285425060699291e-06, + "logits/chosen": -1.0893975496292114, + "logits/rejected": -0.1912747323513031, + "logps/chosen": -194.61566162109375, + "logps/rejected": -182.98367309570312, + "loss": 0.0215, + "rewards/accuracies": 0.9750000238418579, + "rewards/chosen": 8.65707778930664, + "rewards/margins": 12.879467964172363, + "rewards/rejected": -4.222390174865723, + "step": 2480 + }, + { + "epoch": 0.27, + "learning_rate": 8.272407832436784e-06, + "logits/chosen": -0.6341437697410583, + "logits/rejected": 0.3419879674911499, + "logps/chosen": -171.78280639648438, + "logps/rejected": -165.11734008789062, + "loss": 0.019, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 7.791851043701172, + "rewards/margins": 12.213276863098145, + "rewards/rejected": -4.421425819396973, + "step": 2490 + }, + { + "epoch": 0.27, + "learning_rate": 8.2593516880139e-06, + "logits/chosen": -0.5838027000427246, + "logits/rejected": 0.4249269962310791, + "logps/chosen": -202.65524291992188, + "logps/rejected": -171.35055541992188, + "loss": 0.1298, + "rewards/accuracies": 0.9750000238418579, + "rewards/chosen": 8.352968215942383, + "rewards/margins": 12.428132057189941, + "rewards/rejected": -4.0751633644104, + "step": 2500 + }, + { + "epoch": 0.28, + "learning_rate": 8.246256782697045e-06, + "logits/chosen": -0.7317477464675903, + "logits/rejected": 0.41557279229164124, + "logps/chosen": -209.4647674560547, + "logps/rejected": -174.3729248046875, + "loss": 0.0038, + "rewards/accuracies": 1.0, + "rewards/chosen": 8.367586135864258, + "rewards/margins": 12.99022102355957, + "rewards/rejected": -4.622633934020996, + "step": 2510 + }, + { + "epoch": 0.28, + "learning_rate": 8.23312327221358e-06, + "logits/chosen": -0.692236065864563, + "logits/rejected": 0.46434178948402405, + "logps/chosen": -251.1358184814453, + "logps/rejected": -174.54110717773438, + "loss": 0.0319, + "rewards/accuracies": 0.9750000238418579, + "rewards/chosen": 8.932543754577637, + "rewards/margins": 13.945353507995605, + "rewards/rejected": -5.012808799743652, + "step": 2520 + }, + { + "epoch": 0.28, + "learning_rate": 8.219951312749967e-06, + "logits/chosen": -0.6690990328788757, + "logits/rejected": 0.569442868232727, + "logps/chosen": -217.0177001953125, + "logps/rejected": -177.09072875976562, + "loss": 0.032, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 8.575159072875977, + "rewards/margins": 13.195083618164062, + "rewards/rejected": -4.619925022125244, + "step": 2530 + }, + { + "epoch": 0.28, + "learning_rate": 8.20674106094991e-06, + "logits/chosen": -0.4990970492362976, + "logits/rejected": 0.4127965569496155, + "logps/chosen": -195.69264221191406, + "logps/rejected": -175.87966918945312, + "loss": 0.0311, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 7.397913455963135, + "rewards/margins": 11.768786430358887, + "rewards/rejected": -4.370872974395752, + "step": 2540 + }, + { + "epoch": 0.28, + "learning_rate": 8.193492673912495e-06, + "logits/chosen": -0.5780926942825317, + "logits/rejected": 0.48505130410194397, + "logps/chosen": -183.80203247070312, + "logps/rejected": -175.343994140625, + "loss": 0.1525, + "rewards/accuracies": 0.9624999761581421, + "rewards/chosen": 8.53159236907959, + "rewards/margins": 13.045809745788574, + "rewards/rejected": -4.514214515686035, + "step": 2550 + }, + { + "epoch": 0.28, + "learning_rate": 8.18020630919032e-06, + "logits/chosen": -0.5260237455368042, + "logits/rejected": 0.5624647736549377, + "logps/chosen": -175.13485717773438, + "logps/rejected": -170.07009887695312, + "loss": 0.008, + "rewards/accuracies": 1.0, + "rewards/chosen": 7.5792131423950195, + "rewards/margins": 11.603866577148438, + "rewards/rejected": -4.024654388427734, + "step": 2560 + }, + { + "epoch": 0.28, + "learning_rate": 8.166882124787623e-06, + "logits/chosen": -0.47336331009864807, + "logits/rejected": 0.6014837026596069, + "logps/chosen": -202.11178588867188, + "logps/rejected": -182.88864135742188, + "loss": 0.0248, + "rewards/accuracies": 0.9750000238418579, + "rewards/chosen": 7.935551643371582, + "rewards/margins": 12.890029907226562, + "rewards/rejected": -4.954479217529297, + "step": 2570 + }, + { + "epoch": 0.28, + "learning_rate": 8.153520279158396e-06, + "logits/chosen": -0.34318774938583374, + "logits/rejected": 0.6932908892631531, + "logps/chosen": -174.56124877929688, + "logps/rejected": -174.02755737304688, + "loss": 0.0931, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": 6.897540092468262, + "rewards/margins": 11.409690856933594, + "rewards/rejected": -4.512150764465332, + "step": 2580 + }, + { + "epoch": 0.28, + "learning_rate": 8.140120931204512e-06, + "logits/chosen": -0.6497650742530823, + "logits/rejected": 0.5149052739143372, + "logps/chosen": -243.260009765625, + "logps/rejected": -170.65261840820312, + "loss": 0.0969, + "rewards/accuracies": 0.9750000238418579, + "rewards/chosen": 8.357889175415039, + "rewards/margins": 12.79127311706543, + "rewards/rejected": -4.433384418487549, + "step": 2590 + }, + { + "epoch": 0.29, + "learning_rate": 8.126684240273827e-06, + "logits/chosen": -0.6540253162384033, + "logits/rejected": 0.3523900508880615, + "logps/chosen": -200.621337890625, + "logps/rejected": -171.65789794921875, + "loss": 0.0253, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 7.952597618103027, + "rewards/margins": 12.75285530090332, + "rewards/rejected": -4.800256729125977, + "step": 2600 + }, + { + "epoch": 0.29, + "learning_rate": 8.113210366158289e-06, + "logits/chosen": -0.9135942459106445, + "logits/rejected": -0.017345678061246872, + "logps/chosen": -194.4808807373047, + "logps/rejected": -160.66390991210938, + "loss": 0.1005, + "rewards/accuracies": 0.9624999761581421, + "rewards/chosen": 7.563905239105225, + "rewards/margins": 11.910043716430664, + "rewards/rejected": -4.346137523651123, + "step": 2610 + }, + { + "epoch": 0.29, + "learning_rate": 8.099699469092035e-06, + "logits/chosen": -1.0905654430389404, + "logits/rejected": -0.7532132863998413, + "logps/chosen": -199.40904235839844, + "logps/rejected": -162.73121643066406, + "loss": 0.0104, + "rewards/accuracies": 1.0, + "rewards/chosen": 8.886347770690918, + "rewards/margins": 13.453600883483887, + "rewards/rejected": -4.567252159118652, + "step": 2620 + }, + { + "epoch": 0.29, + "learning_rate": 8.086151709749488e-06, + "logits/chosen": -0.793353796005249, + "logits/rejected": 0.19234681129455566, + "logps/chosen": -172.5845489501953, + "logps/rejected": -171.7484893798828, + "loss": 0.0845, + "rewards/accuracies": 0.9624999761581421, + "rewards/chosen": 8.292520523071289, + "rewards/margins": 12.046894073486328, + "rewards/rejected": -3.7543742656707764, + "step": 2630 + }, + { + "epoch": 0.29, + "learning_rate": 8.072567249243444e-06, + "logits/chosen": -1.1524503231048584, + "logits/rejected": -0.16285154223442078, + "logps/chosen": -229.02871704101562, + "logps/rejected": -168.34915161132812, + "loss": 0.099, + "rewards/accuracies": 0.9624999761581421, + "rewards/chosen": 9.448681831359863, + "rewards/margins": 13.124113082885742, + "rewards/rejected": -3.6754310131073, + "step": 2640 + }, + { + "epoch": 0.29, + "learning_rate": 8.058946249123155e-06, + "logits/chosen": -1.0330545902252197, + "logits/rejected": -0.21650974452495575, + "logps/chosen": -231.02783203125, + "logps/rejected": -186.7714385986328, + "loss": 0.0003, + "rewards/accuracies": 1.0, + "rewards/chosen": 8.94650650024414, + "rewards/margins": 14.374897956848145, + "rewards/rejected": -5.428391456604004, + "step": 2650 + }, + { + "epoch": 0.29, + "learning_rate": 8.04528887137241e-06, + "logits/chosen": -0.8955181241035461, + "logits/rejected": -0.10996793210506439, + "logps/chosen": -197.42031860351562, + "logps/rejected": -171.0691680908203, + "loss": 0.0405, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 8.19475269317627, + "rewards/margins": 12.796459197998047, + "rewards/rejected": -4.601706027984619, + "step": 2660 + }, + { + "epoch": 0.29, + "learning_rate": 8.031595278407613e-06, + "logits/chosen": -1.051721215248108, + "logits/rejected": -0.3572627604007721, + "logps/chosen": -206.19784545898438, + "logps/rejected": -167.8656005859375, + "loss": 0.0466, + "rewards/accuracies": 0.9750000238418579, + "rewards/chosen": 9.205878257751465, + "rewards/margins": 13.923624992370605, + "rewards/rejected": -4.717745780944824, + "step": 2670 + }, + { + "epoch": 0.29, + "learning_rate": 8.017865633075846e-06, + "logits/chosen": -0.9133942723274231, + "logits/rejected": -0.07959525287151337, + "logps/chosen": -202.22283935546875, + "logps/rejected": -167.33413696289062, + "loss": 0.0702, + "rewards/accuracies": 0.9624999761581421, + "rewards/chosen": 8.584909439086914, + "rewards/margins": 12.664779663085938, + "rewards/rejected": -4.079871654510498, + "step": 2680 + }, + { + "epoch": 0.3, + "learning_rate": 8.004100098652926e-06, + "logits/chosen": -0.7203928232192993, + "logits/rejected": -0.05325474217534065, + "logps/chosen": -182.52957153320312, + "logps/rejected": -163.93081665039062, + "loss": 0.1392, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": 8.526405334472656, + "rewards/margins": 12.884794235229492, + "rewards/rejected": -4.358389854431152, + "step": 2690 + }, + { + "epoch": 0.3, + "learning_rate": 7.990298838841481e-06, + "logits/chosen": -0.976810097694397, + "logits/rejected": -0.26609936356544495, + "logps/chosen": -211.3890838623047, + "logps/rejected": -176.94076538085938, + "loss": 0.0172, + "rewards/accuracies": 1.0, + "rewards/chosen": 8.936230659484863, + "rewards/margins": 12.897746086120605, + "rewards/rejected": -3.961515426635742, + "step": 2700 + }, + { + "epoch": 0.3, + "learning_rate": 7.976462017768984e-06, + "logits/chosen": -0.7487978935241699, + "logits/rejected": 0.2039092481136322, + "logps/chosen": -196.99449157714844, + "logps/rejected": -177.93943786621094, + "loss": 0.0199, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 7.814328193664551, + "rewards/margins": 12.83709716796875, + "rewards/rejected": -5.022768497467041, + "step": 2710 + }, + { + "epoch": 0.3, + "learning_rate": 7.962589799985817e-06, + "logits/chosen": -0.9832037091255188, + "logits/rejected": 0.010686754249036312, + "logps/chosen": -230.860595703125, + "logps/rejected": -179.5846710205078, + "loss": 0.1128, + "rewards/accuracies": 0.9624999761581421, + "rewards/chosen": 8.199392318725586, + "rewards/margins": 12.86755657196045, + "rewards/rejected": -4.6681647300720215, + "step": 2720 + }, + { + "epoch": 0.3, + "learning_rate": 7.948682350463299e-06, + "logits/chosen": -0.6001917719841003, + "logits/rejected": 0.41285085678100586, + "logps/chosen": -200.84994506835938, + "logps/rejected": -172.69180297851562, + "loss": 0.0687, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 7.3350114822387695, + "rewards/margins": 12.243888854980469, + "rewards/rejected": -4.908877372741699, + "step": 2730 + }, + { + "epoch": 0.3, + "learning_rate": 7.934739834591743e-06, + "logits/chosen": -0.7725926637649536, + "logits/rejected": 0.5752295255661011, + "logps/chosen": -219.83322143554688, + "logps/rejected": -185.51937866210938, + "loss": 0.0206, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 7.2027106285095215, + "rewards/margins": 12.59252643585205, + "rewards/rejected": -5.389814853668213, + "step": 2740 + }, + { + "epoch": 0.3, + "learning_rate": 7.920762418178467e-06, + "logits/chosen": -0.7085618376731873, + "logits/rejected": 0.6622563600540161, + "logps/chosen": -198.0343017578125, + "logps/rejected": -173.04983520507812, + "loss": 0.0028, + "rewards/accuracies": 1.0, + "rewards/chosen": 7.681931495666504, + "rewards/margins": 12.312196731567383, + "rewards/rejected": -4.630265235900879, + "step": 2750 + }, + { + "epoch": 0.3, + "learning_rate": 7.906750267445842e-06, + "logits/chosen": -0.5810704827308655, + "logits/rejected": 0.5701958537101746, + "logps/chosen": -191.88255310058594, + "logps/rejected": -167.3490447998047, + "loss": 0.1238, + "rewards/accuracies": 0.9624999761581421, + "rewards/chosen": 7.207613468170166, + "rewards/margins": 12.253547668457031, + "rewards/rejected": -5.045934677124023, + "step": 2760 + }, + { + "epoch": 0.3, + "learning_rate": 7.892703549029302e-06, + "logits/chosen": -0.618598997592926, + "logits/rejected": 0.6705722212791443, + "logps/chosen": -200.2035675048828, + "logps/rejected": -180.4983673095703, + "loss": 0.0843, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 7.773738861083984, + "rewards/margins": 13.073773384094238, + "rewards/rejected": -5.3000359535217285, + "step": 2770 + }, + { + "epoch": 0.31, + "learning_rate": 7.878622429975369e-06, + "logits/chosen": -0.4437601566314697, + "logits/rejected": 0.6392486691474915, + "logps/chosen": -186.42111206054688, + "logps/rejected": -176.2037811279297, + "loss": 0.1384, + "rewards/accuracies": 0.9624999761581421, + "rewards/chosen": 7.519711971282959, + "rewards/margins": 12.133156776428223, + "rewards/rejected": -4.613443374633789, + "step": 2780 + }, + { + "epoch": 0.31, + "learning_rate": 7.864507077739667e-06, + "logits/chosen": -0.520032525062561, + "logits/rejected": 0.4803471565246582, + "logps/chosen": -231.8350372314453, + "logps/rejected": -176.01327514648438, + "loss": 0.1153, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 9.256172180175781, + "rewards/margins": 14.020452499389648, + "rewards/rejected": -4.764278888702393, + "step": 2790 + }, + { + "epoch": 0.31, + "learning_rate": 7.850357660184924e-06, + "logits/chosen": -0.6796492338180542, + "logits/rejected": 0.6617184281349182, + "logps/chosen": -227.59487915039062, + "logps/rejected": -177.05235290527344, + "loss": 0.0812, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 7.946455478668213, + "rewards/margins": 13.443826675415039, + "rewards/rejected": -5.497370719909668, + "step": 2800 + }, + { + "epoch": 0.31, + "learning_rate": 7.836174345578982e-06, + "logits/chosen": -0.7396942973136902, + "logits/rejected": 0.3257351517677307, + "logps/chosen": -213.6725311279297, + "logps/rejected": -181.46981811523438, + "loss": 0.0114, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 8.922917366027832, + "rewards/margins": 13.603009223937988, + "rewards/rejected": -4.680091857910156, + "step": 2810 + }, + { + "epoch": 0.31, + "learning_rate": 7.821957302592791e-06, + "logits/chosen": -0.8451019525527954, + "logits/rejected": 0.3018733561038971, + "logps/chosen": -227.48641967773438, + "logps/rejected": -178.04055786132812, + "loss": 0.0381, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 8.504293441772461, + "rewards/margins": 13.973077774047852, + "rewards/rejected": -5.468785762786865, + "step": 2820 + }, + { + "epoch": 0.31, + "learning_rate": 7.807706700298412e-06, + "logits/chosen": -0.7736309766769409, + "logits/rejected": 0.44020017981529236, + "logps/chosen": -212.9801483154297, + "logps/rejected": -175.34637451171875, + "loss": 0.0634, + "rewards/accuracies": 0.9750000238418579, + "rewards/chosen": 8.439635276794434, + "rewards/margins": 13.418614387512207, + "rewards/rejected": -4.978979110717773, + "step": 2830 + }, + { + "epoch": 0.31, + "learning_rate": 7.793422708166996e-06, + "logits/chosen": -0.7209655046463013, + "logits/rejected": 0.425056517124176, + "logps/chosen": -216.6347198486328, + "logps/rejected": -179.53964233398438, + "loss": 0.1194, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 7.9016618728637695, + "rewards/margins": 12.855749130249023, + "rewards/rejected": -4.95408821105957, + "step": 2840 + }, + { + "epoch": 0.31, + "learning_rate": 7.779105496066772e-06, + "logits/chosen": -0.6034275889396667, + "logits/rejected": 0.4955766797065735, + "logps/chosen": -208.5738067626953, + "logps/rejected": -185.705322265625, + "loss": 0.0025, + "rewards/accuracies": 1.0, + "rewards/chosen": 7.3899245262146, + "rewards/margins": 12.2137451171875, + "rewards/rejected": -4.823821067810059, + "step": 2850 + }, + { + "epoch": 0.31, + "learning_rate": 7.76475523426103e-06, + "logits/chosen": -0.6930392980575562, + "logits/rejected": 0.28452375531196594, + "logps/chosen": -206.74337768554688, + "logps/rejected": -181.8390350341797, + "loss": 0.03, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 7.6725358963012695, + "rewards/margins": 12.631759643554688, + "rewards/rejected": -4.959224700927734, + "step": 2860 + }, + { + "epoch": 0.32, + "learning_rate": 7.750372093406095e-06, + "logits/chosen": -0.6658349633216858, + "logits/rejected": 0.31616711616516113, + "logps/chosen": -224.6175079345703, + "logps/rejected": -184.55096435546875, + "loss": 0.0427, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 8.468997955322266, + "rewards/margins": 14.208582878112793, + "rewards/rejected": -5.739585876464844, + "step": 2870 + }, + { + "epoch": 0.32, + "learning_rate": 7.735956244549294e-06, + "logits/chosen": -0.7174617052078247, + "logits/rejected": 0.31869566440582275, + "logps/chosen": -221.75466918945312, + "logps/rejected": -173.40158081054688, + "loss": 0.0961, + "rewards/accuracies": 0.9750000238418579, + "rewards/chosen": 8.690484046936035, + "rewards/margins": 13.39197063446045, + "rewards/rejected": -4.701488494873047, + "step": 2880 + }, + { + "epoch": 0.32, + "learning_rate": 7.721507859126927e-06, + "logits/chosen": -0.6412464380264282, + "logits/rejected": 0.23269173502922058, + "logps/chosen": -191.8711700439453, + "logps/rejected": -181.15994262695312, + "loss": 0.0611, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 7.7382941246032715, + "rewards/margins": 13.182668685913086, + "rewards/rejected": -5.444373607635498, + "step": 2890 + }, + { + "epoch": 0.32, + "learning_rate": 7.707027108962222e-06, + "logits/chosen": -0.4695025384426117, + "logits/rejected": 0.48480135202407837, + "logps/chosen": -224.146240234375, + "logps/rejected": -182.19253540039062, + "loss": 0.0342, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 8.21843147277832, + "rewards/margins": 13.430513381958008, + "rewards/rejected": -5.212081432342529, + "step": 2900 + }, + { + "epoch": 0.32, + "learning_rate": 7.692514166263296e-06, + "logits/chosen": -0.731987476348877, + "logits/rejected": 0.23414798080921173, + "logps/chosen": -183.3697509765625, + "logps/rejected": -180.24000549316406, + "loss": 0.0521, + "rewards/accuracies": 0.9750000238418579, + "rewards/chosen": 7.768662452697754, + "rewards/margins": 12.571802139282227, + "rewards/rejected": -4.80313777923584, + "step": 2910 + }, + { + "epoch": 0.32, + "learning_rate": 7.677969203621113e-06, + "logits/chosen": -0.8885566592216492, + "logits/rejected": 0.126034677028656, + "logps/chosen": -222.8698272705078, + "logps/rejected": -186.18331909179688, + "loss": 0.0438, + "rewards/accuracies": 0.9750000238418579, + "rewards/chosen": 8.468908309936523, + "rewards/margins": 13.904070854187012, + "rewards/rejected": -5.435164451599121, + "step": 2920 + }, + { + "epoch": 0.32, + "learning_rate": 7.663392394007418e-06, + "logits/chosen": -1.0950114727020264, + "logits/rejected": -0.08405710756778717, + "logps/chosen": -244.9022979736328, + "logps/rejected": -170.50209045410156, + "loss": 0.0613, + "rewards/accuracies": 0.9624999761581421, + "rewards/chosen": 8.325568199157715, + "rewards/margins": 13.444650650024414, + "rewards/rejected": -5.119081497192383, + "step": 2930 + }, + { + "epoch": 0.32, + "learning_rate": 7.648783910772687e-06, + "logits/chosen": -0.7430115938186646, + "logits/rejected": 0.2351471483707428, + "logps/chosen": -236.56448364257812, + "logps/rejected": -171.637451171875, + "loss": 0.0975, + "rewards/accuracies": 0.9750000238418579, + "rewards/chosen": 8.123806953430176, + "rewards/margins": 13.381278991699219, + "rewards/rejected": -5.257473945617676, + "step": 2940 + }, + { + "epoch": 0.32, + "learning_rate": 7.634143927644066e-06, + "logits/chosen": -0.6513711214065552, + "logits/rejected": 0.3646014332771301, + "logps/chosen": -179.13702392578125, + "logps/rejected": -180.0673065185547, + "loss": 0.0043, + "rewards/accuracies": 1.0, + "rewards/chosen": 7.1201629638671875, + "rewards/margins": 12.562201499938965, + "rewards/rejected": -5.4420390129089355, + "step": 2950 + }, + { + "epoch": 0.32, + "learning_rate": 7.6194726187233094e-06, + "logits/chosen": -0.4454960823059082, + "logits/rejected": 0.47287583351135254, + "logps/chosen": -201.38282775878906, + "logps/rejected": -178.86441040039062, + "loss": 0.1987, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 7.348745822906494, + "rewards/margins": 12.077168464660645, + "rewards/rejected": -4.728421211242676, + "step": 2960 + }, + { + "epoch": 0.33, + "learning_rate": 7.604770158484697e-06, + "logits/chosen": -0.7678300738334656, + "logits/rejected": 0.533244252204895, + "logps/chosen": -206.36178588867188, + "logps/rejected": -189.08424377441406, + "loss": 0.001, + "rewards/accuracies": 1.0, + "rewards/chosen": 8.136065483093262, + "rewards/margins": 13.558685302734375, + "rewards/rejected": -5.4226179122924805, + "step": 2970 + }, + { + "epoch": 0.33, + "learning_rate": 7.590036721772971e-06, + "logits/chosen": -0.5914806127548218, + "logits/rejected": 0.5361270904541016, + "logps/chosen": -185.54403686523438, + "logps/rejected": -189.84043884277344, + "loss": 0.0353, + "rewards/accuracies": 0.9750000238418579, + "rewards/chosen": 7.511700630187988, + "rewards/margins": 13.264554023742676, + "rewards/rejected": -5.752852916717529, + "step": 2980 + }, + { + "epoch": 0.33, + "learning_rate": 7.57527248380125e-06, + "logits/chosen": -0.5882383584976196, + "logits/rejected": 0.5471413135528564, + "logps/chosen": -196.56063842773438, + "logps/rejected": -181.27120971679688, + "loss": 0.0239, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 7.016335487365723, + "rewards/margins": 12.353779792785645, + "rewards/rejected": -5.3374457359313965, + "step": 2990 + }, + { + "epoch": 0.33, + "learning_rate": 7.560477620148953e-06, + "logits/chosen": -0.6498584151268005, + "logits/rejected": 0.411933571100235, + "logps/chosen": -203.97779846191406, + "logps/rejected": -184.84317016601562, + "loss": 0.0208, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 7.5401105880737305, + "rewards/margins": 13.002525329589844, + "rewards/rejected": -5.462414264678955, + "step": 3000 + }, + { + "epoch": 0.33, + "learning_rate": 7.545652306759698e-06, + "logits/chosen": -0.6753891706466675, + "logits/rejected": 0.2639535665512085, + "logps/chosen": -217.4911346435547, + "logps/rejected": -186.0170135498047, + "loss": 0.0157, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 8.716225624084473, + "rewards/margins": 14.424420356750488, + "rewards/rejected": -5.708194255828857, + "step": 3010 + }, + { + "epoch": 0.33, + "learning_rate": 7.530796719939222e-06, + "logits/chosen": -0.5381367206573486, + "logits/rejected": 0.5319419503211975, + "logps/chosen": -205.999267578125, + "logps/rejected": -176.85818481445312, + "loss": 0.033, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 7.60339879989624, + "rewards/margins": 12.554964065551758, + "rewards/rejected": -4.951564311981201, + "step": 3020 + }, + { + "epoch": 0.33, + "learning_rate": 7.515911036353283e-06, + "logits/chosen": -0.7736371755599976, + "logits/rejected": 0.45761537551879883, + "logps/chosen": -219.46401977539062, + "logps/rejected": -175.74423217773438, + "loss": 0.0162, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 8.509625434875488, + "rewards/margins": 13.809956550598145, + "rewards/rejected": -5.300330638885498, + "step": 3030 + }, + { + "epoch": 0.33, + "learning_rate": 7.500995433025549e-06, + "logits/chosen": -0.7041893601417542, + "logits/rejected": 0.6216388940811157, + "logps/chosen": -185.21334838867188, + "logps/rejected": -176.87351989746094, + "loss": 0.0052, + "rewards/accuracies": 1.0, + "rewards/chosen": 7.409966945648193, + "rewards/margins": 12.747625350952148, + "rewards/rejected": -5.337657451629639, + "step": 3040 + }, + { + "epoch": 0.33, + "learning_rate": 7.486050087335505e-06, + "logits/chosen": -0.6482710838317871, + "logits/rejected": 0.4003063142299652, + "logps/chosen": -236.2769775390625, + "logps/rejected": -177.87246704101562, + "loss": 0.018, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 7.584415435791016, + "rewards/margins": 12.481039047241211, + "rewards/rejected": -4.896624565124512, + "step": 3050 + }, + { + "epoch": 0.34, + "learning_rate": 7.471075177016336e-06, + "logits/chosen": -0.8155606985092163, + "logits/rejected": 0.18223419785499573, + "logps/chosen": -219.41513061523438, + "logps/rejected": -172.69912719726562, + "loss": 0.0615, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 8.329058647155762, + "rewards/margins": 13.742643356323242, + "rewards/rejected": -5.413585662841797, + "step": 3060 + }, + { + "epoch": 0.34, + "learning_rate": 7.456070880152819e-06, + "logits/chosen": -0.7687301635742188, + "logits/rejected": 0.05511612817645073, + "logps/chosen": -205.978515625, + "logps/rejected": -171.88467407226562, + "loss": 0.0958, + "rewards/accuracies": 0.9750000238418579, + "rewards/chosen": 8.545207023620605, + "rewards/margins": 13.174278259277344, + "rewards/rejected": -4.629071235656738, + "step": 3070 + }, + { + "epoch": 0.34, + "learning_rate": 7.441037375179198e-06, + "logits/chosen": -0.7294653654098511, + "logits/rejected": 0.34430450201034546, + "logps/chosen": -198.85122680664062, + "logps/rejected": -170.44239807128906, + "loss": 0.0242, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 7.5057501792907715, + "rewards/margins": 12.262255668640137, + "rewards/rejected": -4.756505966186523, + "step": 3080 + }, + { + "epoch": 0.34, + "learning_rate": 7.425974840877067e-06, + "logits/chosen": -0.9260817766189575, + "logits/rejected": 0.26924628019332886, + "logps/chosen": -217.9792022705078, + "logps/rejected": -185.3968048095703, + "loss": 0.0007, + "rewards/accuracies": 1.0, + "rewards/chosen": 8.855795860290527, + "rewards/margins": 14.105562210083008, + "rewards/rejected": -5.2497639656066895, + "step": 3090 + }, + { + "epoch": 0.34, + "learning_rate": 7.410883456373245e-06, + "logits/chosen": -0.7538719177246094, + "logits/rejected": 0.36721235513687134, + "logps/chosen": -198.8506317138672, + "logps/rejected": -169.50796508789062, + "loss": 0.0133, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 7.756817817687988, + "rewards/margins": 12.394010543823242, + "rewards/rejected": -4.637192726135254, + "step": 3100 + }, + { + "epoch": 0.34, + "learning_rate": 7.395763401137641e-06, + "logits/chosen": -0.9528082609176636, + "logits/rejected": 0.07581549882888794, + "logps/chosen": -230.6846466064453, + "logps/rejected": -195.3170166015625, + "loss": 0.0026, + "rewards/accuracies": 1.0, + "rewards/chosen": 8.874914169311523, + "rewards/margins": 14.411273002624512, + "rewards/rejected": -5.536358833312988, + "step": 3110 + }, + { + "epoch": 0.34, + "learning_rate": 7.380614854981125e-06, + "logits/chosen": -0.8315442800521851, + "logits/rejected": 0.11488503217697144, + "logps/chosen": -206.02499389648438, + "logps/rejected": -175.9861297607422, + "loss": 0.0093, + "rewards/accuracies": 1.0, + "rewards/chosen": 8.4043550491333, + "rewards/margins": 13.608711242675781, + "rewards/rejected": -5.2043561935424805, + "step": 3120 + }, + { + "epoch": 0.34, + "learning_rate": 7.365437998053384e-06, + "logits/chosen": -0.8514736294746399, + "logits/rejected": 0.032159604132175446, + "logps/chosen": -205.56396484375, + "logps/rejected": -172.4398193359375, + "loss": 0.036, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 8.313314437866211, + "rewards/margins": 13.146917343139648, + "rewards/rejected": -4.833603382110596, + "step": 3130 + }, + { + "epoch": 0.34, + "learning_rate": 7.350233010840785e-06, + "logits/chosen": -0.8363455533981323, + "logits/rejected": 0.1365525871515274, + "logps/chosen": -206.1605682373047, + "logps/rejected": -171.68728637695312, + "loss": 0.0308, + "rewards/accuracies": 0.9750000238418579, + "rewards/chosen": 8.239455223083496, + "rewards/margins": 13.51172161102295, + "rewards/rejected": -5.272265434265137, + "step": 3140 + }, + { + "epoch": 0.35, + "learning_rate": 7.3350000741642246e-06, + "logits/chosen": -1.1742407083511353, + "logits/rejected": -0.14099028706550598, + "logps/chosen": -203.42398071289062, + "logps/rejected": -181.8571014404297, + "loss": 0.0168, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 8.417989730834961, + "rewards/margins": 13.784405708312988, + "rewards/rejected": -5.366415977478027, + "step": 3150 + }, + { + "epoch": 0.35, + "learning_rate": 7.319739369176983e-06, + "logits/chosen": -1.0201138257980347, + "logits/rejected": -0.4388630986213684, + "logps/chosen": -191.3087158203125, + "logps/rejected": -178.9386749267578, + "loss": 0.0393, + "rewards/accuracies": 0.9624999761581421, + "rewards/chosen": 8.256948471069336, + "rewards/margins": 13.083456039428711, + "rewards/rejected": -4.826507568359375, + "step": 3160 + }, + { + "epoch": 0.35, + "learning_rate": 7.304451077362562e-06, + "logits/chosen": -1.056004285812378, + "logits/rejected": -0.18533262610435486, + "logps/chosen": -162.4598388671875, + "logps/rejected": -182.8240966796875, + "loss": 0.0098, + "rewards/accuracies": 1.0, + "rewards/chosen": 8.280412673950195, + "rewards/margins": 13.727160453796387, + "rewards/rejected": -5.44674825668335, + "step": 3170 + }, + { + "epoch": 0.35, + "learning_rate": 7.2891353805325375e-06, + "logits/chosen": -0.8735336065292358, + "logits/rejected": -0.16031955182552338, + "logps/chosen": -200.70687866210938, + "logps/rejected": -179.16900634765625, + "loss": 0.043, + "rewards/accuracies": 0.9750000238418579, + "rewards/chosen": 8.549886703491211, + "rewards/margins": 13.584611892700195, + "rewards/rejected": -5.034725189208984, + "step": 3180 + }, + { + "epoch": 0.35, + "learning_rate": 7.273792460824387e-06, + "logits/chosen": -1.1638219356536865, + "logits/rejected": -0.1735357642173767, + "logps/chosen": -191.1776885986328, + "logps/rejected": -173.22299194335938, + "loss": 0.0318, + "rewards/accuracies": 0.9750000238418579, + "rewards/chosen": 7.903469085693359, + "rewards/margins": 12.790934562683105, + "rewards/rejected": -4.887463569641113, + "step": 3190 + }, + { + "epoch": 0.35, + "learning_rate": 7.258422500699332e-06, + "logits/chosen": -1.1490964889526367, + "logits/rejected": -0.12994259595870972, + "logps/chosen": -197.9826202392578, + "logps/rejected": -178.72518920898438, + "loss": 0.0094, + "rewards/accuracies": 1.0, + "rewards/chosen": 8.963010787963867, + "rewards/margins": 13.800790786743164, + "rewards/rejected": -4.837780952453613, + "step": 3200 + }, + { + "epoch": 0.35, + "learning_rate": 7.243025682940161e-06, + "logits/chosen": -1.1130224466323853, + "logits/rejected": -0.243750661611557, + "logps/chosen": -201.7894287109375, + "logps/rejected": -174.1001739501953, + "loss": 0.045, + "rewards/accuracies": 0.9750000238418579, + "rewards/chosen": 8.307756423950195, + "rewards/margins": 13.264605522155762, + "rewards/rejected": -4.956850051879883, + "step": 3210 + }, + { + "epoch": 0.35, + "learning_rate": 7.227602190649062e-06, + "logits/chosen": -1.3907796144485474, + "logits/rejected": -0.4542415738105774, + "logps/chosen": -234.4176788330078, + "logps/rejected": -174.8280487060547, + "loss": 0.0028, + "rewards/accuracies": 1.0, + "rewards/chosen": 9.151853561401367, + "rewards/margins": 13.938626289367676, + "rewards/rejected": -4.786771297454834, + "step": 3220 + }, + { + "epoch": 0.35, + "learning_rate": 7.212152207245437e-06, + "logits/chosen": -1.2498596906661987, + "logits/rejected": -0.5259402990341187, + "logps/chosen": -223.18423461914062, + "logps/rejected": -161.2213897705078, + "loss": 0.0033, + "rewards/accuracies": 1.0, + "rewards/chosen": 8.031622886657715, + "rewards/margins": 12.378576278686523, + "rewards/rejected": -4.34695291519165, + "step": 3230 + }, + { + "epoch": 0.36, + "learning_rate": 7.196675916463735e-06, + "logits/chosen": -1.2956684827804565, + "logits/rejected": -0.44460758566856384, + "logps/chosen": -217.7495574951172, + "logps/rejected": -175.48681640625, + "loss": 0.0428, + "rewards/accuracies": 0.9750000238418579, + "rewards/chosen": 8.582476615905762, + "rewards/margins": 13.193758964538574, + "rewards/rejected": -4.611281871795654, + "step": 3240 + }, + { + "epoch": 0.36, + "learning_rate": 7.1811735023512465e-06, + "logits/chosen": -1.1968815326690674, + "logits/rejected": -0.4859544634819031, + "logps/chosen": -213.48983764648438, + "logps/rejected": -173.18142700195312, + "loss": 0.0071, + "rewards/accuracies": 1.0, + "rewards/chosen": 8.331686973571777, + "rewards/margins": 12.9066162109375, + "rewards/rejected": -4.5749287605285645, + "step": 3250 + }, + { + "epoch": 0.36, + "learning_rate": 7.165645149265936e-06, + "logits/chosen": -0.9111707806587219, + "logits/rejected": -0.08867861330509186, + "logps/chosen": -186.7786407470703, + "logps/rejected": -184.19287109375, + "loss": 0.0178, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 8.086751937866211, + "rewards/margins": 13.239950180053711, + "rewards/rejected": -5.1531982421875, + "step": 3260 + }, + { + "epoch": 0.36, + "learning_rate": 7.150091041874235e-06, + "logits/chosen": -1.2011375427246094, + "logits/rejected": -0.4614068865776062, + "logps/chosen": -219.5398406982422, + "logps/rejected": -170.01565551757812, + "loss": 0.104, + "rewards/accuracies": 0.9624999761581421, + "rewards/chosen": 8.490550994873047, + "rewards/margins": 13.052210807800293, + "rewards/rejected": -4.561659812927246, + "step": 3270 + }, + { + "epoch": 0.36, + "learning_rate": 7.13451136514885e-06, + "logits/chosen": -0.9552543759346008, + "logits/rejected": -0.3252144753932953, + "logps/chosen": -171.53054809570312, + "logps/rejected": -161.9144287109375, + "loss": 0.0112, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 7.514858245849609, + "rewards/margins": 11.921689987182617, + "rewards/rejected": -4.406831741333008, + "step": 3280 + }, + { + "epoch": 0.36, + "learning_rate": 7.118906304366566e-06, + "logits/chosen": -1.2413477897644043, + "logits/rejected": -0.7624537348747253, + "logps/chosen": -181.61965942382812, + "logps/rejected": -174.33909606933594, + "loss": 0.0334, + "rewards/accuracies": 0.9750000238418579, + "rewards/chosen": 8.039202690124512, + "rewards/margins": 12.913854598999023, + "rewards/rejected": -4.874651908874512, + "step": 3290 + }, + { + "epoch": 0.36, + "learning_rate": 7.103276045106038e-06, + "logits/chosen": -1.0170934200286865, + "logits/rejected": -0.688651442527771, + "logps/chosen": -251.27127075195312, + "logps/rejected": -158.6680450439453, + "loss": 0.0197, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 9.681428909301758, + "rewards/margins": 14.192190170288086, + "rewards/rejected": -4.5107622146606445, + "step": 3300 + }, + { + "epoch": 0.36, + "learning_rate": 7.087620773245589e-06, + "logits/chosen": -1.139358401298523, + "logits/rejected": -0.4071386456489563, + "logps/chosen": -217.7616729736328, + "logps/rejected": -179.41697692871094, + "loss": 0.0456, + "rewards/accuracies": 0.9750000238418579, + "rewards/chosen": 8.994406700134277, + "rewards/margins": 14.127166748046875, + "rewards/rejected": -5.1327595710754395, + "step": 3310 + }, + { + "epoch": 0.36, + "learning_rate": 7.071940674960995e-06, + "logits/chosen": -0.7804111242294312, + "logits/rejected": 0.06227637454867363, + "logps/chosen": -170.31918334960938, + "logps/rejected": -180.69093322753906, + "loss": 0.1843, + "rewards/accuracies": 0.9750000238418579, + "rewards/chosen": 8.393743515014648, + "rewards/margins": 13.32214069366455, + "rewards/rejected": -4.928398132324219, + "step": 3320 + }, + { + "epoch": 0.37, + "learning_rate": 7.056235936723276e-06, + "logits/chosen": -1.0827653408050537, + "logits/rejected": -0.2059178650379181, + "logps/chosen": -192.95138549804688, + "logps/rejected": -179.58836364746094, + "loss": 0.0013, + "rewards/accuracies": 1.0, + "rewards/chosen": 8.857095718383789, + "rewards/margins": 14.390403747558594, + "rewards/rejected": -5.533308506011963, + "step": 3330 + }, + { + "epoch": 0.37, + "learning_rate": 7.04050674529647e-06, + "logits/chosen": -0.7707561254501343, + "logits/rejected": 0.20873978734016418, + "logps/chosen": -199.53280639648438, + "logps/rejected": -180.01365661621094, + "loss": 0.011, + "rewards/accuracies": 1.0, + "rewards/chosen": 7.726457118988037, + "rewards/margins": 13.3985013961792, + "rewards/rejected": -5.6720452308654785, + "step": 3340 + }, + { + "epoch": 0.37, + "learning_rate": 7.024753287735424e-06, + "logits/chosen": -0.7886511087417603, + "logits/rejected": 0.23599179089069366, + "logps/chosen": -215.7147979736328, + "logps/rejected": -172.8582305908203, + "loss": 0.0465, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 8.29658317565918, + "rewards/margins": 13.51270580291748, + "rewards/rejected": -5.216121673583984, + "step": 3350 + }, + { + "epoch": 0.37, + "learning_rate": 7.008975751383558e-06, + "logits/chosen": -0.9375608563423157, + "logits/rejected": -0.068226158618927, + "logps/chosen": -241.32113647460938, + "logps/rejected": -169.71624755859375, + "loss": 0.0164, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 8.509544372558594, + "rewards/margins": 13.582138061523438, + "rewards/rejected": -5.07259464263916, + "step": 3360 + }, + { + "epoch": 0.37, + "learning_rate": 6.993174323870643e-06, + "logits/chosen": -0.9534242749214172, + "logits/rejected": 0.08418476581573486, + "logps/chosen": -236.45877075195312, + "logps/rejected": -176.23739624023438, + "loss": 0.0387, + "rewards/accuracies": 0.9750000238418579, + "rewards/chosen": 8.700987815856934, + "rewards/margins": 13.830378532409668, + "rewards/rejected": -5.129391193389893, + "step": 3370 + }, + { + "epoch": 0.37, + "learning_rate": 6.977349193110573e-06, + "logits/chosen": -0.8227748870849609, + "logits/rejected": 0.2689315378665924, + "logps/chosen": -156.1683807373047, + "logps/rejected": -170.1246795654297, + "loss": 0.0056, + "rewards/accuracies": 1.0, + "rewards/chosen": 7.770416259765625, + "rewards/margins": 12.481344223022461, + "rewards/rejected": -4.710928440093994, + "step": 3380 + }, + { + "epoch": 0.37, + "learning_rate": 6.961500547299118e-06, + "logits/chosen": -0.631439208984375, + "logits/rejected": 0.4038637578487396, + "logps/chosen": -208.94827270507812, + "logps/rejected": -173.27694702148438, + "loss": 0.0083, + "rewards/accuracies": 1.0, + "rewards/chosen": 8.294332504272461, + "rewards/margins": 13.42784595489502, + "rewards/rejected": -5.133512020111084, + "step": 3390 + }, + { + "epoch": 0.37, + "learning_rate": 6.945628574911704e-06, + "logits/chosen": -0.9172015190124512, + "logits/rejected": 0.11328015476465225, + "logps/chosen": -219.79269409179688, + "logps/rejected": -178.8690643310547, + "loss": 0.0166, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 8.629083633422852, + "rewards/margins": 13.775530815124512, + "rewards/rejected": -5.146448612213135, + "step": 3400 + }, + { + "epoch": 0.37, + "learning_rate": 6.92973346470115e-06, + "logits/chosen": -1.0190643072128296, + "logits/rejected": -0.174514502286911, + "logps/chosen": -197.52166748046875, + "logps/rejected": -165.14697265625, + "loss": 0.0503, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 9.552925109863281, + "rewards/margins": 14.470596313476562, + "rewards/rejected": -4.917671203613281, + "step": 3410 + }, + { + "epoch": 0.38, + "learning_rate": 6.913815405695445e-06, + "logits/chosen": -0.8897427320480347, + "logits/rejected": 0.3352202773094177, + "logps/chosen": -191.97222900390625, + "logps/rejected": -182.82968139648438, + "loss": 0.0297, + "rewards/accuracies": 0.9750000238418579, + "rewards/chosen": 8.750223159790039, + "rewards/margins": 13.577245712280273, + "rewards/rejected": -4.827023506164551, + "step": 3420 + }, + { + "epoch": 0.38, + "learning_rate": 6.897874587195486e-06, + "logits/chosen": -0.8339163064956665, + "logits/rejected": 0.45157933235168457, + "logps/chosen": -193.5912322998047, + "logps/rejected": -181.64743041992188, + "loss": 0.0201, + "rewards/accuracies": 1.0, + "rewards/chosen": 8.07734489440918, + "rewards/margins": 13.753377914428711, + "rewards/rejected": -5.676033973693848, + "step": 3430 + }, + { + "epoch": 0.38, + "learning_rate": 6.881911198772827e-06, + "logits/chosen": -0.8414053916931152, + "logits/rejected": 0.21240687370300293, + "logps/chosen": -203.11184692382812, + "logps/rejected": -176.51124572753906, + "loss": 0.0022, + "rewards/accuracies": 1.0, + "rewards/chosen": 7.822409152984619, + "rewards/margins": 12.566364288330078, + "rewards/rejected": -4.743955135345459, + "step": 3440 + }, + { + "epoch": 0.38, + "learning_rate": 6.865925430267435e-06, + "logits/chosen": -0.7175145149230957, + "logits/rejected": 0.18644139170646667, + "logps/chosen": -222.4505615234375, + "logps/rejected": -187.85580444335938, + "loss": 0.0004, + "rewards/accuracies": 1.0, + "rewards/chosen": 9.484896659851074, + "rewards/margins": 14.990574836730957, + "rewards/rejected": -5.505678176879883, + "step": 3450 + }, + { + "epoch": 0.38, + "learning_rate": 6.849917471785419e-06, + "logits/chosen": -0.8808102607727051, + "logits/rejected": 0.006777075119316578, + "logps/chosen": -233.87301635742188, + "logps/rejected": -178.4818572998047, + "loss": 0.0062, + "rewards/accuracies": 1.0, + "rewards/chosen": 10.213820457458496, + "rewards/margins": 15.250112533569336, + "rewards/rejected": -5.03629207611084, + "step": 3460 + }, + { + "epoch": 0.38, + "learning_rate": 6.833887513696783e-06, + "logits/chosen": -0.8258196115493774, + "logits/rejected": 0.04845712333917618, + "logps/chosen": -222.073486328125, + "logps/rejected": -169.7540740966797, + "loss": 0.0108, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 8.345000267028809, + "rewards/margins": 13.234458923339844, + "rewards/rejected": -4.889459133148193, + "step": 3470 + }, + { + "epoch": 0.38, + "learning_rate": 6.817835746633148e-06, + "logits/chosen": -0.7366988658905029, + "logits/rejected": 0.28878456354141235, + "logps/chosen": -182.4452362060547, + "logps/rejected": -172.08187866210938, + "loss": 0.0069, + "rewards/accuracies": 1.0, + "rewards/chosen": 8.180105209350586, + "rewards/margins": 13.290735244750977, + "rewards/rejected": -5.110630512237549, + "step": 3480 + }, + { + "epoch": 0.38, + "learning_rate": 6.801762361485498e-06, + "logits/chosen": -0.7602673768997192, + "logits/rejected": 0.3302949368953705, + "logps/chosen": -198.2094268798828, + "logps/rejected": -181.18487548828125, + "loss": 0.0854, + "rewards/accuracies": 0.9624999761581421, + "rewards/chosen": 8.039267539978027, + "rewards/margins": 12.643938064575195, + "rewards/rejected": -4.604669094085693, + "step": 3490 + }, + { + "epoch": 0.38, + "learning_rate": 6.785667549401898e-06, + "logits/chosen": -0.5550008416175842, + "logits/rejected": 0.26477155089378357, + "logps/chosen": -219.5014190673828, + "logps/rejected": -176.40765380859375, + "loss": 0.062, + "rewards/accuracies": 0.9750000238418579, + "rewards/chosen": 8.702611923217773, + "rewards/margins": 13.270441055297852, + "rewards/rejected": -4.5678277015686035, + "step": 3500 + }, + { + "epoch": 0.39, + "learning_rate": 6.76955150178523e-06, + "logits/chosen": -0.62385493516922, + "logits/rejected": 0.7666346430778503, + "logps/chosen": -200.52011108398438, + "logps/rejected": -186.7143096923828, + "loss": 0.0271, + "rewards/accuracies": 0.9624999761581421, + "rewards/chosen": 7.345146179199219, + "rewards/margins": 13.502595901489258, + "rewards/rejected": -6.1574482917785645, + "step": 3510 + }, + { + "epoch": 0.39, + "learning_rate": 6.7534144102909134e-06, + "logits/chosen": -0.3296525478363037, + "logits/rejected": 0.9785606265068054, + "logps/chosen": -219.3638458251953, + "logps/rejected": -187.53102111816406, + "loss": 0.0361, + "rewards/accuracies": 0.9750000238418579, + "rewards/chosen": 7.748207092285156, + "rewards/margins": 13.580224990844727, + "rewards/rejected": -5.8320183753967285, + "step": 3520 + }, + { + "epoch": 0.39, + "learning_rate": 6.737256466824626e-06, + "logits/chosen": -0.8198176622390747, + "logits/rejected": 0.6690760850906372, + "logps/chosen": -206.1396026611328, + "logps/rejected": -189.7904052734375, + "loss": 0.0015, + "rewards/accuracies": 1.0, + "rewards/chosen": 8.307302474975586, + "rewards/margins": 13.590047836303711, + "rewards/rejected": -5.282746315002441, + "step": 3530 + }, + { + "epoch": 0.39, + "learning_rate": 6.72107786354002e-06, + "logits/chosen": -0.6701766848564148, + "logits/rejected": 0.6413156986236572, + "logps/chosen": -206.9150848388672, + "logps/rejected": -170.31906127929688, + "loss": 0.175, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": 7.471018314361572, + "rewards/margins": 12.1195650100708, + "rewards/rejected": -4.648547172546387, + "step": 3540 + }, + { + "epoch": 0.39, + "learning_rate": 6.7048787928364375e-06, + "logits/chosen": -0.5567693114280701, + "logits/rejected": 0.4171839654445648, + "logps/chosen": -198.5066680908203, + "logps/rejected": -179.05430603027344, + "loss": 0.0032, + "rewards/accuracies": 1.0, + "rewards/chosen": 8.734719276428223, + "rewards/margins": 14.26312255859375, + "rewards/rejected": -5.528403282165527, + "step": 3550 + }, + { + "epoch": 0.39, + "learning_rate": 6.688659447356629e-06, + "logits/chosen": -0.6520235538482666, + "logits/rejected": 0.18082484602928162, + "logps/chosen": -166.6741485595703, + "logps/rejected": -185.9934539794922, + "loss": 0.0061, + "rewards/accuracies": 1.0, + "rewards/chosen": 7.737581729888916, + "rewards/margins": 13.130480766296387, + "rewards/rejected": -5.392898082733154, + "step": 3560 + }, + { + "epoch": 0.39, + "learning_rate": 6.672420019984451e-06, + "logits/chosen": -0.8938458561897278, + "logits/rejected": -0.0020337298046797514, + "logps/chosen": -210.1428680419922, + "logps/rejected": -184.53843688964844, + "loss": 0.0081, + "rewards/accuracies": 1.0, + "rewards/chosen": 8.547704696655273, + "rewards/margins": 13.74889087677002, + "rewards/rejected": -5.201184272766113, + "step": 3570 + }, + { + "epoch": 0.39, + "learning_rate": 6.656160703842581e-06, + "logits/chosen": -1.0050981044769287, + "logits/rejected": 0.10824527591466904, + "logps/chosen": -213.1746826171875, + "logps/rejected": -182.98135375976562, + "loss": 0.0229, + "rewards/accuracies": 0.9750000238418579, + "rewards/chosen": 9.332930564880371, + "rewards/margins": 14.27111530303955, + "rewards/rejected": -4.938185691833496, + "step": 3580 + }, + { + "epoch": 0.39, + "learning_rate": 6.639881692290218e-06, + "logits/chosen": -1.049739122390747, + "logits/rejected": -0.01060022134333849, + "logps/chosen": -218.0283203125, + "logps/rejected": -176.4106903076172, + "loss": 0.01, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 8.26351547241211, + "rewards/margins": 13.89555835723877, + "rewards/rejected": -5.63204288482666, + "step": 3590 + }, + { + "epoch": 0.4, + "learning_rate": 6.623583178920778e-06, + "logits/chosen": -0.806925892829895, + "logits/rejected": 0.427852064371109, + "logps/chosen": -185.04141235351562, + "logps/rejected": -178.02113342285156, + "loss": 0.0101, + "rewards/accuracies": 1.0, + "rewards/chosen": 8.551798820495605, + "rewards/margins": 13.595972061157227, + "rewards/rejected": -5.044173717498779, + "step": 3600 + }, + { + "epoch": 0.4, + "learning_rate": 6.607265357559605e-06, + "logits/chosen": -0.9502663612365723, + "logits/rejected": 0.04503049701452255, + "logps/chosen": -197.19424438476562, + "logps/rejected": -179.08316040039062, + "loss": 0.0022, + "rewards/accuracies": 1.0, + "rewards/chosen": 8.920248031616211, + "rewards/margins": 13.982839584350586, + "rewards/rejected": -5.062591075897217, + "step": 3610 + }, + { + "epoch": 0.4, + "learning_rate": 6.590928422261653e-06, + "logits/chosen": -0.9806275367736816, + "logits/rejected": 0.28563347458839417, + "logps/chosen": -186.7781982421875, + "logps/rejected": -180.20065307617188, + "loss": 0.0956, + "rewards/accuracies": 0.9375, + "rewards/chosen": 8.173318862915039, + "rewards/margins": 13.065643310546875, + "rewards/rejected": -4.8923234939575195, + "step": 3620 + }, + { + "epoch": 0.4, + "learning_rate": 6.574572567309183e-06, + "logits/chosen": -0.7013615965843201, + "logits/rejected": 0.14542338252067566, + "logps/chosen": -179.23634338378906, + "logps/rejected": -175.63487243652344, + "loss": 0.1534, + "rewards/accuracies": 0.9750000238418579, + "rewards/chosen": 8.403069496154785, + "rewards/margins": 13.367837905883789, + "rewards/rejected": -4.964768409729004, + "step": 3630 + }, + { + "epoch": 0.4, + "learning_rate": 6.558197987209452e-06, + "logits/chosen": -0.7640706300735474, + "logits/rejected": 0.01694454811513424, + "logps/chosen": -224.9866180419922, + "logps/rejected": -170.11865234375, + "loss": 0.2523, + "rewards/accuracies": 0.9375, + "rewards/chosen": 8.545625686645508, + "rewards/margins": 13.341995239257812, + "rewards/rejected": -4.796370029449463, + "step": 3640 + }, + { + "epoch": 0.4, + "learning_rate": 6.541804876692402e-06, + "logits/chosen": -0.6622705459594727, + "logits/rejected": 0.2925678789615631, + "logps/chosen": -196.38992309570312, + "logps/rejected": -184.2986297607422, + "loss": 0.0221, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 9.203376770019531, + "rewards/margins": 14.579988479614258, + "rewards/rejected": -5.376611232757568, + "step": 3650 + }, + { + "epoch": 0.4, + "learning_rate": 6.525393430708344e-06, + "logits/chosen": -1.0506999492645264, + "logits/rejected": 0.32442235946655273, + "logps/chosen": -253.0437774658203, + "logps/rejected": -183.85394287109375, + "loss": 0.0204, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 9.904923439025879, + "rewards/margins": 15.480748176574707, + "rewards/rejected": -5.575823783874512, + "step": 3660 + }, + { + "epoch": 0.4, + "learning_rate": 6.508963844425632e-06, + "logits/chosen": -0.6444002985954285, + "logits/rejected": 0.6485086679458618, + "logps/chosen": -191.58934020996094, + "logps/rejected": -192.52139282226562, + "loss": 0.0916, + "rewards/accuracies": 0.9750000238418579, + "rewards/chosen": 8.343785285949707, + "rewards/margins": 14.21680736541748, + "rewards/rejected": -5.873022079467773, + "step": 3670 + }, + { + "epoch": 0.4, + "learning_rate": 6.492516313228356e-06, + "logits/chosen": -0.7919450998306274, + "logits/rejected": 0.4014405608177185, + "logps/chosen": -229.5850067138672, + "logps/rejected": -183.91876220703125, + "loss": 0.0052, + "rewards/accuracies": 1.0, + "rewards/chosen": 8.037152290344238, + "rewards/margins": 13.521562576293945, + "rewards/rejected": -5.484410285949707, + "step": 3680 + }, + { + "epoch": 0.41, + "learning_rate": 6.476051032714005e-06, + "logits/chosen": -0.4600529074668884, + "logits/rejected": 0.9336981773376465, + "logps/chosen": -219.58517456054688, + "logps/rejected": -185.51295471191406, + "loss": 0.0471, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 7.673447608947754, + "rewards/margins": 13.267809867858887, + "rewards/rejected": -5.594361305236816, + "step": 3690 + }, + { + "epoch": 0.41, + "learning_rate": 6.459568198691151e-06, + "logits/chosen": -0.4215783476829529, + "logits/rejected": 0.8345744013786316, + "logps/chosen": -186.2758026123047, + "logps/rejected": -181.0073699951172, + "loss": 0.0364, + "rewards/accuracies": 0.9750000238418579, + "rewards/chosen": 7.7432403564453125, + "rewards/margins": 13.588427543640137, + "rewards/rejected": -5.845186710357666, + "step": 3700 + }, + { + "epoch": 0.41, + "learning_rate": 6.443068007177111e-06, + "logits/chosen": -0.6773524284362793, + "logits/rejected": 0.46389707922935486, + "logps/chosen": -206.9965057373047, + "logps/rejected": -185.75704956054688, + "loss": 0.0259, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 9.33317756652832, + "rewards/margins": 15.035089492797852, + "rewards/rejected": -5.701910972595215, + "step": 3710 + }, + { + "epoch": 0.41, + "learning_rate": 6.426550654395625e-06, + "logits/chosen": -1.1971046924591064, + "logits/rejected": -0.25961732864379883, + "logps/chosen": -225.5023651123047, + "logps/rejected": -171.3182830810547, + "loss": 0.0076, + "rewards/accuracies": 1.0, + "rewards/chosen": 8.178506851196289, + "rewards/margins": 13.565916061401367, + "rewards/rejected": -5.387409210205078, + "step": 3720 + }, + { + "epoch": 0.41, + "learning_rate": 6.410016336774518e-06, + "logits/chosen": -1.3878405094146729, + "logits/rejected": -0.26950788497924805, + "logps/chosen": -224.96218872070312, + "logps/rejected": -178.20921325683594, + "loss": 0.0103, + "rewards/accuracies": 1.0, + "rewards/chosen": 8.885696411132812, + "rewards/margins": 14.716214179992676, + "rewards/rejected": -5.830516338348389, + "step": 3730 + }, + { + "epoch": 0.41, + "learning_rate": 6.393465250943361e-06, + "logits/chosen": -0.8421357274055481, + "logits/rejected": 0.17760539054870605, + "logps/chosen": -175.39810180664062, + "logps/rejected": -186.3933563232422, + "loss": 0.0144, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 7.695733547210693, + "rewards/margins": 13.4041748046875, + "rewards/rejected": -5.708441734313965, + "step": 3740 + }, + { + "epoch": 0.41, + "learning_rate": 6.376897593731142e-06, + "logits/chosen": -0.7231656312942505, + "logits/rejected": 0.6062003970146179, + "logps/chosen": -204.56570434570312, + "logps/rejected": -182.39492797851562, + "loss": 0.0585, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 8.838582038879395, + "rewards/margins": 14.02421760559082, + "rewards/rejected": -5.185635566711426, + "step": 3750 + }, + { + "epoch": 0.41, + "learning_rate": 6.360313562163911e-06, + "logits/chosen": -0.5835901498794556, + "logits/rejected": 0.6440936923027039, + "logps/chosen": -228.4535675048828, + "logps/rejected": -177.8348388671875, + "loss": 0.0631, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 8.324292182922363, + "rewards/margins": 13.978315353393555, + "rewards/rejected": -5.654022693634033, + "step": 3760 + }, + { + "epoch": 0.41, + "learning_rate": 6.343713353462455e-06, + "logits/chosen": -0.7640455365180969, + "logits/rejected": 0.56242436170578, + "logps/chosen": -239.3903350830078, + "logps/rejected": -190.3300018310547, + "loss": 0.0025, + "rewards/accuracies": 1.0, + "rewards/chosen": 8.516837120056152, + "rewards/margins": 14.49145221710205, + "rewards/rejected": -5.974615097045898, + "step": 3770 + }, + { + "epoch": 0.41, + "learning_rate": 6.327097165039935e-06, + "logits/chosen": -0.9984546899795532, + "logits/rejected": 0.4714936316013336, + "logps/chosen": -224.21435546875, + "logps/rejected": -175.8540802001953, + "loss": 0.1021, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": 7.434506416320801, + "rewards/margins": 13.248006820678711, + "rewards/rejected": -5.813500881195068, + "step": 3780 + }, + { + "epoch": 0.42, + "learning_rate": 6.310465194499551e-06, + "logits/chosen": -0.6461219787597656, + "logits/rejected": 0.43689781427383423, + "logps/chosen": -193.32254028320312, + "logps/rejected": -181.96963500976562, + "loss": 0.2086, + "rewards/accuracies": 0.9750000238418579, + "rewards/chosen": 8.078817367553711, + "rewards/margins": 14.114912033081055, + "rewards/rejected": -6.03609561920166, + "step": 3790 + }, + { + "epoch": 0.42, + "learning_rate": 6.293817639632185e-06, + "logits/chosen": -0.7805728912353516, + "logits/rejected": 0.47109612822532654, + "logps/chosen": -205.6940460205078, + "logps/rejected": -175.33810424804688, + "loss": 0.0277, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 8.098638534545898, + "rewards/margins": 13.161134719848633, + "rewards/rejected": -5.062497138977051, + "step": 3800 + }, + { + "epoch": 0.42, + "learning_rate": 6.27715469841405e-06, + "logits/chosen": -0.7545350790023804, + "logits/rejected": 0.278158962726593, + "logps/chosen": -249.19912719726562, + "logps/rejected": -178.79348754882812, + "loss": 0.0342, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 9.315717697143555, + "rewards/margins": 14.928706169128418, + "rewards/rejected": -5.6129865646362305, + "step": 3810 + }, + { + "epoch": 0.42, + "learning_rate": 6.26047656900434e-06, + "logits/chosen": -1.2794287204742432, + "logits/rejected": -0.08282064646482468, + "logps/chosen": -254.2216339111328, + "logps/rejected": -178.14540100097656, + "loss": 0.0036, + "rewards/accuracies": 1.0, + "rewards/chosen": 9.349328994750977, + "rewards/margins": 14.528002738952637, + "rewards/rejected": -5.178675174713135, + "step": 3820 + }, + { + "epoch": 0.42, + "learning_rate": 6.243783449742867e-06, + "logits/chosen": -0.8679929971694946, + "logits/rejected": 0.18156543374061584, + "logps/chosen": -198.11172485351562, + "logps/rejected": -179.1707763671875, + "loss": 0.0477, + "rewards/accuracies": 0.9750000238418579, + "rewards/chosen": 8.136297225952148, + "rewards/margins": 13.436251640319824, + "rewards/rejected": -5.299954414367676, + "step": 3830 + }, + { + "epoch": 0.42, + "learning_rate": 6.2270755391477085e-06, + "logits/chosen": -1.0633732080459595, + "logits/rejected": -0.045348815619945526, + "logps/chosen": -202.6431121826172, + "logps/rejected": -175.68087768554688, + "loss": 0.0327, + "rewards/accuracies": 0.9750000238418579, + "rewards/chosen": 8.798636436462402, + "rewards/margins": 13.709843635559082, + "rewards/rejected": -4.91120719909668, + "step": 3840 + }, + { + "epoch": 0.42, + "learning_rate": 6.210353035912843e-06, + "logits/chosen": -0.9639671444892883, + "logits/rejected": 0.27169322967529297, + "logps/chosen": -215.02865600585938, + "logps/rejected": -181.08828735351562, + "loss": 0.1967, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": 8.17973518371582, + "rewards/margins": 13.478195190429688, + "rewards/rejected": -5.298459053039551, + "step": 3850 + }, + { + "epoch": 0.42, + "learning_rate": 6.1936161389057825e-06, + "logits/chosen": -0.9356621503829956, + "logits/rejected": 0.37208524346351624, + "logps/chosen": -156.39849853515625, + "logps/rejected": -181.62852478027344, + "loss": 0.0884, + "rewards/accuracies": 0.9624999761581421, + "rewards/chosen": 8.369498252868652, + "rewards/margins": 13.029362678527832, + "rewards/rejected": -4.659864902496338, + "step": 3860 + }, + { + "epoch": 0.42, + "learning_rate": 6.1768650471652214e-06, + "logits/chosen": -0.9376729726791382, + "logits/rejected": 0.08415332436561584, + "logps/chosen": -211.53738403320312, + "logps/rejected": -173.09225463867188, + "loss": 0.0221, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 9.1871337890625, + "rewards/margins": 14.147705078125, + "rewards/rejected": -4.960569858551025, + "step": 3870 + }, + { + "epoch": 0.43, + "learning_rate": 6.160099959898653e-06, + "logits/chosen": -0.8393093347549438, + "logits/rejected": 0.16262702643871307, + "logps/chosen": -179.29786682128906, + "logps/rejected": -177.23260498046875, + "loss": 0.0124, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 8.05333137512207, + "rewards/margins": 12.956738471984863, + "rewards/rejected": -4.903406620025635, + "step": 3880 + }, + { + "epoch": 0.43, + "learning_rate": 6.143321076480015e-06, + "logits/chosen": -0.9487682580947876, + "logits/rejected": 0.06015775725245476, + "logps/chosen": -200.8483123779297, + "logps/rejected": -176.43978881835938, + "loss": 0.2718, + "rewards/accuracies": 0.9375, + "rewards/chosen": 8.434505462646484, + "rewards/margins": 13.368351936340332, + "rewards/rejected": -4.933847904205322, + "step": 3890 + }, + { + "epoch": 0.43, + "learning_rate": 6.126528596447307e-06, + "logits/chosen": -0.7597723007202148, + "logits/rejected": 0.09596171230077744, + "logps/chosen": -210.8147430419922, + "logps/rejected": -178.126708984375, + "loss": 0.1329, + "rewards/accuracies": 0.9750000238418579, + "rewards/chosen": 8.51466178894043, + "rewards/margins": 13.376920700073242, + "rewards/rejected": -4.862259387969971, + "step": 3900 + }, + { + "epoch": 0.43, + "learning_rate": 6.109722719500223e-06, + "logits/chosen": -0.7797644734382629, + "logits/rejected": 0.13296420872211456, + "logps/chosen": -212.66232299804688, + "logps/rejected": -180.22055053710938, + "loss": 0.0224, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 8.710895538330078, + "rewards/margins": 13.915922164916992, + "rewards/rejected": -5.205025672912598, + "step": 3910 + }, + { + "epoch": 0.43, + "learning_rate": 6.0929036454977785e-06, + "logits/chosen": -0.9360426068305969, + "logits/rejected": 0.3349997401237488, + "logps/chosen": -219.0043182373047, + "logps/rejected": -178.1255645751953, + "loss": 0.1055, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 8.514394760131836, + "rewards/margins": 14.165583610534668, + "rewards/rejected": -5.651188850402832, + "step": 3920 + }, + { + "epoch": 0.43, + "learning_rate": 6.07607157445593e-06, + "logits/chosen": -0.6394746899604797, + "logits/rejected": 0.6828135848045349, + "logps/chosen": -209.89675903320312, + "logps/rejected": -195.5568084716797, + "loss": 0.0534, + "rewards/accuracies": 0.9750000238418579, + "rewards/chosen": 8.524816513061523, + "rewards/margins": 14.102399826049805, + "rewards/rejected": -5.577584266662598, + "step": 3930 + }, + { + "epoch": 0.43, + "learning_rate": 6.059226706545195e-06, + "logits/chosen": -0.6330808997154236, + "logits/rejected": 0.3643871247768402, + "logps/chosen": -201.40184020996094, + "logps/rejected": -169.64480590820312, + "loss": 0.1252, + "rewards/accuracies": 0.9624999761581421, + "rewards/chosen": 7.679750919342041, + "rewards/margins": 12.795146942138672, + "rewards/rejected": -5.115396022796631, + "step": 3940 + }, + { + "epoch": 0.43, + "learning_rate": 6.042369242088278e-06, + "logits/chosen": -0.7723215818405151, + "logits/rejected": 0.5415152311325073, + "logps/chosen": -186.429443359375, + "logps/rejected": -192.01861572265625, + "loss": 0.0285, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 8.944454193115234, + "rewards/margins": 14.168388366699219, + "rewards/rejected": -5.223933696746826, + "step": 3950 + }, + { + "epoch": 0.43, + "learning_rate": 6.025499381557681e-06, + "logits/chosen": -0.9444753527641296, + "logits/rejected": 0.1270388662815094, + "logps/chosen": -251.1237335205078, + "logps/rejected": -177.24929809570312, + "loss": 0.0014, + "rewards/accuracies": 1.0, + "rewards/chosen": 9.247361183166504, + "rewards/margins": 15.089139938354492, + "rewards/rejected": -5.841780662536621, + "step": 3960 + }, + { + "epoch": 0.44, + "learning_rate": 6.008617325573326e-06, + "logits/chosen": -0.8049014210700989, + "logits/rejected": 0.27902358770370483, + "logps/chosen": -227.78076171875, + "logps/rejected": -177.12806701660156, + "loss": 0.0055, + "rewards/accuracies": 1.0, + "rewards/chosen": 8.506020545959473, + "rewards/margins": 13.976173400878906, + "rewards/rejected": -5.47015380859375, + "step": 3970 + }, + { + "epoch": 0.44, + "learning_rate": 5.991723274900163e-06, + "logits/chosen": -0.6406025290489197, + "logits/rejected": 0.7000995874404907, + "logps/chosen": -213.0384521484375, + "logps/rejected": -188.4704132080078, + "loss": 0.001, + "rewards/accuracies": 1.0, + "rewards/chosen": 8.605549812316895, + "rewards/margins": 14.436958312988281, + "rewards/rejected": -5.831408977508545, + "step": 3980 + }, + { + "epoch": 0.44, + "learning_rate": 5.974817430445785e-06, + "logits/chosen": -0.7313351631164551, + "logits/rejected": 0.415059894323349, + "logps/chosen": -238.2436065673828, + "logps/rejected": -188.76449584960938, + "loss": 0.002, + "rewards/accuracies": 1.0, + "rewards/chosen": 9.01826286315918, + "rewards/margins": 15.245407104492188, + "rewards/rejected": -6.227145671844482, + "step": 3990 + }, + { + "epoch": 0.44, + "learning_rate": 5.957899993258043e-06, + "logits/chosen": -0.5861080884933472, + "logits/rejected": 0.5385454893112183, + "logps/chosen": -207.50637817382812, + "logps/rejected": -178.4709930419922, + "loss": 0.021, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 7.816317558288574, + "rewards/margins": 12.857678413391113, + "rewards/rejected": -5.04135799407959, + "step": 4000 + }, + { + "epoch": 0.44, + "learning_rate": 5.940971164522648e-06, + "logits/chosen": -0.8059609532356262, + "logits/rejected": 0.5670924186706543, + "logps/chosen": -201.50241088867188, + "logps/rejected": -175.1974639892578, + "loss": 0.0332, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 7.556725978851318, + "rewards/margins": 12.698358535766602, + "rewards/rejected": -5.141633987426758, + "step": 4010 + }, + { + "epoch": 0.44, + "learning_rate": 5.924031145560782e-06, + "logits/chosen": -0.7465982437133789, + "logits/rejected": 0.4708995223045349, + "logps/chosen": -208.65615844726562, + "logps/rejected": -174.00592041015625, + "loss": 0.0091, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 8.70871353149414, + "rewards/margins": 13.875978469848633, + "rewards/rejected": -5.167263984680176, + "step": 4020 + }, + { + "epoch": 0.44, + "learning_rate": 5.9070801378267025e-06, + "logits/chosen": -0.7655161023139954, + "logits/rejected": 0.3438930809497833, + "logps/chosen": -194.76939392089844, + "logps/rejected": -186.3983917236328, + "loss": 0.03, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 8.572443008422852, + "rewards/margins": 14.008273124694824, + "rewards/rejected": -5.435831069946289, + "step": 4030 + }, + { + "epoch": 0.44, + "learning_rate": 5.890118342905351e-06, + "logits/chosen": -0.6657806634902954, + "logits/rejected": 0.3586101233959198, + "logps/chosen": -148.8475341796875, + "logps/rejected": -170.58200073242188, + "loss": 0.0445, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 7.5649590492248535, + "rewards/margins": 12.670374870300293, + "rewards/rejected": -5.105416297912598, + "step": 4040 + }, + { + "epoch": 0.44, + "learning_rate": 5.873145962509952e-06, + "logits/chosen": -0.9431330561637878, + "logits/rejected": 0.0603022575378418, + "logps/chosen": -212.57278442382812, + "logps/rejected": -176.8751220703125, + "loss": 0.0576, + "rewards/accuracies": 0.9750000238418579, + "rewards/chosen": 8.594672203063965, + "rewards/margins": 14.1024169921875, + "rewards/rejected": -5.507744312286377, + "step": 4050 + }, + { + "epoch": 0.45, + "learning_rate": 5.856163198479608e-06, + "logits/chosen": -0.9707878828048706, + "logits/rejected": -0.02843034267425537, + "logps/chosen": -234.7915802001953, + "logps/rejected": -186.56942749023438, + "loss": 0.0518, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 8.970584869384766, + "rewards/margins": 15.075932502746582, + "rewards/rejected": -6.105348587036133, + "step": 4060 + }, + { + "epoch": 0.45, + "learning_rate": 5.839170252776919e-06, + "logits/chosen": -1.0416876077651978, + "logits/rejected": 0.07860129326581955, + "logps/chosen": -219.79745483398438, + "logps/rejected": -194.494384765625, + "loss": 0.0016, + "rewards/accuracies": 1.0, + "rewards/chosen": 9.382036209106445, + "rewards/margins": 15.135244369506836, + "rewards/rejected": -5.753209114074707, + "step": 4070 + }, + { + "epoch": 0.45, + "learning_rate": 5.822167327485553e-06, + "logits/chosen": -1.1095116138458252, + "logits/rejected": -0.07318861782550812, + "logps/chosen": -225.02468872070312, + "logps/rejected": -182.23403930664062, + "loss": 0.0012, + "rewards/accuracies": 1.0, + "rewards/chosen": 8.628786087036133, + "rewards/margins": 14.620928764343262, + "rewards/rejected": -5.9921417236328125, + "step": 4080 + }, + { + "epoch": 0.45, + "learning_rate": 5.8051546248078675e-06, + "logits/chosen": -0.7371264696121216, + "logits/rejected": 0.24817201495170593, + "logps/chosen": -215.85293579101562, + "logps/rejected": -180.95050048828125, + "loss": 0.0247, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 9.270589828491211, + "rewards/margins": 14.834858894348145, + "rewards/rejected": -5.56427001953125, + "step": 4090 + }, + { + "epoch": 0.45, + "learning_rate": 5.788132347062492e-06, + "logits/chosen": -0.7942949533462524, + "logits/rejected": 0.11628608405590057, + "logps/chosen": -213.6909637451172, + "logps/rejected": -175.28065490722656, + "loss": 0.1704, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": 7.790574073791504, + "rewards/margins": 13.311141967773438, + "rewards/rejected": -5.520567893981934, + "step": 4100 + }, + { + "epoch": 0.45, + "learning_rate": 5.77110069668192e-06, + "logits/chosen": -0.840557873249054, + "logits/rejected": -0.02615346387028694, + "logps/chosen": -201.24459838867188, + "logps/rejected": -182.7463836669922, + "loss": 0.0706, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 8.210264205932617, + "rewards/margins": 13.914891242980957, + "rewards/rejected": -5.704627990722656, + "step": 4110 + }, + { + "epoch": 0.45, + "learning_rate": 5.754059876210116e-06, + "logits/chosen": -0.6189905405044556, + "logits/rejected": 0.3134820759296417, + "logps/chosen": -188.04808044433594, + "logps/rejected": -183.6875457763672, + "loss": 0.0125, + "rewards/accuracies": 1.0, + "rewards/chosen": 7.7136101722717285, + "rewards/margins": 13.423299789428711, + "rewards/rejected": -5.709689617156982, + "step": 4120 + }, + { + "epoch": 0.45, + "learning_rate": 5.7370100883000865e-06, + "logits/chosen": -1.0680980682373047, + "logits/rejected": -0.1996569186449051, + "logps/chosen": -214.7783203125, + "logps/rejected": -191.61502075195312, + "loss": 0.0106, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 9.058492660522461, + "rewards/margins": 14.59411334991455, + "rewards/rejected": -5.535621643066406, + "step": 4130 + }, + { + "epoch": 0.45, + "learning_rate": 5.719951535711491e-06, + "logits/chosen": -0.8319818377494812, + "logits/rejected": 0.038639236241579056, + "logps/chosen": -172.58309936523438, + "logps/rejected": -180.60061645507812, + "loss": 0.1281, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": 7.336398124694824, + "rewards/margins": 12.854833602905273, + "rewards/rejected": -5.518437385559082, + "step": 4140 + }, + { + "epoch": 0.46, + "learning_rate": 5.702884421308211e-06, + "logits/chosen": -1.290502905845642, + "logits/rejected": 0.06660832464694977, + "logps/chosen": -273.30731201171875, + "logps/rejected": -187.1593017578125, + "loss": 0.0063, + "rewards/accuracies": 1.0, + "rewards/chosen": 10.415641784667969, + "rewards/margins": 16.28099250793457, + "rewards/rejected": -5.865350246429443, + "step": 4150 + }, + { + "epoch": 0.46, + "learning_rate": 5.685808948055953e-06, + "logits/chosen": -0.6947154402732849, + "logits/rejected": 0.28074347972869873, + "logps/chosen": -195.54896545410156, + "logps/rejected": -192.75460815429688, + "loss": 0.0115, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 7.675352573394775, + "rewards/margins": 13.444735527038574, + "rewards/rejected": -5.769382476806641, + "step": 4160 + }, + { + "epoch": 0.46, + "learning_rate": 5.668725319019825e-06, + "logits/chosen": -0.9217936396598816, + "logits/rejected": 0.1883951872587204, + "logps/chosen": -202.77401733398438, + "logps/rejected": -185.0608367919922, + "loss": 0.0525, + "rewards/accuracies": 0.9750000238418579, + "rewards/chosen": 7.657066345214844, + "rewards/margins": 13.645123481750488, + "rewards/rejected": -5.988056182861328, + "step": 4170 + }, + { + "epoch": 0.46, + "learning_rate": 5.6516337373619305e-06, + "logits/chosen": -0.630860447883606, + "logits/rejected": 0.629377007484436, + "logps/chosen": -181.9828338623047, + "logps/rejected": -184.57089233398438, + "loss": 0.0137, + "rewards/accuracies": 1.0, + "rewards/chosen": 7.93264627456665, + "rewards/margins": 13.083791732788086, + "rewards/rejected": -5.151145935058594, + "step": 4180 + }, + { + "epoch": 0.46, + "learning_rate": 5.63453440633894e-06, + "logits/chosen": -0.7545987963676453, + "logits/rejected": 0.7898412942886353, + "logps/chosen": -195.5791473388672, + "logps/rejected": -181.28187561035156, + "loss": 0.0002, + "rewards/accuracies": 1.0, + "rewards/chosen": 8.81618595123291, + "rewards/margins": 14.339630126953125, + "rewards/rejected": -5.523444175720215, + "step": 4190 + }, + { + "epoch": 0.46, + "learning_rate": 5.617427529299685e-06, + "logits/chosen": -0.8006548881530762, + "logits/rejected": 0.23377592861652374, + "logps/chosen": -206.93130493164062, + "logps/rejected": -177.07424926757812, + "loss": 0.0141, + "rewards/accuracies": 1.0, + "rewards/chosen": 8.404057502746582, + "rewards/margins": 13.534795761108398, + "rewards/rejected": -5.130738258361816, + "step": 4200 + }, + { + "epoch": 0.46, + "learning_rate": 5.6003133096827385e-06, + "logits/chosen": -0.6817094087600708, + "logits/rejected": 0.5712557435035706, + "logps/chosen": -186.84117126464844, + "logps/rejected": -184.6552734375, + "loss": 0.0075, + "rewards/accuracies": 1.0, + "rewards/chosen": 7.465065002441406, + "rewards/margins": 12.955281257629395, + "rewards/rejected": -5.490214824676514, + "step": 4210 + }, + { + "epoch": 0.46, + "learning_rate": 5.58319195101399e-06, + "logits/chosen": -0.6665258407592773, + "logits/rejected": 0.8567044138908386, + "logps/chosen": -203.0146026611328, + "logps/rejected": -189.3658447265625, + "loss": 0.0309, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 7.46883487701416, + "rewards/margins": 13.536404609680176, + "rewards/rejected": -6.06757116317749, + "step": 4220 + }, + { + "epoch": 0.46, + "learning_rate": 5.566063656904225e-06, + "logits/chosen": -0.31451043486595154, + "logits/rejected": 0.6520514488220215, + "logps/chosen": -165.8535614013672, + "logps/rejected": -195.01766967773438, + "loss": 0.0342, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 6.8415117263793945, + "rewards/margins": 13.461689949035645, + "rewards/rejected": -6.62017822265625, + "step": 4230 + }, + { + "epoch": 0.47, + "learning_rate": 5.548928631046713e-06, + "logits/chosen": -0.6523194909095764, + "logits/rejected": 0.40543240308761597, + "logps/chosen": -228.09481811523438, + "logps/rejected": -184.53695678710938, + "loss": 0.0315, + "rewards/accuracies": 0.9750000238418579, + "rewards/chosen": 8.172582626342773, + "rewards/margins": 14.0900297164917, + "rewards/rejected": -5.917446613311768, + "step": 4240 + }, + { + "epoch": 0.47, + "learning_rate": 5.531787077214774e-06, + "logits/chosen": -0.7589923739433289, + "logits/rejected": 0.3502810001373291, + "logps/chosen": -213.7614288330078, + "logps/rejected": -174.38406372070312, + "loss": 0.0251, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 8.07192611694336, + "rewards/margins": 13.847890853881836, + "rewards/rejected": -5.775967121124268, + "step": 4250 + }, + { + "epoch": 0.47, + "learning_rate": 5.5146391992593624e-06, + "logits/chosen": -0.8038038015365601, + "logits/rejected": 0.24132230877876282, + "logps/chosen": -216.2270965576172, + "logps/rejected": -179.63720703125, + "loss": 0.0082, + "rewards/accuracies": 1.0, + "rewards/chosen": 7.894244194030762, + "rewards/margins": 13.700251579284668, + "rewards/rejected": -5.80600643157959, + "step": 4260 + }, + { + "epoch": 0.47, + "learning_rate": 5.497485201106642e-06, + "logits/chosen": -0.9761279225349426, + "logits/rejected": 0.08464209735393524, + "logps/chosen": -216.7760467529297, + "logps/rejected": -179.83787536621094, + "loss": 0.0161, + "rewards/accuracies": 1.0, + "rewards/chosen": 8.562865257263184, + "rewards/margins": 13.760907173156738, + "rewards/rejected": -5.198040962219238, + "step": 4270 + }, + { + "epoch": 0.47, + "learning_rate": 5.480325286755554e-06, + "logits/chosen": -1.038810133934021, + "logits/rejected": 0.24261553585529327, + "logps/chosen": -214.66812133789062, + "logps/rejected": -184.3121795654297, + "loss": 0.002, + "rewards/accuracies": 1.0, + "rewards/chosen": 8.51683235168457, + "rewards/margins": 14.517120361328125, + "rewards/rejected": -6.000286102294922, + "step": 4280 + }, + { + "epoch": 0.47, + "learning_rate": 5.4631596602754e-06, + "logits/chosen": -0.6260752081871033, + "logits/rejected": 0.5203390121459961, + "logps/chosen": -203.24740600585938, + "logps/rejected": -190.6055450439453, + "loss": 0.004, + "rewards/accuracies": 1.0, + "rewards/chosen": 8.892824172973633, + "rewards/margins": 14.532137870788574, + "rewards/rejected": -5.639313697814941, + "step": 4290 + }, + { + "epoch": 0.47, + "learning_rate": 5.445988525803411e-06, + "logits/chosen": -0.688744068145752, + "logits/rejected": 0.5824708938598633, + "logps/chosen": -206.97756958007812, + "logps/rejected": -178.03558349609375, + "loss": 0.0467, + "rewards/accuracies": 0.9624999761581421, + "rewards/chosen": 8.408650398254395, + "rewards/margins": 14.216756820678711, + "rewards/rejected": -5.808108329772949, + "step": 4300 + }, + { + "epoch": 0.47, + "learning_rate": 5.428812087542322e-06, + "logits/chosen": -0.45431241393089294, + "logits/rejected": 0.9027924537658691, + "logps/chosen": -177.77171325683594, + "logps/rejected": -192.270263671875, + "loss": 0.0026, + "rewards/accuracies": 1.0, + "rewards/chosen": 7.24237585067749, + "rewards/margins": 12.832300186157227, + "rewards/rejected": -5.5899248123168945, + "step": 4310 + }, + { + "epoch": 0.47, + "learning_rate": 5.411630549757938e-06, + "logits/chosen": -0.7048603892326355, + "logits/rejected": 0.5619685053825378, + "logps/chosen": -213.6250457763672, + "logps/rejected": -184.37179565429688, + "loss": 0.0096, + "rewards/accuracies": 1.0, + "rewards/chosen": 7.697765350341797, + "rewards/margins": 13.373123168945312, + "rewards/rejected": -5.675358772277832, + "step": 4320 + }, + { + "epoch": 0.48, + "learning_rate": 5.3944441167767105e-06, + "logits/chosen": -0.6434752345085144, + "logits/rejected": 0.6572640538215637, + "logps/chosen": -202.946533203125, + "logps/rejected": -186.07513427734375, + "loss": 0.0085, + "rewards/accuracies": 1.0, + "rewards/chosen": 8.156560897827148, + "rewards/margins": 13.988609313964844, + "rewards/rejected": -5.832047939300537, + "step": 4330 + }, + { + "epoch": 0.48, + "learning_rate": 5.377252992983305e-06, + "logits/chosen": -0.34207403659820557, + "logits/rejected": 1.097680926322937, + "logps/chosen": -212.28158569335938, + "logps/rejected": -186.0108642578125, + "loss": 0.0455, + "rewards/accuracies": 0.9750000238418579, + "rewards/chosen": 8.398354530334473, + "rewards/margins": 14.180485725402832, + "rewards/rejected": -5.782132625579834, + "step": 4340 + }, + { + "epoch": 0.48, + "learning_rate": 5.360057382818173e-06, + "logits/chosen": -0.643896222114563, + "logits/rejected": 0.8727615475654602, + "logps/chosen": -237.4642333984375, + "logps/rejected": -190.25949096679688, + "loss": 0.0005, + "rewards/accuracies": 1.0, + "rewards/chosen": 8.536140441894531, + "rewards/margins": 14.670672416687012, + "rewards/rejected": -6.1345319747924805, + "step": 4350 + }, + { + "epoch": 0.48, + "learning_rate": 5.342857490775119e-06, + "logits/chosen": -0.3995625078678131, + "logits/rejected": 0.9093090295791626, + "logps/chosen": -167.27120971679688, + "logps/rejected": -169.56849670410156, + "loss": 0.0875, + "rewards/accuracies": 0.9750000238418579, + "rewards/chosen": 7.069319725036621, + "rewards/margins": 12.561596870422363, + "rewards/rejected": -5.492278099060059, + "step": 4360 + }, + { + "epoch": 0.48, + "learning_rate": 5.3256535213988635e-06, + "logits/chosen": -0.7793258428573608, + "logits/rejected": 0.26302796602249146, + "logps/chosen": -205.7571258544922, + "logps/rejected": -180.6692352294922, + "loss": 0.0033, + "rewards/accuracies": 1.0, + "rewards/chosen": 7.952573299407959, + "rewards/margins": 13.353075981140137, + "rewards/rejected": -5.400501728057861, + "step": 4370 + }, + { + "epoch": 0.48, + "learning_rate": 5.308445679282621e-06, + "logits/chosen": -0.952564537525177, + "logits/rejected": -0.22094249725341797, + "logps/chosen": -220.21170043945312, + "logps/rejected": -171.63125610351562, + "loss": 0.0074, + "rewards/accuracies": 1.0, + "rewards/chosen": 8.442703247070312, + "rewards/margins": 13.643461227416992, + "rewards/rejected": -5.200758934020996, + "step": 4380 + }, + { + "epoch": 0.48, + "learning_rate": 5.29123416906566e-06, + "logits/chosen": -0.7407370805740356, + "logits/rejected": 0.4141349196434021, + "logps/chosen": -185.99813842773438, + "logps/rejected": -181.07876586914062, + "loss": 0.0526, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 8.205766677856445, + "rewards/margins": 14.15662956237793, + "rewards/rejected": -5.950863838195801, + "step": 4390 + }, + { + "epoch": 0.48, + "learning_rate": 5.2740191954308705e-06, + "logits/chosen": -0.8846628069877625, + "logits/rejected": 0.17318400740623474, + "logps/chosen": -205.9633026123047, + "logps/rejected": -185.07827758789062, + "loss": 0.0185, + "rewards/accuracies": 1.0, + "rewards/chosen": 8.46017837524414, + "rewards/margins": 14.25574779510498, + "rewards/rejected": -5.795570373535156, + "step": 4400 + }, + { + "epoch": 0.48, + "learning_rate": 5.256800963102326e-06, + "logits/chosen": -0.6387470960617065, + "logits/rejected": 0.9342175722122192, + "logps/chosen": -205.51809692382812, + "logps/rejected": -181.8217315673828, + "loss": 0.0008, + "rewards/accuracies": 1.0, + "rewards/chosen": 8.550053596496582, + "rewards/margins": 14.11253547668457, + "rewards/rejected": -5.5624799728393555, + "step": 4410 + }, + { + "epoch": 0.49, + "learning_rate": 5.239579676842865e-06, + "logits/chosen": -0.6032951474189758, + "logits/rejected": 0.6683710813522339, + "logps/chosen": -168.44583129882812, + "logps/rejected": -182.81939697265625, + "loss": 0.0055, + "rewards/accuracies": 1.0, + "rewards/chosen": 7.677305698394775, + "rewards/margins": 12.796509742736816, + "rewards/rejected": -5.119204998016357, + "step": 4420 + }, + { + "epoch": 0.49, + "learning_rate": 5.222355541451628e-06, + "logits/chosen": -0.6749385595321655, + "logits/rejected": 0.4154078960418701, + "logps/chosen": -209.6761016845703, + "logps/rejected": -185.3213348388672, + "loss": 0.1704, + "rewards/accuracies": 0.9375, + "rewards/chosen": 7.994574069976807, + "rewards/margins": 13.584689140319824, + "rewards/rejected": -5.590114593505859, + "step": 4430 + }, + { + "epoch": 0.49, + "learning_rate": 5.205128761761652e-06, + "logits/chosen": -0.7134186029434204, + "logits/rejected": 0.6157000660896301, + "logps/chosen": -202.11390686035156, + "logps/rejected": -185.5843048095703, + "loss": 0.033, + "rewards/accuracies": 0.9750000238418579, + "rewards/chosen": 7.875798225402832, + "rewards/margins": 13.671160697937012, + "rewards/rejected": -5.795362949371338, + "step": 4440 + }, + { + "epoch": 0.49, + "learning_rate": 5.187899542637411e-06, + "logits/chosen": -0.3815995752811432, + "logits/rejected": 0.8913718461990356, + "logps/chosen": -183.87107849121094, + "logps/rejected": -183.4707489013672, + "loss": 0.0138, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 7.612691402435303, + "rewards/margins": 13.06416130065918, + "rewards/rejected": -5.451469421386719, + "step": 4450 + }, + { + "epoch": 0.49, + "learning_rate": 5.170668088972394e-06, + "logits/chosen": -0.41499918699264526, + "logits/rejected": 0.6295440196990967, + "logps/chosen": -189.31924438476562, + "logps/rejected": -179.04672241210938, + "loss": 0.0802, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 8.15086841583252, + "rewards/margins": 13.810157775878906, + "rewards/rejected": -5.659287452697754, + "step": 4460 + }, + { + "epoch": 0.49, + "learning_rate": 5.153434605686666e-06, + "logits/chosen": -0.954401969909668, + "logits/rejected": 0.6129035949707031, + "logps/chosen": -238.8607940673828, + "logps/rejected": -190.6068878173828, + "loss": 0.0084, + "rewards/accuracies": 1.0, + "rewards/chosen": 8.373200416564941, + "rewards/margins": 14.763836860656738, + "rewards/rejected": -6.390635967254639, + "step": 4470 + }, + { + "epoch": 0.49, + "learning_rate": 5.136199297724418e-06, + "logits/chosen": -0.4334503710269928, + "logits/rejected": 0.7219494581222534, + "logps/chosen": -182.95472717285156, + "logps/rejected": -179.6592254638672, + "loss": 0.1548, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": 6.549548149108887, + "rewards/margins": 11.936734199523926, + "rewards/rejected": -5.387185573577881, + "step": 4480 + }, + { + "epoch": 0.49, + "learning_rate": 5.118962370051555e-06, + "logits/chosen": -0.9024391174316406, + "logits/rejected": 0.20435722172260284, + "logps/chosen": -195.71685791015625, + "logps/rejected": -182.54238891601562, + "loss": 0.0093, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 8.693868637084961, + "rewards/margins": 14.144244194030762, + "rewards/rejected": -5.450375080108643, + "step": 4490 + }, + { + "epoch": 0.49, + "learning_rate": 5.10172402765323e-06, + "logits/chosen": -0.9569647908210754, + "logits/rejected": 0.4153072237968445, + "logps/chosen": -203.05850219726562, + "logps/rejected": -186.6887969970703, + "loss": 0.009, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 9.513406753540039, + "rewards/margins": 15.661529541015625, + "rewards/rejected": -6.148122787475586, + "step": 4500 + }, + { + "epoch": 0.5, + "learning_rate": 5.084484475531433e-06, + "logits/chosen": -0.7545306086540222, + "logits/rejected": 0.04692879319190979, + "logps/chosen": -202.9059600830078, + "logps/rejected": -179.72238159179688, + "loss": 0.0034, + "rewards/accuracies": 1.0, + "rewards/chosen": 9.072336196899414, + "rewards/margins": 14.245084762573242, + "rewards/rejected": -5.1727495193481445, + "step": 4510 + }, + { + "epoch": 0.5, + "learning_rate": 5.06724391870253e-06, + "logits/chosen": -0.5913695096969604, + "logits/rejected": 0.2880650758743286, + "logps/chosen": -164.234375, + "logps/rejected": -181.58956909179688, + "loss": 0.0326, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 7.800601005554199, + "rewards/margins": 13.816488265991211, + "rewards/rejected": -6.015888690948486, + "step": 4520 + }, + { + "epoch": 0.5, + "learning_rate": 5.05000256219484e-06, + "logits/chosen": -0.6467888951301575, + "logits/rejected": 0.4142070412635803, + "logps/chosen": -173.09030151367188, + "logps/rejected": -171.8232421875, + "loss": 0.0672, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 7.243517875671387, + "rewards/margins": 12.07048225402832, + "rewards/rejected": -4.826964855194092, + "step": 4530 + }, + { + "epoch": 0.5, + "learning_rate": 5.032760611046191e-06, + "logits/chosen": -1.1887036561965942, + "logits/rejected": -0.17404894530773163, + "logps/chosen": -235.6390380859375, + "logps/rejected": -182.72372436523438, + "loss": 0.0165, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 9.090173721313477, + "rewards/margins": 14.526514053344727, + "rewards/rejected": -5.436339378356934, + "step": 4540 + }, + { + "epoch": 0.5, + "learning_rate": 5.015518270301484e-06, + "logits/chosen": -0.9421218633651733, + "logits/rejected": 0.044965796172618866, + "logps/chosen": -184.64517211914062, + "logps/rejected": -175.2994842529297, + "loss": 0.0425, + "rewards/accuracies": 0.9750000238418579, + "rewards/chosen": 7.8314313888549805, + "rewards/margins": 13.028918266296387, + "rewards/rejected": -5.19748592376709, + "step": 4550 + }, + { + "epoch": 0.5, + "learning_rate": 4.998275745010251e-06, + "logits/chosen": -0.8001323938369751, + "logits/rejected": 0.012469601817429066, + "logps/chosen": -174.6398468017578, + "logps/rejected": -176.4419708251953, + "loss": 0.0107, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 8.210771560668945, + "rewards/margins": 13.202166557312012, + "rewards/rejected": -4.991394996643066, + "step": 4560 + }, + { + "epoch": 0.5, + "learning_rate": 4.98103324022422e-06, + "logits/chosen": -1.0280877351760864, + "logits/rejected": 0.10168919712305069, + "logps/chosen": -207.6393585205078, + "logps/rejected": -183.3862762451172, + "loss": 0.014, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 8.735610961914062, + "rewards/margins": 14.226269721984863, + "rewards/rejected": -5.490657806396484, + "step": 4570 + }, + { + "epoch": 0.5, + "learning_rate": 4.963790960994877e-06, + "logits/chosen": -0.9576606750488281, + "logits/rejected": 0.06801211833953857, + "logps/chosen": -182.6002655029297, + "logps/rejected": -187.04812622070312, + "loss": 0.1293, + "rewards/accuracies": 0.9750000238418579, + "rewards/chosen": 8.300407409667969, + "rewards/margins": 14.172993659973145, + "rewards/rejected": -5.872585296630859, + "step": 4580 + }, + { + "epoch": 0.5, + "learning_rate": 4.946549112371019e-06, + "logits/chosen": -1.3741296529769897, + "logits/rejected": -0.16209416091442108, + "logps/chosen": -236.72238159179688, + "logps/rejected": -173.78762817382812, + "loss": 0.0044, + "rewards/accuracies": 1.0, + "rewards/chosen": 10.192652702331543, + "rewards/margins": 15.731355667114258, + "rewards/rejected": -5.538702964782715, + "step": 4590 + }, + { + "epoch": 0.5, + "learning_rate": 4.929307899396329e-06, + "logits/chosen": -0.734040379524231, + "logits/rejected": 0.34174931049346924, + "logps/chosen": -157.61892700195312, + "logps/rejected": -185.26611328125, + "loss": 0.2312, + "rewards/accuracies": 0.9624999761581421, + "rewards/chosen": 6.588873863220215, + "rewards/margins": 12.040619850158691, + "rewards/rejected": -5.451745510101318, + "step": 4600 + }, + { + "epoch": 0.51, + "learning_rate": 4.91206752710693e-06, + "logits/chosen": -0.7689696550369263, + "logits/rejected": 0.3360385298728943, + "logps/chosen": -202.33737182617188, + "logps/rejected": -181.90505981445312, + "loss": 0.0299, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 8.812803268432617, + "rewards/margins": 14.005401611328125, + "rewards/rejected": -5.192598819732666, + "step": 4610 + }, + { + "epoch": 0.51, + "learning_rate": 4.894828200528942e-06, + "logits/chosen": -1.2357251644134521, + "logits/rejected": 0.29786649346351624, + "logps/chosen": -248.1954345703125, + "logps/rejected": -184.00601196289062, + "loss": 0.0537, + "rewards/accuracies": 0.9624999761581421, + "rewards/chosen": 9.47974967956543, + "rewards/margins": 14.87872314453125, + "rewards/rejected": -5.39897346496582, + "step": 4620 + }, + { + "epoch": 0.51, + "learning_rate": 4.877590124676056e-06, + "logits/chosen": -0.5456809401512146, + "logits/rejected": 0.6100348234176636, + "logps/chosen": -159.06454467773438, + "logps/rejected": -180.49423217773438, + "loss": 0.0988, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 7.2138519287109375, + "rewards/margins": 12.452654838562012, + "rewards/rejected": -5.238803863525391, + "step": 4630 + }, + { + "epoch": 0.51, + "learning_rate": 4.860353504547082e-06, + "logits/chosen": -0.9501746296882629, + "logits/rejected": 0.3249739110469818, + "logps/chosen": -217.73880004882812, + "logps/rejected": -185.79574584960938, + "loss": 0.1009, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": 9.2525634765625, + "rewards/margins": 14.809916496276855, + "rewards/rejected": -5.5573530197143555, + "step": 4640 + }, + { + "epoch": 0.51, + "learning_rate": 4.843118545123527e-06, + "logits/chosen": -0.6699641942977905, + "logits/rejected": 0.7281413078308105, + "logps/chosen": -194.4771728515625, + "logps/rejected": -190.56600952148438, + "loss": 0.0879, + "rewards/accuracies": 0.9750000238418579, + "rewards/chosen": 8.88044548034668, + "rewards/margins": 14.634503364562988, + "rewards/rejected": -5.754056453704834, + "step": 4650 + }, + { + "epoch": 0.51, + "learning_rate": 4.82588545136714e-06, + "logits/chosen": -0.7630641460418701, + "logits/rejected": 0.6130872964859009, + "logps/chosen": -241.93911743164062, + "logps/rejected": -188.03677368164062, + "loss": 0.0435, + "rewards/accuracies": 0.9750000238418579, + "rewards/chosen": 8.197587013244629, + "rewards/margins": 13.9602632522583, + "rewards/rejected": -5.76267671585083, + "step": 4660 + }, + { + "epoch": 0.51, + "learning_rate": 4.808654428217489e-06, + "logits/chosen": -0.6123110055923462, + "logits/rejected": 0.7432438135147095, + "logps/chosen": -179.47496032714844, + "logps/rejected": -184.6314239501953, + "loss": 0.0159, + "rewards/accuracies": 1.0, + "rewards/chosen": 7.713681221008301, + "rewards/margins": 13.63196849822998, + "rewards/rejected": -5.91828727722168, + "step": 4670 + }, + { + "epoch": 0.51, + "learning_rate": 4.791425680589517e-06, + "logits/chosen": -0.6790275573730469, + "logits/rejected": 0.9657877087593079, + "logps/chosen": -216.1737060546875, + "logps/rejected": -191.90419006347656, + "loss": 0.0074, + "rewards/accuracies": 1.0, + "rewards/chosen": 7.9500532150268555, + "rewards/margins": 14.165197372436523, + "rewards/rejected": -6.215145587921143, + "step": 4680 + }, + { + "epoch": 0.51, + "learning_rate": 4.7741994133711015e-06, + "logits/chosen": -0.7168338894844055, + "logits/rejected": 0.5399123430252075, + "logps/chosen": -200.19393920898438, + "logps/rejected": -189.21070861816406, + "loss": 0.0094, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 8.394033432006836, + "rewards/margins": 14.358769416809082, + "rewards/rejected": -5.9647345542907715, + "step": 4690 + }, + { + "epoch": 0.52, + "learning_rate": 4.756975831420628e-06, + "logits/chosen": -0.7052415609359741, + "logits/rejected": 0.476767361164093, + "logps/chosen": -188.21324157714844, + "logps/rejected": -175.54049682617188, + "loss": 0.3601, + "rewards/accuracies": 0.9624999761581421, + "rewards/chosen": 7.519739627838135, + "rewards/margins": 12.337279319763184, + "rewards/rejected": -4.817539691925049, + "step": 4700 + }, + { + "epoch": 0.52, + "learning_rate": 4.739755139564542e-06, + "logits/chosen": -0.5414751768112183, + "logits/rejected": 0.4507359564304352, + "logps/chosen": -195.75192260742188, + "logps/rejected": -180.8434600830078, + "loss": 0.02, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 7.585279941558838, + "rewards/margins": 13.487016677856445, + "rewards/rejected": -5.901736259460449, + "step": 4710 + }, + { + "epoch": 0.52, + "learning_rate": 4.72253754259493e-06, + "logits/chosen": -1.0838048458099365, + "logits/rejected": -0.16562095284461975, + "logps/chosen": -208.2504119873047, + "logps/rejected": -183.0047149658203, + "loss": 0.0001, + "rewards/accuracies": 1.0, + "rewards/chosen": 9.205527305603027, + "rewards/margins": 14.716331481933594, + "rewards/rejected": -5.510802745819092, + "step": 4720 + }, + { + "epoch": 0.52, + "learning_rate": 4.705323245267062e-06, + "logits/chosen": -0.9069498181343079, + "logits/rejected": 0.1422511339187622, + "logps/chosen": -195.85240173339844, + "logps/rejected": -180.38314819335938, + "loss": 0.0048, + "rewards/accuracies": 1.0, + "rewards/chosen": 8.861783981323242, + "rewards/margins": 14.293611526489258, + "rewards/rejected": -5.431827545166016, + "step": 4730 + }, + { + "epoch": 0.52, + "learning_rate": 4.688112452296975e-06, + "logits/chosen": -1.028296947479248, + "logits/rejected": 0.2485489845275879, + "logps/chosen": -191.14749145507812, + "logps/rejected": -170.748779296875, + "loss": 0.054, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 8.081120491027832, + "rewards/margins": 13.48466682434082, + "rewards/rejected": -5.403545379638672, + "step": 4740 + }, + { + "epoch": 0.52, + "learning_rate": 4.670905368359026e-06, + "logits/chosen": -0.9464433789253235, + "logits/rejected": 0.24524500966072083, + "logps/chosen": -195.4891815185547, + "logps/rejected": -183.15708923339844, + "loss": 0.0837, + "rewards/accuracies": 0.9750000238418579, + "rewards/chosen": 7.677785396575928, + "rewards/margins": 12.884183883666992, + "rewards/rejected": -5.206398963928223, + "step": 4750 + }, + { + "epoch": 0.52, + "learning_rate": 4.653702198083471e-06, + "logits/chosen": -0.5533558130264282, + "logits/rejected": 0.5586198568344116, + "logps/chosen": -162.0868377685547, + "logps/rejected": -164.98544311523438, + "loss": 0.2216, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": 7.787288665771484, + "rewards/margins": 12.697490692138672, + "rewards/rejected": -4.910201072692871, + "step": 4760 + }, + { + "epoch": 0.52, + "learning_rate": 4.636503146054018e-06, + "logits/chosen": -1.0734999179840088, + "logits/rejected": 0.4145767092704773, + "logps/chosen": -236.3977508544922, + "logps/rejected": -177.86154174804688, + "loss": 0.0389, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 9.071322441101074, + "rewards/margins": 14.562589645385742, + "rewards/rejected": -5.491265773773193, + "step": 4770 + }, + { + "epoch": 0.52, + "learning_rate": 4.6193084168054e-06, + "logits/chosen": -0.971180260181427, + "logits/rejected": 0.36026087403297424, + "logps/chosen": -233.7312469482422, + "logps/rejected": -186.50393676757812, + "loss": 0.0087, + "rewards/accuracies": 1.0, + "rewards/chosen": 9.337386131286621, + "rewards/margins": 15.04766845703125, + "rewards/rejected": -5.710282325744629, + "step": 4780 + }, + { + "epoch": 0.53, + "learning_rate": 4.602118214820943e-06, + "logits/chosen": -0.9507328271865845, + "logits/rejected": 0.6068797707557678, + "logps/chosen": -216.6476593017578, + "logps/rejected": -195.8607940673828, + "loss": 0.0244, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 7.4736647605896, + "rewards/margins": 14.297311782836914, + "rewards/rejected": -6.823646545410156, + "step": 4790 + }, + { + "epoch": 0.53, + "learning_rate": 4.5849327445301386e-06, + "logits/chosen": -0.6386322379112244, + "logits/rejected": 0.5726214647293091, + "logps/chosen": -219.59201049804688, + "logps/rejected": -180.49945068359375, + "loss": 0.0139, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 8.98166561126709, + "rewards/margins": 14.612759590148926, + "rewards/rejected": -5.631093502044678, + "step": 4800 + }, + { + "epoch": 0.53, + "learning_rate": 4.567752210306205e-06, + "logits/chosen": -0.8701270222663879, + "logits/rejected": 0.3848048448562622, + "logps/chosen": -230.98086547851562, + "logps/rejected": -182.6688995361328, + "loss": 0.1318, + "rewards/accuracies": 0.9750000238418579, + "rewards/chosen": 8.458674430847168, + "rewards/margins": 14.620373725891113, + "rewards/rejected": -6.161698341369629, + "step": 4810 + }, + { + "epoch": 0.53, + "learning_rate": 4.550576816463653e-06, + "logits/chosen": -0.6686583757400513, + "logits/rejected": 0.672886848449707, + "logps/chosen": -220.35287475585938, + "logps/rejected": -190.5135498046875, + "loss": 0.0006, + "rewards/accuracies": 1.0, + "rewards/chosen": 9.000184059143066, + "rewards/margins": 14.84306812286377, + "rewards/rejected": -5.842884063720703, + "step": 4820 + }, + { + "epoch": 0.53, + "learning_rate": 4.5334067672558744e-06, + "logits/chosen": -0.917603611946106, + "logits/rejected": 0.7294964790344238, + "logps/chosen": -198.56381225585938, + "logps/rejected": -180.45167541503906, + "loss": 0.0036, + "rewards/accuracies": 1.0, + "rewards/chosen": 7.6046929359436035, + "rewards/margins": 13.14905834197998, + "rewards/rejected": -5.544366359710693, + "step": 4830 + }, + { + "epoch": 0.53, + "learning_rate": 4.5162422668726945e-06, + "logits/chosen": -1.0886448621749878, + "logits/rejected": 0.18819625675678253, + "logps/chosen": -223.3118896484375, + "logps/rejected": -180.97540283203125, + "loss": 0.0092, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 8.280972480773926, + "rewards/margins": 13.785672187805176, + "rewards/rejected": -5.504698276519775, + "step": 4840 + }, + { + "epoch": 0.53, + "learning_rate": 4.499083519437952e-06, + "logits/chosen": -1.0019845962524414, + "logits/rejected": 0.30729231238365173, + "logps/chosen": -227.0318145751953, + "logps/rejected": -182.9668426513672, + "loss": 0.0285, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 8.17985725402832, + "rewards/margins": 14.323872566223145, + "rewards/rejected": -6.144015312194824, + "step": 4850 + }, + { + "epoch": 0.53, + "learning_rate": 4.48193072900707e-06, + "logits/chosen": -1.0907975435256958, + "logits/rejected": 0.2590438425540924, + "logps/chosen": -201.3037109375, + "logps/rejected": -179.9174041748047, + "loss": 0.0031, + "rewards/accuracies": 1.0, + "rewards/chosen": 8.420979499816895, + "rewards/margins": 13.7420015335083, + "rewards/rejected": -5.321022033691406, + "step": 4860 + }, + { + "epoch": 0.53, + "learning_rate": 4.464784099564632e-06, + "logits/chosen": -0.9262186288833618, + "logits/rejected": 0.4501849114894867, + "logps/chosen": -169.0706024169922, + "logps/rejected": -181.32388305664062, + "loss": 0.071, + "rewards/accuracies": 0.9624999761581421, + "rewards/chosen": 7.7464399337768555, + "rewards/margins": 12.95796012878418, + "rewards/rejected": -5.211521148681641, + "step": 4870 + }, + { + "epoch": 0.54, + "learning_rate": 4.447643835021953e-06, + "logits/chosen": -0.8851696848869324, + "logits/rejected": 0.6138371229171753, + "logps/chosen": -215.5679168701172, + "logps/rejected": -192.98678588867188, + "loss": 0.0096, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 9.003774642944336, + "rewards/margins": 15.069601058959961, + "rewards/rejected": -6.065826892852783, + "step": 4880 + }, + { + "epoch": 0.54, + "learning_rate": 4.4305101392146515e-06, + "logits/chosen": -0.7926458716392517, + "logits/rejected": 0.39124152064323425, + "logps/chosen": -183.29782104492188, + "logps/rejected": -177.13916015625, + "loss": 0.0211, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 7.430880069732666, + "rewards/margins": 12.904255867004395, + "rewards/rejected": -5.4733757972717285, + "step": 4890 + }, + { + "epoch": 0.54, + "learning_rate": 4.413383215900232e-06, + "logits/chosen": -0.835827648639679, + "logits/rejected": 0.1583215296268463, + "logps/chosen": -186.98434448242188, + "logps/rejected": -172.3527069091797, + "loss": 0.0568, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 6.354737758636475, + "rewards/margins": 11.641669273376465, + "rewards/rejected": -5.286931991577148, + "step": 4900 + }, + { + "epoch": 0.54, + "learning_rate": 4.3962632687556626e-06, + "logits/chosen": -1.0084965229034424, + "logits/rejected": 0.08662146329879761, + "logps/chosen": -213.5675048828125, + "logps/rejected": -174.35342407226562, + "loss": 0.0135, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 8.125871658325195, + "rewards/margins": 13.612256050109863, + "rewards/rejected": -5.48638391494751, + "step": 4910 + }, + { + "epoch": 0.54, + "learning_rate": 4.379150501374943e-06, + "logits/chosen": -0.8009958267211914, + "logits/rejected": 0.21286901831626892, + "logps/chosen": -185.96267700195312, + "logps/rejected": -175.7305145263672, + "loss": 0.1647, + "rewards/accuracies": 0.9750000238418579, + "rewards/chosen": 8.525096893310547, + "rewards/margins": 13.921238899230957, + "rewards/rejected": -5.396140098571777, + "step": 4920 + }, + { + "epoch": 0.54, + "learning_rate": 4.3620451172666925e-06, + "logits/chosen": -1.2459161281585693, + "logits/rejected": 0.020923614501953125, + "logps/chosen": -216.74545288085938, + "logps/rejected": -192.02957153320312, + "loss": 0.0091, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 9.152045249938965, + "rewards/margins": 15.310752868652344, + "rewards/rejected": -6.1587066650390625, + "step": 4930 + }, + { + "epoch": 0.54, + "learning_rate": 4.344947319851729e-06, + "logits/chosen": -1.0657240152359009, + "logits/rejected": -0.05446208640933037, + "logps/chosen": -220.2706298828125, + "logps/rejected": -180.03269958496094, + "loss": 0.0081, + "rewards/accuracies": 1.0, + "rewards/chosen": 9.367846488952637, + "rewards/margins": 14.622970581054688, + "rewards/rejected": -5.255122184753418, + "step": 4940 + }, + { + "epoch": 0.54, + "learning_rate": 4.327857312460645e-06, + "logits/chosen": -1.074399471282959, + "logits/rejected": 0.07393421977758408, + "logps/chosen": -191.76834106445312, + "logps/rejected": -190.2980499267578, + "loss": 0.1383, + "rewards/accuracies": 0.9750000238418579, + "rewards/chosen": 8.580821990966797, + "rewards/margins": 14.130426406860352, + "rewards/rejected": -5.549603462219238, + "step": 4950 + }, + { + "epoch": 0.54, + "learning_rate": 4.310775298331393e-06, + "logits/chosen": -1.1435766220092773, + "logits/rejected": -0.20423726737499237, + "logps/chosen": -200.04046630859375, + "logps/rejected": -192.34320068359375, + "loss": 0.0113, + "rewards/accuracies": 1.0, + "rewards/chosen": 9.40774917602539, + "rewards/margins": 15.171661376953125, + "rewards/rejected": -5.763912677764893, + "step": 4960 + }, + { + "epoch": 0.55, + "learning_rate": 4.2937014806068664e-06, + "logits/chosen": -1.0613980293273926, + "logits/rejected": -0.039209458976984024, + "logps/chosen": -173.2572784423828, + "logps/rejected": -175.23521423339844, + "loss": 0.1319, + "rewards/accuracies": 0.9750000238418579, + "rewards/chosen": 8.649354934692383, + "rewards/margins": 13.494479179382324, + "rewards/rejected": -4.845124244689941, + "step": 4970 + }, + { + "epoch": 0.55, + "learning_rate": 4.276636062332491e-06, + "logits/chosen": -1.3733454942703247, + "logits/rejected": -0.1296839416027069, + "logps/chosen": -215.390380859375, + "logps/rejected": -184.770751953125, + "loss": 0.0317, + "rewards/accuracies": 0.9750000238418579, + "rewards/chosen": 9.224201202392578, + "rewards/margins": 15.182487487792969, + "rewards/rejected": -5.958284854888916, + "step": 4980 + }, + { + "epoch": 0.55, + "learning_rate": 4.259579246453798e-06, + "logits/chosen": -1.366075873374939, + "logits/rejected": -0.14353640377521515, + "logps/chosen": -236.23165893554688, + "logps/rejected": -186.31289672851562, + "loss": 0.0535, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 9.38115119934082, + "rewards/margins": 14.998846054077148, + "rewards/rejected": -5.617697715759277, + "step": 4990 + }, + { + "epoch": 0.55, + "learning_rate": 4.242531235814021e-06, + "logits/chosen": -1.0184311866760254, + "logits/rejected": 0.03931038826704025, + "logps/chosen": -210.69790649414062, + "logps/rejected": -187.8129425048828, + "loss": 0.0031, + "rewards/accuracies": 1.0, + "rewards/chosen": 9.429067611694336, + "rewards/margins": 15.270278930664062, + "rewards/rejected": -5.841213226318359, + "step": 5000 + }, + { + "epoch": 0.55, + "learning_rate": 4.225492233151675e-06, + "logits/chosen": -0.9547805786132812, + "logits/rejected": 0.23295803368091583, + "logps/chosen": -213.426513671875, + "logps/rejected": -185.5917205810547, + "loss": 0.0302, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 8.588269233703613, + "rewards/margins": 14.15275764465332, + "rewards/rejected": -5.564488887786865, + "step": 5010 + }, + { + "epoch": 0.55, + "learning_rate": 4.20846244109816e-06, + "logits/chosen": -0.7889807820320129, + "logits/rejected": 0.6832569241523743, + "logps/chosen": -195.78858947753906, + "logps/rejected": -189.51736450195312, + "loss": 0.0181, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 8.591264724731445, + "rewards/margins": 13.917966842651367, + "rewards/rejected": -5.32670259475708, + "step": 5020 + }, + { + "epoch": 0.55, + "learning_rate": 4.1914420621753304e-06, + "logits/chosen": -1.2002742290496826, + "logits/rejected": -0.10469566285610199, + "logps/chosen": -217.95425415039062, + "logps/rejected": -187.55615234375, + "loss": 0.0721, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 8.623835563659668, + "rewards/margins": 14.374601364135742, + "rewards/rejected": -5.750766754150391, + "step": 5030 + }, + { + "epoch": 0.55, + "learning_rate": 4.174431298793104e-06, + "logits/chosen": -0.9864124059677124, + "logits/rejected": 0.24313950538635254, + "logps/chosen": -218.65234375, + "logps/rejected": -183.7274169921875, + "loss": 0.004, + "rewards/accuracies": 1.0, + "rewards/chosen": 8.897613525390625, + "rewards/margins": 14.740809440612793, + "rewards/rejected": -5.843194484710693, + "step": 5040 + }, + { + "epoch": 0.55, + "learning_rate": 4.1574303532470495e-06, + "logits/chosen": -0.9470454454421997, + "logits/rejected": 0.11348254978656769, + "logps/chosen": -200.40408325195312, + "logps/rejected": -173.1186981201172, + "loss": 0.0328, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 7.482891082763672, + "rewards/margins": 13.272237777709961, + "rewards/rejected": -5.7893476486206055, + "step": 5050 + }, + { + "epoch": 0.56, + "learning_rate": 4.140439427715974e-06, + "logits/chosen": -1.0601842403411865, + "logits/rejected": 0.2971873879432678, + "logps/chosen": -179.07620239257812, + "logps/rejected": -185.2488250732422, + "loss": 0.0037, + "rewards/accuracies": 1.0, + "rewards/chosen": 8.427019119262695, + "rewards/margins": 13.96496295928955, + "rewards/rejected": -5.537942886352539, + "step": 5060 + }, + { + "epoch": 0.56, + "learning_rate": 4.123458724259531e-06, + "logits/chosen": -0.9963864088058472, + "logits/rejected": 0.11450407654047012, + "logps/chosen": -197.88583374023438, + "logps/rejected": -185.1710205078125, + "loss": 0.0196, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 8.091116905212402, + "rewards/margins": 13.69628620147705, + "rewards/rejected": -5.605168342590332, + "step": 5070 + }, + { + "epoch": 0.56, + "learning_rate": 4.106488444815804e-06, + "logits/chosen": -1.1627365350723267, + "logits/rejected": -0.11678476631641388, + "logps/chosen": -185.05099487304688, + "logps/rejected": -192.7058868408203, + "loss": 0.0007, + "rewards/accuracies": 1.0, + "rewards/chosen": 8.497145652770996, + "rewards/margins": 14.18517017364502, + "rewards/rejected": -5.688024044036865, + "step": 5080 + }, + { + "epoch": 0.56, + "learning_rate": 4.089528791198921e-06, + "logits/chosen": -1.2952816486358643, + "logits/rejected": -0.2672204077243805, + "logps/chosen": -227.4910888671875, + "logps/rejected": -181.75279235839844, + "loss": 0.0058, + "rewards/accuracies": 1.0, + "rewards/chosen": 9.179617881774902, + "rewards/margins": 14.288859367370605, + "rewards/rejected": -5.109241008758545, + "step": 5090 + }, + { + "epoch": 0.56, + "learning_rate": 4.072579965096636e-06, + "logits/chosen": -1.0086488723754883, + "logits/rejected": 0.1738024353981018, + "logps/chosen": -179.86331176757812, + "logps/rejected": -174.23855590820312, + "loss": 0.0046, + "rewards/accuracies": 1.0, + "rewards/chosen": 8.37100887298584, + "rewards/margins": 13.666338920593262, + "rewards/rejected": -5.29533052444458, + "step": 5100 + }, + { + "epoch": 0.56, + "learning_rate": 4.055642168067948e-06, + "logits/chosen": -1.1480693817138672, + "logits/rejected": -0.19907662272453308, + "logps/chosen": -198.89398193359375, + "logps/rejected": -177.84854125976562, + "loss": 0.1353, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": 8.131129264831543, + "rewards/margins": 12.798421859741211, + "rewards/rejected": -4.667293548583984, + "step": 5110 + }, + { + "epoch": 0.56, + "learning_rate": 4.038715601540688e-06, + "logits/chosen": -1.1078460216522217, + "logits/rejected": -0.3208760619163513, + "logps/chosen": -197.23672485351562, + "logps/rejected": -170.53231811523438, + "loss": 0.0387, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 8.54615592956543, + "rewards/margins": 13.196466445922852, + "rewards/rejected": -4.650310039520264, + "step": 5120 + }, + { + "epoch": 0.56, + "learning_rate": 4.021800466809139e-06, + "logits/chosen": -1.2008130550384521, + "logits/rejected": 0.003859803080558777, + "logps/chosen": -189.840087890625, + "logps/rejected": -174.17877197265625, + "loss": 0.0059, + "rewards/accuracies": 1.0, + "rewards/chosen": 8.222555160522461, + "rewards/margins": 13.430081367492676, + "rewards/rejected": -5.207527160644531, + "step": 5130 + }, + { + "epoch": 0.56, + "learning_rate": 4.004896965031632e-06, + "logits/chosen": -1.2304285764694214, + "logits/rejected": -0.2553466260433197, + "logps/chosen": -173.52183532714844, + "logps/rejected": -182.59384155273438, + "loss": 0.0073, + "rewards/accuracies": 1.0, + "rewards/chosen": 8.06871223449707, + "rewards/margins": 13.533531188964844, + "rewards/rejected": -5.464818954467773, + "step": 5140 + }, + { + "epoch": 0.57, + "learning_rate": 3.98800529722815e-06, + "logits/chosen": -0.8401597142219543, + "logits/rejected": 0.15512582659721375, + "logps/chosen": -173.1329345703125, + "logps/rejected": -170.1700439453125, + "loss": 0.0884, + "rewards/accuracies": 0.9750000238418579, + "rewards/chosen": 7.484180450439453, + "rewards/margins": 12.161384582519531, + "rewards/rejected": -4.67720365524292, + "step": 5150 + }, + { + "epoch": 0.57, + "learning_rate": 3.971125664277958e-06, + "logits/chosen": -1.2346577644348145, + "logits/rejected": -0.20221209526062012, + "logps/chosen": -217.12118530273438, + "logps/rejected": -182.79701232910156, + "loss": 0.0097, + "rewards/accuracies": 1.0, + "rewards/chosen": 9.30477237701416, + "rewards/margins": 14.273828506469727, + "rewards/rejected": -4.969056606292725, + "step": 5160 + }, + { + "epoch": 0.57, + "learning_rate": 3.954258266917187e-06, + "logits/chosen": -0.9965928196907043, + "logits/rejected": 0.033581119030714035, + "logps/chosen": -179.70750427246094, + "logps/rejected": -184.35324096679688, + "loss": 0.051, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 8.968358993530273, + "rewards/margins": 14.013829231262207, + "rewards/rejected": -5.045471668243408, + "step": 5170 + }, + { + "epoch": 0.57, + "learning_rate": 3.9374033057364666e-06, + "logits/chosen": -1.0810410976409912, + "logits/rejected": 0.013114121742546558, + "logps/chosen": -218.6728515625, + "logps/rejected": -172.34471130371094, + "loss": 0.0183, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 8.672164916992188, + "rewards/margins": 13.6160306930542, + "rewards/rejected": -4.9438676834106445, + "step": 5180 + }, + { + "epoch": 0.57, + "learning_rate": 3.9205609811785275e-06, + "logits/chosen": -1.0918828248977661, + "logits/rejected": -0.008401790633797646, + "logps/chosen": -172.5182342529297, + "logps/rejected": -180.76766967773438, + "loss": 0.0124, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 8.240156173706055, + "rewards/margins": 14.163264274597168, + "rewards/rejected": -5.92310905456543, + "step": 5190 + }, + { + "epoch": 0.57, + "learning_rate": 3.90373149353583e-06, + "logits/chosen": -0.9498961567878723, + "logits/rejected": -0.0253736674785614, + "logps/chosen": -200.4408416748047, + "logps/rejected": -168.7683868408203, + "loss": 0.0034, + "rewards/accuracies": 1.0, + "rewards/chosen": 7.947314262390137, + "rewards/margins": 13.175023078918457, + "rewards/rejected": -5.227709770202637, + "step": 5200 + }, + { + "epoch": 0.57, + "learning_rate": 3.88691504294817e-06, + "logits/chosen": -1.008616328239441, + "logits/rejected": 0.012529527768492699, + "logps/chosen": -209.7909698486328, + "logps/rejected": -189.36770629882812, + "loss": 0.0066, + "rewards/accuracies": 1.0, + "rewards/chosen": 9.624855041503906, + "rewards/margins": 15.426839828491211, + "rewards/rejected": -5.80198335647583, + "step": 5210 + }, + { + "epoch": 0.57, + "learning_rate": 3.8701118294003044e-06, + "logits/chosen": -1.0405100584030151, + "logits/rejected": 0.148539200425148, + "logps/chosen": -196.4014892578125, + "logps/rejected": -187.49044799804688, + "loss": 0.0408, + "rewards/accuracies": 0.9750000238418579, + "rewards/chosen": 7.668177127838135, + "rewards/margins": 13.154162406921387, + "rewards/rejected": -5.485984802246094, + "step": 5220 + }, + { + "epoch": 0.57, + "learning_rate": 3.853322052719574e-06, + "logits/chosen": -1.100319743156433, + "logits/rejected": -0.06421175599098206, + "logps/chosen": -195.27235412597656, + "logps/rejected": -179.41915893554688, + "loss": 0.0613, + "rewards/accuracies": 0.9624999761581421, + "rewards/chosen": 8.20017147064209, + "rewards/margins": 13.827512741088867, + "rewards/rejected": -5.627340316772461, + "step": 5230 + }, + { + "epoch": 0.58, + "learning_rate": 3.836545912573525e-06, + "logits/chosen": -1.089259386062622, + "logits/rejected": -0.06584231555461884, + "logps/chosen": -224.3650665283203, + "logps/rejected": -189.21188354492188, + "loss": 0.0071, + "rewards/accuracies": 1.0, + "rewards/chosen": 9.759481430053711, + "rewards/margins": 15.483747482299805, + "rewards/rejected": -5.724265098571777, + "step": 5240 + }, + { + "epoch": 0.58, + "learning_rate": 3.819783608467537e-06, + "logits/chosen": -1.021034836769104, + "logits/rejected": 0.06812737137079239, + "logps/chosen": -189.48294067382812, + "logps/rejected": -186.53060913085938, + "loss": 0.1201, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 8.534686088562012, + "rewards/margins": 14.287933349609375, + "rewards/rejected": -5.753246784210205, + "step": 5250 + }, + { + "epoch": 0.58, + "learning_rate": 3.8030353397424424e-06, + "logits/chosen": -1.1220064163208008, + "logits/rejected": 0.1265530288219452, + "logps/chosen": -156.4834747314453, + "logps/rejected": -184.54100036621094, + "loss": 0.0194, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 8.267587661743164, + "rewards/margins": 13.787984848022461, + "rewards/rejected": -5.5203986167907715, + "step": 5260 + }, + { + "epoch": 0.58, + "learning_rate": 3.78630130557217e-06, + "logits/chosen": -0.9782911539077759, + "logits/rejected": 0.26680728793144226, + "logps/chosen": -170.0625, + "logps/rejected": -181.2744598388672, + "loss": 0.2418, + "rewards/accuracies": 0.9624999761581421, + "rewards/chosen": 8.016959190368652, + "rewards/margins": 13.305875778198242, + "rewards/rejected": -5.288916110992432, + "step": 5270 + }, + { + "epoch": 0.58, + "learning_rate": 3.769581704961363e-06, + "logits/chosen": -0.7643753290176392, + "logits/rejected": 0.03026052750647068, + "logps/chosen": -196.53579711914062, + "logps/rejected": -177.55812072753906, + "loss": 0.0137, + "rewards/accuracies": 1.0, + "rewards/chosen": 9.043940544128418, + "rewards/margins": 13.810079574584961, + "rewards/rejected": -4.766139984130859, + "step": 5280 + }, + { + "epoch": 0.58, + "learning_rate": 3.7528767367430186e-06, + "logits/chosen": -1.0079470872879028, + "logits/rejected": -0.10408540070056915, + "logps/chosen": -166.36965942382812, + "logps/rejected": -184.6284637451172, + "loss": 0.0017, + "rewards/accuracies": 1.0, + "rewards/chosen": 8.412015914916992, + "rewards/margins": 13.71405029296875, + "rewards/rejected": -5.302033424377441, + "step": 5290 + }, + { + "epoch": 0.58, + "learning_rate": 3.7361865995761227e-06, + "logits/chosen": -1.2601544857025146, + "logits/rejected": -0.15969321131706238, + "logps/chosen": -224.0481719970703, + "logps/rejected": -177.85546875, + "loss": 0.0222, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 9.396010398864746, + "rewards/margins": 14.916166305541992, + "rewards/rejected": -5.520155906677246, + "step": 5300 + }, + { + "epoch": 0.58, + "learning_rate": 3.7195114919432885e-06, + "logits/chosen": -1.165015459060669, + "logits/rejected": -0.24738237261772156, + "logps/chosen": -203.65562438964844, + "logps/rejected": -182.73056030273438, + "loss": 0.0264, + "rewards/accuracies": 0.9750000238418579, + "rewards/chosen": 8.304207801818848, + "rewards/margins": 14.152432441711426, + "rewards/rejected": -5.8482255935668945, + "step": 5310 + }, + { + "epoch": 0.58, + "learning_rate": 3.702851612148394e-06, + "logits/chosen": -1.174020767211914, + "logits/rejected": -0.3383522629737854, + "logps/chosen": -207.3739776611328, + "logps/rejected": -168.86866760253906, + "loss": 0.0232, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 9.243462562561035, + "rewards/margins": 14.532663345336914, + "rewards/rejected": -5.289200305938721, + "step": 5320 + }, + { + "epoch": 0.59, + "learning_rate": 3.6862071583142233e-06, + "logits/chosen": -1.6077499389648438, + "logits/rejected": -0.7026212811470032, + "logps/chosen": -261.7918395996094, + "logps/rejected": -177.78353881835938, + "loss": 0.0002, + "rewards/accuracies": 1.0, + "rewards/chosen": 10.589468002319336, + "rewards/margins": 16.030941009521484, + "rewards/rejected": -5.44147253036499, + "step": 5330 + }, + { + "epoch": 0.59, + "learning_rate": 3.669578328380112e-06, + "logits/chosen": -1.264427661895752, + "logits/rejected": -0.6264600157737732, + "logps/chosen": -188.02459716796875, + "logps/rejected": -160.68272399902344, + "loss": 0.02, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 8.80695915222168, + "rewards/margins": 13.235427856445312, + "rewards/rejected": -4.428469657897949, + "step": 5340 + }, + { + "epoch": 0.59, + "learning_rate": 3.652965320099598e-06, + "logits/chosen": -1.3935978412628174, + "logits/rejected": -0.44697514176368713, + "logps/chosen": -189.55398559570312, + "logps/rejected": -170.17755126953125, + "loss": 0.0098, + "rewards/accuracies": 1.0, + "rewards/chosen": 8.62216567993164, + "rewards/margins": 13.726305961608887, + "rewards/rejected": -5.104140281677246, + "step": 5350 + }, + { + "epoch": 0.59, + "learning_rate": 3.636368331038056e-06, + "logits/chosen": -1.0938408374786377, + "logits/rejected": -0.08468153327703476, + "logps/chosen": -200.24697875976562, + "logps/rejected": -192.48544311523438, + "loss": 0.0617, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 8.6934232711792, + "rewards/margins": 14.218844413757324, + "rewards/rejected": -5.525421142578125, + "step": 5360 + }, + { + "epoch": 0.59, + "learning_rate": 3.6197875585703645e-06, + "logits/chosen": -1.3205479383468628, + "logits/rejected": -0.23628750443458557, + "logps/chosen": -189.9931182861328, + "logps/rejected": -172.4279327392578, + "loss": 0.0018, + "rewards/accuracies": 1.0, + "rewards/chosen": 8.973957061767578, + "rewards/margins": 14.350814819335938, + "rewards/rejected": -5.376858711242676, + "step": 5370 + }, + { + "epoch": 0.59, + "learning_rate": 3.6032231998785426e-06, + "logits/chosen": -1.3135552406311035, + "logits/rejected": -0.229862779378891, + "logps/chosen": -210.47164916992188, + "logps/rejected": -170.59133911132812, + "loss": 0.0743, + "rewards/accuracies": 0.9624999761581421, + "rewards/chosen": 9.246811866760254, + "rewards/margins": 14.259259223937988, + "rewards/rejected": -5.01244592666626, + "step": 5380 + }, + { + "epoch": 0.59, + "learning_rate": 3.5866754519494264e-06, + "logits/chosen": -1.487305998802185, + "logits/rejected": -0.3329917788505554, + "logps/chosen": -243.86764526367188, + "logps/rejected": -176.1786346435547, + "loss": 0.0329, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 8.889004707336426, + "rewards/margins": 14.251480102539062, + "rewards/rejected": -5.362476825714111, + "step": 5390 + }, + { + "epoch": 0.59, + "learning_rate": 3.5701445115722988e-06, + "logits/chosen": -1.1767661571502686, + "logits/rejected": 0.04709335416555405, + "logps/chosen": -229.74862670898438, + "logps/rejected": -185.2811279296875, + "loss": 0.0302, + "rewards/accuracies": 0.9750000238418579, + "rewards/chosen": 9.00699520111084, + "rewards/margins": 14.211896896362305, + "rewards/rejected": -5.204900741577148, + "step": 5400 + }, + { + "epoch": 0.59, + "learning_rate": 3.553630575336573e-06, + "logits/chosen": -1.030590295791626, + "logits/rejected": -0.13648587465286255, + "logps/chosen": -196.59556579589844, + "logps/rejected": -174.63861083984375, + "loss": 0.0405, + "rewards/accuracies": 0.9750000238418579, + "rewards/chosen": 8.899336814880371, + "rewards/margins": 14.128782272338867, + "rewards/rejected": -5.229445457458496, + "step": 5410 + }, + { + "epoch": 0.59, + "learning_rate": 3.5371338396294438e-06, + "logits/chosen": -0.9977045059204102, + "logits/rejected": 0.019182879477739334, + "logps/chosen": -193.71250915527344, + "logps/rejected": -178.93331909179688, + "loss": 0.0311, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 8.904383659362793, + "rewards/margins": 13.72663402557373, + "rewards/rejected": -4.8222503662109375, + "step": 5420 + }, + { + "epoch": 0.6, + "learning_rate": 3.5206545006335503e-06, + "logits/chosen": -0.9094837307929993, + "logits/rejected": 0.06615578383207321, + "logps/chosen": -189.33526611328125, + "logps/rejected": -186.91700744628906, + "loss": 0.003, + "rewards/accuracies": 1.0, + "rewards/chosen": 8.510332107543945, + "rewards/margins": 13.925036430358887, + "rewards/rejected": -5.4147047996521, + "step": 5430 + }, + { + "epoch": 0.6, + "learning_rate": 3.5041927543246512e-06, + "logits/chosen": -0.8606934547424316, + "logits/rejected": 0.24336138367652893, + "logps/chosen": -177.2454376220703, + "logps/rejected": -183.53671264648438, + "loss": 0.0012, + "rewards/accuracies": 1.0, + "rewards/chosen": 8.01698112487793, + "rewards/margins": 13.420774459838867, + "rewards/rejected": -5.403792858123779, + "step": 5440 + }, + { + "epoch": 0.6, + "learning_rate": 3.4877487964692826e-06, + "logits/chosen": -1.0251572132110596, + "logits/rejected": 0.06962103396654129, + "logps/chosen": -257.5361328125, + "logps/rejected": -175.0287322998047, + "loss": 0.0165, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 8.47108268737793, + "rewards/margins": 13.400647163391113, + "rewards/rejected": -4.929563522338867, + "step": 5450 + }, + { + "epoch": 0.6, + "learning_rate": 3.4713228226224457e-06, + "logits/chosen": -0.9619434475898743, + "logits/rejected": 0.11097557842731476, + "logps/chosen": -218.35708618164062, + "logps/rejected": -181.64364624023438, + "loss": 0.0122, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 8.344583511352539, + "rewards/margins": 13.485153198242188, + "rewards/rejected": -5.140568733215332, + "step": 5460 + }, + { + "epoch": 0.6, + "learning_rate": 3.4549150281252635e-06, + "logits/chosen": -1.0553277730941772, + "logits/rejected": 0.2147468775510788, + "logps/chosen": -226.7494659423828, + "logps/rejected": -187.18087768554688, + "loss": 0.0007, + "rewards/accuracies": 1.0, + "rewards/chosen": 8.790245056152344, + "rewards/margins": 14.778207778930664, + "rewards/rejected": -5.98796272277832, + "step": 5470 + }, + { + "epoch": 0.6, + "learning_rate": 3.4385256081026726e-06, + "logits/chosen": -0.9712486267089844, + "logits/rejected": 0.2894454598426819, + "logps/chosen": -212.61227416992188, + "logps/rejected": -175.61264038085938, + "loss": 0.0748, + "rewards/accuracies": 0.9750000238418579, + "rewards/chosen": 8.447061538696289, + "rewards/margins": 13.487568855285645, + "rewards/rejected": -5.0405073165893555, + "step": 5480 + }, + { + "epoch": 0.6, + "learning_rate": 3.42215475746109e-06, + "logits/chosen": -1.1190592050552368, + "logits/rejected": 0.18061628937721252, + "logps/chosen": -233.0695343017578, + "logps/rejected": -191.27474975585938, + "loss": 0.004, + "rewards/accuracies": 1.0, + "rewards/chosen": 9.968509674072266, + "rewards/margins": 16.05965232849121, + "rewards/rejected": -6.091143608093262, + "step": 5490 + }, + { + "epoch": 0.6, + "learning_rate": 3.4058026708861093e-06, + "logits/chosen": -0.7445631623268127, + "logits/rejected": 0.43725594878196716, + "logps/chosen": -234.87533569335938, + "logps/rejected": -186.11325073242188, + "loss": 0.0118, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 8.430233001708984, + "rewards/margins": 14.502713203430176, + "rewards/rejected": -6.072480201721191, + "step": 5500 + }, + { + "epoch": 0.6, + "learning_rate": 3.3894695428401734e-06, + "logits/chosen": -1.08945631980896, + "logits/rejected": -0.04441007971763611, + "logps/chosen": -255.0380096435547, + "logps/rejected": -184.3826904296875, + "loss": 0.0157, + "rewards/accuracies": 1.0, + "rewards/chosen": 9.017770767211914, + "rewards/margins": 15.233999252319336, + "rewards/rejected": -6.216231346130371, + "step": 5510 + }, + { + "epoch": 0.61, + "learning_rate": 3.3731555675602636e-06, + "logits/chosen": -1.1750695705413818, + "logits/rejected": 0.24161013960838318, + "logps/chosen": -215.9639129638672, + "logps/rejected": -185.32827758789062, + "loss": 0.006, + "rewards/accuracies": 1.0, + "rewards/chosen": 8.233799934387207, + "rewards/margins": 13.935168266296387, + "rewards/rejected": -5.7013678550720215, + "step": 5520 + }, + { + "epoch": 0.61, + "learning_rate": 3.3568609390556016e-06, + "logits/chosen": -0.9278244972229004, + "logits/rejected": 0.3396621346473694, + "logps/chosen": -209.61813354492188, + "logps/rejected": -181.01705932617188, + "loss": 0.0527, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 8.667435646057129, + "rewards/margins": 14.57538890838623, + "rewards/rejected": -5.90795373916626, + "step": 5530 + }, + { + "epoch": 0.61, + "learning_rate": 3.3405858511053246e-06, + "logits/chosen": -0.8884087800979614, + "logits/rejected": 0.2673329710960388, + "logps/chosen": -186.288818359375, + "logps/rejected": -194.6363525390625, + "loss": 0.0009, + "rewards/accuracies": 1.0, + "rewards/chosen": 8.108078956604004, + "rewards/margins": 14.056683540344238, + "rewards/rejected": -5.948603630065918, + "step": 5540 + }, + { + "epoch": 0.61, + "learning_rate": 3.324330497256194e-06, + "logits/chosen": -1.0837953090667725, + "logits/rejected": 0.3320137858390808, + "logps/chosen": -194.463134765625, + "logps/rejected": -187.18167114257812, + "loss": 0.0048, + "rewards/accuracies": 1.0, + "rewards/chosen": 8.932564735412598, + "rewards/margins": 14.862157821655273, + "rewards/rejected": -5.929591655731201, + "step": 5550 + }, + { + "epoch": 0.61, + "learning_rate": 3.3080950708202835e-06, + "logits/chosen": -0.9159994125366211, + "logits/rejected": -0.017248010262846947, + "logps/chosen": -202.34439086914062, + "logps/rejected": -178.2544708251953, + "loss": 0.0345, + "rewards/accuracies": 0.9750000238418579, + "rewards/chosen": 7.400962829589844, + "rewards/margins": 12.550459861755371, + "rewards/rejected": -5.149497032165527, + "step": 5560 + }, + { + "epoch": 0.61, + "learning_rate": 3.2918797648726943e-06, + "logits/chosen": -1.045116662979126, + "logits/rejected": 0.26237210631370544, + "logps/chosen": -173.43325805664062, + "logps/rejected": -178.5285186767578, + "loss": 0.1248, + "rewards/accuracies": 0.9624999761581421, + "rewards/chosen": 8.17446517944336, + "rewards/margins": 13.443216323852539, + "rewards/rejected": -5.26875114440918, + "step": 5570 + }, + { + "epoch": 0.61, + "learning_rate": 3.275684772249245e-06, + "logits/chosen": -1.0298752784729004, + "logits/rejected": 0.053309548646211624, + "logps/chosen": -229.3160400390625, + "logps/rejected": -176.22567749023438, + "loss": 0.0112, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 8.307023048400879, + "rewards/margins": 13.509603500366211, + "rewards/rejected": -5.202579021453857, + "step": 5580 + }, + { + "epoch": 0.61, + "learning_rate": 3.2611268059644535e-06, + "logits/chosen": -0.9335830807685852, + "logits/rejected": 0.03628344461321831, + "logps/chosen": -188.78448486328125, + "logps/rejected": -188.6182861328125, + "loss": 0.091, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 8.40925407409668, + "rewards/margins": 14.89020824432373, + "rewards/rejected": -6.480953216552734, + "step": 5590 + }, + { + "epoch": 0.61, + "learning_rate": 3.2449709390525232e-06, + "logits/chosen": -0.7816407084465027, + "logits/rejected": 0.2446083277463913, + "logps/chosen": -174.77828979492188, + "logps/rejected": -178.8984375, + "loss": 0.0605, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 7.598228454589844, + "rewards/margins": 12.591958999633789, + "rewards/rejected": -4.993730068206787, + "step": 5600 + }, + { + "epoch": 0.62, + "learning_rate": 3.228835943314328e-06, + "logits/chosen": -0.8714820146560669, + "logits/rejected": 0.20546278357505798, + "logps/chosen": -218.04971313476562, + "logps/rejected": -187.475341796875, + "loss": 0.0002, + "rewards/accuracies": 1.0, + "rewards/chosen": 9.142573356628418, + "rewards/margins": 15.050715446472168, + "rewards/rejected": -5.908143520355225, + "step": 5610 + }, + { + "epoch": 0.62, + "learning_rate": 3.2127220106306213e-06, + "logits/chosen": -0.9456221461296082, + "logits/rejected": 0.3141826391220093, + "logps/chosen": -224.47695922851562, + "logps/rejected": -194.45501708984375, + "loss": 0.0002, + "rewards/accuracies": 1.0, + "rewards/chosen": 8.80375862121582, + "rewards/margins": 14.868741035461426, + "rewards/rejected": -6.064981460571289, + "step": 5620 + }, + { + "epoch": 0.62, + "learning_rate": 3.1966293326316737e-06, + "logits/chosen": -1.0490838289260864, + "logits/rejected": -0.05881056934595108, + "logps/chosen": -211.04824829101562, + "logps/rejected": -184.84347534179688, + "loss": 0.0247, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 8.471969604492188, + "rewards/margins": 14.602516174316406, + "rewards/rejected": -6.130547046661377, + "step": 5630 + }, + { + "epoch": 0.62, + "learning_rate": 3.1805581006949856e-06, + "logits/chosen": -0.9726631045341492, + "logits/rejected": -0.05700913071632385, + "logps/chosen": -195.0536651611328, + "logps/rejected": -180.6584930419922, + "loss": 0.0006, + "rewards/accuracies": 1.0, + "rewards/chosen": 8.448801040649414, + "rewards/margins": 14.354080200195312, + "rewards/rejected": -5.905278205871582, + "step": 5640 + }, + { + "epoch": 0.62, + "learning_rate": 3.164508505943017e-06, + "logits/chosen": -1.2344918251037598, + "logits/rejected": -0.01758049614727497, + "logps/chosen": -199.63905334472656, + "logps/rejected": -183.68402099609375, + "loss": 0.0038, + "rewards/accuracies": 1.0, + "rewards/chosen": 8.820901870727539, + "rewards/margins": 14.194856643676758, + "rewards/rejected": -5.373953342437744, + "step": 5650 + }, + { + "epoch": 0.62, + "learning_rate": 3.14848073924092e-06, + "logits/chosen": -1.2216378450393677, + "logits/rejected": 0.007369053550064564, + "logps/chosen": -196.869384765625, + "logps/rejected": -190.6266326904297, + "loss": 0.0418, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 8.45164680480957, + "rewards/margins": 14.704195976257324, + "rewards/rejected": -6.252549648284912, + "step": 5660 + }, + { + "epoch": 0.62, + "learning_rate": 3.1324749911942536e-06, + "logits/chosen": -1.2782294750213623, + "logits/rejected": -0.029264485463500023, + "logps/chosen": -214.2439727783203, + "logps/rejected": -189.81402587890625, + "loss": 0.0634, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 8.663556098937988, + "rewards/margins": 14.860873222351074, + "rewards/rejected": -6.197319030761719, + "step": 5670 + }, + { + "epoch": 0.62, + "learning_rate": 3.116491452146733e-06, + "logits/chosen": -1.2257130146026611, + "logits/rejected": -0.21547213196754456, + "logps/chosen": -200.75865173339844, + "logps/rejected": -196.5606231689453, + "loss": 0.0857, + "rewards/accuracies": 0.9624999761581421, + "rewards/chosen": 9.141754150390625, + "rewards/margins": 15.067355155944824, + "rewards/rejected": -5.925600528717041, + "step": 5680 + }, + { + "epoch": 0.62, + "learning_rate": 3.100530312177956e-06, + "logits/chosen": -0.9037885665893555, + "logits/rejected": 0.1715497076511383, + "logps/chosen": -182.9431915283203, + "logps/rejected": -192.597900390625, + "loss": 0.0188, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 8.08227825164795, + "rewards/margins": 14.035197257995605, + "rewards/rejected": -5.952918529510498, + "step": 5690 + }, + { + "epoch": 0.63, + "learning_rate": 3.08459176110115e-06, + "logits/chosen": -1.1575199365615845, + "logits/rejected": 0.04202013462781906, + "logps/chosen": -235.9733428955078, + "logps/rejected": -187.81007385253906, + "loss": 0.013, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 8.920618057250977, + "rewards/margins": 15.382345199584961, + "rewards/rejected": -6.461728096008301, + "step": 5700 + }, + { + "epoch": 0.63, + "learning_rate": 3.068675988460906e-06, + "logits/chosen": -0.9626846313476562, + "logits/rejected": 0.19772040843963623, + "logps/chosen": -195.8952178955078, + "logps/rejected": -188.57432556152344, + "loss": 0.0553, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 8.037010192871094, + "rewards/margins": 14.410204887390137, + "rewards/rejected": -6.373194217681885, + "step": 5710 + }, + { + "epoch": 0.63, + "learning_rate": 3.0527831835309307e-06, + "logits/chosen": -0.9246395826339722, + "logits/rejected": 0.20215901732444763, + "logps/chosen": -197.05067443847656, + "logps/rejected": -181.14773559570312, + "loss": 0.0548, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 7.646282196044922, + "rewards/margins": 13.679468154907227, + "rewards/rejected": -6.033186435699463, + "step": 5720 + }, + { + "epoch": 0.63, + "learning_rate": 3.036913535311793e-06, + "logits/chosen": -0.7127299904823303, + "logits/rejected": 0.4527320861816406, + "logps/chosen": -190.44869995117188, + "logps/rejected": -194.6103057861328, + "loss": 0.1284, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 7.0552167892456055, + "rewards/margins": 12.662599563598633, + "rewards/rejected": -5.607382774353027, + "step": 5730 + }, + { + "epoch": 0.63, + "learning_rate": 3.0210672325286806e-06, + "logits/chosen": -1.211329460144043, + "logits/rejected": -0.09112808108329773, + "logps/chosen": -191.41470336914062, + "logps/rejected": -183.78598022460938, + "loss": 0.0115, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 8.155797004699707, + "rewards/margins": 13.693120002746582, + "rewards/rejected": -5.537322521209717, + "step": 5740 + }, + { + "epoch": 0.63, + "learning_rate": 3.005244463629149e-06, + "logits/chosen": -1.0562803745269775, + "logits/rejected": 0.004866314120590687, + "logps/chosen": -219.1081085205078, + "logps/rejected": -179.34768676757812, + "loss": 0.0149, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 7.7882843017578125, + "rewards/margins": 14.035421371459961, + "rewards/rejected": -6.247137069702148, + "step": 5750 + }, + { + "epoch": 0.63, + "learning_rate": 2.989445416780884e-06, + "logits/chosen": -1.0745458602905273, + "logits/rejected": 0.22102828323841095, + "logps/chosen": -200.72052001953125, + "logps/rejected": -174.39718627929688, + "loss": 0.0412, + "rewards/accuracies": 0.9750000238418579, + "rewards/chosen": 7.619148254394531, + "rewards/margins": 12.893455505371094, + "rewards/rejected": -5.274306297302246, + "step": 5760 + }, + { + "epoch": 0.63, + "learning_rate": 2.973670279869468e-06, + "logits/chosen": -1.0522193908691406, + "logits/rejected": 0.01774367317557335, + "logps/chosen": -205.6265869140625, + "logps/rejected": -183.48049926757812, + "loss": 0.1129, + "rewards/accuracies": 0.9624999761581421, + "rewards/chosen": 7.756146430969238, + "rewards/margins": 13.651895523071289, + "rewards/rejected": -5.895748138427734, + "step": 5770 + }, + { + "epoch": 0.63, + "learning_rate": 2.9579192404961356e-06, + "logits/chosen": -0.9277795553207397, + "logits/rejected": -0.10022075474262238, + "logps/chosen": -189.68736267089844, + "logps/rejected": -183.51992797851562, + "loss": 0.0122, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 9.03683090209961, + "rewards/margins": 14.807385444641113, + "rewards/rejected": -5.770554542541504, + "step": 5780 + }, + { + "epoch": 0.64, + "learning_rate": 2.9421924859755525e-06, + "logits/chosen": -0.9462844729423523, + "logits/rejected": -0.24103613197803497, + "logps/chosen": -187.8827667236328, + "logps/rejected": -189.0424346923828, + "loss": 0.1394, + "rewards/accuracies": 0.9624999761581421, + "rewards/chosen": 8.581372261047363, + "rewards/margins": 14.009033203125, + "rewards/rejected": -5.427661895751953, + "step": 5790 + }, + { + "epoch": 0.64, + "learning_rate": 2.9264902033335798e-06, + "logits/chosen": -1.3170831203460693, + "logits/rejected": -0.08774524927139282, + "logps/chosen": -251.01779174804688, + "logps/rejected": -181.05410766601562, + "loss": 0.2022, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 8.921258926391602, + "rewards/margins": 14.600123405456543, + "rewards/rejected": -5.678863525390625, + "step": 5800 + }, + { + "epoch": 0.64, + "learning_rate": 2.910812579305059e-06, + "logits/chosen": -1.2133440971374512, + "logits/rejected": 0.10181646049022675, + "logps/chosen": -191.245849609375, + "logps/rejected": -182.93052673339844, + "loss": 0.0427, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 8.285595893859863, + "rewards/margins": 14.027323722839355, + "rewards/rejected": -5.741728782653809, + "step": 5810 + }, + { + "epoch": 0.64, + "learning_rate": 2.895159800331584e-06, + "logits/chosen": -0.9897164106369019, + "logits/rejected": -0.015085858292877674, + "logps/chosen": -208.8571319580078, + "logps/rejected": -176.52316284179688, + "loss": 0.0026, + "rewards/accuracies": 1.0, + "rewards/chosen": 8.15942096710205, + "rewards/margins": 13.899198532104492, + "rewards/rejected": -5.739777565002441, + "step": 5820 + }, + { + "epoch": 0.64, + "learning_rate": 2.8795320525592822e-06, + "logits/chosen": -1.237201452255249, + "logits/rejected": -0.1343379020690918, + "logps/chosen": -233.84548950195312, + "logps/rejected": -185.21981811523438, + "loss": 0.0011, + "rewards/accuracies": 1.0, + "rewards/chosen": 9.747323989868164, + "rewards/margins": 15.545846939086914, + "rewards/rejected": -5.798523902893066, + "step": 5830 + }, + { + "epoch": 0.64, + "learning_rate": 2.8639295218366115e-06, + "logits/chosen": -1.213356614112854, + "logits/rejected": -0.17545083165168762, + "logps/chosen": -216.81039428710938, + "logps/rejected": -186.24696350097656, + "loss": 0.0249, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 8.343297958374023, + "rewards/margins": 14.747121810913086, + "rewards/rejected": -6.403825283050537, + "step": 5840 + }, + { + "epoch": 0.64, + "learning_rate": 2.8483523937121405e-06, + "logits/chosen": -1.0333861112594604, + "logits/rejected": -0.023768095299601555, + "logps/chosen": -227.94253540039062, + "logps/rejected": -182.49606323242188, + "loss": 0.0205, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 8.91602611541748, + "rewards/margins": 14.8850679397583, + "rewards/rejected": -5.969041347503662, + "step": 5850 + }, + { + "epoch": 0.64, + "learning_rate": 2.8328008534323436e-06, + "logits/chosen": -1.0451548099517822, + "logits/rejected": 0.0541093647480011, + "logps/chosen": -232.60842895507812, + "logps/rejected": -184.9303436279297, + "loss": 0.0232, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 9.005636215209961, + "rewards/margins": 14.7823486328125, + "rewards/rejected": -5.7767133712768555, + "step": 5860 + }, + { + "epoch": 0.64, + "learning_rate": 2.8172750859393975e-06, + "logits/chosen": -1.1292508840560913, + "logits/rejected": -0.012727809138596058, + "logps/chosen": -189.83975219726562, + "logps/rejected": -173.8275909423828, + "loss": 0.0339, + "rewards/accuracies": 0.9750000238418579, + "rewards/chosen": 7.9906816482543945, + "rewards/margins": 13.579411506652832, + "rewards/rejected": -5.5887298583984375, + "step": 5870 + }, + { + "epoch": 0.65, + "learning_rate": 2.80177527586899e-06, + "logits/chosen": -0.9195248484611511, + "logits/rejected": 0.15079766511917114, + "logps/chosen": -199.79298400878906, + "logps/rejected": -193.98916625976562, + "loss": 0.1863, + "rewards/accuracies": 0.9750000238418579, + "rewards/chosen": 8.202776908874512, + "rewards/margins": 13.84611988067627, + "rewards/rejected": -5.643342971801758, + "step": 5880 + }, + { + "epoch": 0.65, + "learning_rate": 2.78630160754811e-06, + "logits/chosen": -0.8377178311347961, + "logits/rejected": 0.11564052104949951, + "logps/chosen": -189.81103515625, + "logps/rejected": -187.5968780517578, + "loss": 0.0004, + "rewards/accuracies": 1.0, + "rewards/chosen": 8.025246620178223, + "rewards/margins": 13.469091415405273, + "rewards/rejected": -5.443844318389893, + "step": 5890 + }, + { + "epoch": 0.65, + "learning_rate": 2.7708542649928705e-06, + "logits/chosen": -0.9481223821640015, + "logits/rejected": 0.20021291077136993, + "logps/chosen": -189.6348876953125, + "logps/rejected": -183.4775848388672, + "loss": 0.0215, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 7.536964416503906, + "rewards/margins": 13.63830852508545, + "rewards/rejected": -6.101346015930176, + "step": 5900 + }, + { + "epoch": 0.65, + "learning_rate": 2.7554334319063064e-06, + "logits/chosen": -1.0069230794906616, + "logits/rejected": 0.37985721230506897, + "logps/chosen": -203.15484619140625, + "logps/rejected": -187.3056182861328, + "loss": 0.0283, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 7.802495002746582, + "rewards/margins": 13.8259916305542, + "rewards/rejected": -6.023496150970459, + "step": 5910 + }, + { + "epoch": 0.65, + "learning_rate": 2.7400392916762024e-06, + "logits/chosen": -0.9302213788032532, + "logits/rejected": 0.13181108236312866, + "logps/chosen": -201.9678955078125, + "logps/rejected": -179.43331909179688, + "loss": 0.0835, + "rewards/accuracies": 0.9624999761581421, + "rewards/chosen": 7.21148681640625, + "rewards/margins": 12.817831039428711, + "rewards/rejected": -5.606344699859619, + "step": 5920 + }, + { + "epoch": 0.65, + "learning_rate": 2.7246720273729e-06, + "logits/chosen": -0.8341774940490723, + "logits/rejected": 0.36767578125, + "logps/chosen": -181.20968627929688, + "logps/rejected": -178.75027465820312, + "loss": 0.1092, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 7.399981498718262, + "rewards/margins": 13.118809700012207, + "rewards/rejected": -5.7188286781311035, + "step": 5930 + }, + { + "epoch": 0.65, + "learning_rate": 2.709331821747133e-06, + "logits/chosen": -1.2560975551605225, + "logits/rejected": 0.1485179364681244, + "logps/chosen": -279.9898681640625, + "logps/rejected": -183.17840576171875, + "loss": 0.0141, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 9.59929084777832, + "rewards/margins": 16.1364803314209, + "rewards/rejected": -6.537188529968262, + "step": 5940 + }, + { + "epoch": 0.65, + "learning_rate": 2.6940188572278426e-06, + "logits/chosen": -0.9459528923034668, + "logits/rejected": 0.3523729741573334, + "logps/chosen": -215.90017700195312, + "logps/rejected": -193.37371826171875, + "loss": 0.0092, + "rewards/accuracies": 1.0, + "rewards/chosen": 8.62419605255127, + "rewards/margins": 14.588279724121094, + "rewards/rejected": -5.964083671569824, + "step": 5950 + }, + { + "epoch": 0.65, + "learning_rate": 2.6787333159200155e-06, + "logits/chosen": -0.7440406680107117, + "logits/rejected": 0.5182219743728638, + "logps/chosen": -182.66246032714844, + "logps/rejected": -175.5028076171875, + "loss": 0.0945, + "rewards/accuracies": 0.9624999761581421, + "rewards/chosen": 8.029151916503906, + "rewards/margins": 13.541165351867676, + "rewards/rejected": -5.512013912200928, + "step": 5960 + }, + { + "epoch": 0.66, + "learning_rate": 2.6634753796025146e-06, + "logits/chosen": -0.7508438229560852, + "logits/rejected": 0.4019510746002197, + "logps/chosen": -184.87550354003906, + "logps/rejected": -182.425537109375, + "loss": 0.0831, + "rewards/accuracies": 0.9624999761581421, + "rewards/chosen": 8.700386047363281, + "rewards/margins": 14.049428939819336, + "rewards/rejected": -5.3490424156188965, + "step": 5970 + }, + { + "epoch": 0.66, + "learning_rate": 2.64824522972592e-06, + "logits/chosen": -0.7918148040771484, + "logits/rejected": 0.18190129101276398, + "logps/chosen": -224.34860229492188, + "logps/rejected": -181.943359375, + "loss": 0.0804, + "rewards/accuracies": 0.9750000238418579, + "rewards/chosen": 7.495013236999512, + "rewards/margins": 13.476895332336426, + "rewards/rejected": -5.9818830490112305, + "step": 5980 + }, + { + "epoch": 0.66, + "learning_rate": 2.63304304741037e-06, + "logits/chosen": -1.2457331418991089, + "logits/rejected": 0.09142757952213287, + "logps/chosen": -192.3347625732422, + "logps/rejected": -201.03953552246094, + "loss": 0.0076, + "rewards/accuracies": 1.0, + "rewards/chosen": 8.189595222473145, + "rewards/margins": 14.678304672241211, + "rewards/rejected": -6.488709926605225, + "step": 5990 + }, + { + "epoch": 0.66, + "learning_rate": 2.617869013443405e-06, + "logits/chosen": -0.8004394769668579, + "logits/rejected": 0.19559602439403534, + "logps/chosen": -191.4009246826172, + "logps/rejected": -183.6040802001953, + "loss": 0.1006, + "rewards/accuracies": 0.9624999761581421, + "rewards/chosen": 7.5390119552612305, + "rewards/margins": 13.366350173950195, + "rewards/rejected": -5.827338218688965, + "step": 6000 + }, + { + "epoch": 0.66, + "learning_rate": 2.602723308277818e-06, + "logits/chosen": -0.9042922258377075, + "logits/rejected": 0.19536356627941132, + "logps/chosen": -191.7236785888672, + "logps/rejected": -177.6675567626953, + "loss": 0.0127, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 9.085429191589355, + "rewards/margins": 14.54327392578125, + "rewards/rejected": -5.457844257354736, + "step": 6010 + }, + { + "epoch": 0.66, + "learning_rate": 2.5876061120295116e-06, + "logits/chosen": -1.078558325767517, + "logits/rejected": -0.27258172631263733, + "logps/chosen": -188.9149932861328, + "logps/rejected": -175.99302673339844, + "loss": 0.0068, + "rewards/accuracies": 1.0, + "rewards/chosen": 8.493955612182617, + "rewards/margins": 14.13653564453125, + "rewards/rejected": -5.642579078674316, + "step": 6020 + }, + { + "epoch": 0.66, + "learning_rate": 2.5725176044753564e-06, + "logits/chosen": -1.0515755414962769, + "logits/rejected": 0.1628810614347458, + "logps/chosen": -208.49575805664062, + "logps/rejected": -190.539794921875, + "loss": 0.0074, + "rewards/accuracies": 1.0, + "rewards/chosen": 8.529611587524414, + "rewards/margins": 14.779321670532227, + "rewards/rejected": -6.249711036682129, + "step": 6030 + }, + { + "epoch": 0.66, + "learning_rate": 2.55745796505105e-06, + "logits/chosen": -1.009021282196045, + "logits/rejected": -0.08621358871459961, + "logps/chosen": -223.6774139404297, + "logps/rejected": -188.89695739746094, + "loss": 0.0112, + "rewards/accuracies": 1.0, + "rewards/chosen": 9.048395156860352, + "rewards/margins": 14.870767593383789, + "rewards/rejected": -5.822371006011963, + "step": 6040 + }, + { + "epoch": 0.66, + "learning_rate": 2.5424273728489834e-06, + "logits/chosen": -0.9302619695663452, + "logits/rejected": 0.06558765470981598, + "logps/chosen": -213.0177001953125, + "logps/rejected": -188.65896606445312, + "loss": 0.0101, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 8.779377937316895, + "rewards/margins": 14.969032287597656, + "rewards/rejected": -6.189655303955078, + "step": 6050 + }, + { + "epoch": 0.67, + "learning_rate": 2.5274260066161083e-06, + "logits/chosen": -0.9044649004936218, + "logits/rejected": 0.38030341267585754, + "logps/chosen": -197.818359375, + "logps/rejected": -186.36831665039062, + "loss": 0.0692, + "rewards/accuracies": 0.9624999761581421, + "rewards/chosen": 7.796571254730225, + "rewards/margins": 13.810815811157227, + "rewards/rejected": -6.014243125915527, + "step": 6060 + }, + { + "epoch": 0.67, + "learning_rate": 2.5124540447518208e-06, + "logits/chosen": -1.1478192806243896, + "logits/rejected": 0.12909851968288422, + "logps/chosen": -226.08975219726562, + "logps/rejected": -198.24192810058594, + "loss": 0.0161, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 8.917764663696289, + "rewards/margins": 15.206369400024414, + "rewards/rejected": -6.288603782653809, + "step": 6070 + }, + { + "epoch": 0.67, + "learning_rate": 2.4975116653058353e-06, + "logits/chosen": -0.8903303146362305, + "logits/rejected": 0.3929794132709503, + "logps/chosen": -197.5379638671875, + "logps/rejected": -189.542236328125, + "loss": 0.0111, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 7.872075080871582, + "rewards/margins": 14.023582458496094, + "rewards/rejected": -6.151508808135986, + "step": 6080 + }, + { + "epoch": 0.67, + "learning_rate": 2.482599045976059e-06, + "logits/chosen": -1.2484896183013916, + "logits/rejected": -0.1056201234459877, + "logps/chosen": -237.899658203125, + "logps/rejected": -181.1265106201172, + "loss": 0.0079, + "rewards/accuracies": 1.0, + "rewards/chosen": 8.903064727783203, + "rewards/margins": 14.716474533081055, + "rewards/rejected": -5.813408851623535, + "step": 6090 + }, + { + "epoch": 0.67, + "learning_rate": 2.4677163641064883e-06, + "logits/chosen": -1.0946948528289795, + "logits/rejected": 0.2692286968231201, + "logps/chosen": -209.6959991455078, + "logps/rejected": -192.5835723876953, + "loss": 0.0004, + "rewards/accuracies": 1.0, + "rewards/chosen": 8.24077033996582, + "rewards/margins": 14.295206069946289, + "rewards/rejected": -6.054433345794678, + "step": 6100 + }, + { + "epoch": 0.67, + "learning_rate": 2.4528637966850995e-06, + "logits/chosen": -1.003951072692871, + "logits/rejected": 0.20093221962451935, + "logps/chosen": -239.9269561767578, + "logps/rejected": -189.2813720703125, + "loss": 0.0019, + "rewards/accuracies": 1.0, + "rewards/chosen": 9.052816390991211, + "rewards/margins": 14.761378288269043, + "rewards/rejected": -5.708563804626465, + "step": 6110 + }, + { + "epoch": 0.67, + "learning_rate": 2.4380415203417414e-06, + "logits/chosen": -1.1368443965911865, + "logits/rejected": -0.038627639412879944, + "logps/chosen": -240.48291015625, + "logps/rejected": -186.58209228515625, + "loss": 0.002, + "rewards/accuracies": 1.0, + "rewards/chosen": 8.884922981262207, + "rewards/margins": 14.394433975219727, + "rewards/rejected": -5.509511947631836, + "step": 6120 + }, + { + "epoch": 0.67, + "learning_rate": 2.4232497113460303e-06, + "logits/chosen": -1.022178292274475, + "logits/rejected": 0.09224029630422592, + "logps/chosen": -195.36639404296875, + "logps/rejected": -185.6175079345703, + "loss": 0.018, + "rewards/accuracies": 0.9750000238418579, + "rewards/chosen": 8.93779182434082, + "rewards/margins": 14.736639022827148, + "rewards/rejected": -5.798847198486328, + "step": 6130 + }, + { + "epoch": 0.67, + "learning_rate": 2.408488545605265e-06, + "logits/chosen": -0.7985199093818665, + "logits/rejected": 0.3752705454826355, + "logps/chosen": -196.5523223876953, + "logps/rejected": -183.46192932128906, + "loss": 0.0727, + "rewards/accuracies": 0.9750000238418579, + "rewards/chosen": 7.818743705749512, + "rewards/margins": 13.770419120788574, + "rewards/rejected": -5.951673984527588, + "step": 6140 + }, + { + "epoch": 0.68, + "learning_rate": 2.3937581986623226e-06, + "logits/chosen": -0.8995197415351868, + "logits/rejected": 0.47575241327285767, + "logps/chosen": -175.42457580566406, + "logps/rejected": -183.8799591064453, + "loss": 0.1632, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 8.25619125366211, + "rewards/margins": 14.069437026977539, + "rewards/rejected": -5.8132452964782715, + "step": 6150 + }, + { + "epoch": 0.68, + "learning_rate": 2.379058845693577e-06, + "logits/chosen": -1.152989387512207, + "logits/rejected": 0.042288508266210556, + "logps/chosen": -225.2469024658203, + "logps/rejected": -174.23788452148438, + "loss": 0.0083, + "rewards/accuracies": 1.0, + "rewards/chosen": 8.122655868530273, + "rewards/margins": 13.928024291992188, + "rewards/rejected": -5.8053669929504395, + "step": 6160 + }, + { + "epoch": 0.68, + "learning_rate": 2.36439066150682e-06, + "logits/chosen": -0.9735552668571472, + "logits/rejected": 0.21133749186992645, + "logps/chosen": -219.2079620361328, + "logps/rejected": -181.19412231445312, + "loss": 0.0501, + "rewards/accuracies": 0.9624999761581421, + "rewards/chosen": 8.290315628051758, + "rewards/margins": 13.71661376953125, + "rewards/rejected": -5.426297187805176, + "step": 6170 + }, + { + "epoch": 0.68, + "learning_rate": 2.3497538205391764e-06, + "logits/chosen": -1.023113489151001, + "logits/rejected": 0.21690067648887634, + "logps/chosen": -211.17568969726562, + "logps/rejected": -183.7834930419922, + "loss": 0.0544, + "rewards/accuracies": 0.9624999761581421, + "rewards/chosen": 8.615364074707031, + "rewards/margins": 14.555577278137207, + "rewards/rejected": -5.940211296081543, + "step": 6180 + }, + { + "epoch": 0.68, + "learning_rate": 2.3351484968550264e-06, + "logits/chosen": -0.8495364189147949, + "logits/rejected": 0.28810757398605347, + "logps/chosen": -220.89572143554688, + "logps/rejected": -175.24998474121094, + "loss": 0.0009, + "rewards/accuracies": 1.0, + "rewards/chosen": 8.465044975280762, + "rewards/margins": 14.139215469360352, + "rewards/rejected": -5.674169063568115, + "step": 6190 + }, + { + "epoch": 0.68, + "learning_rate": 2.320574864143942e-06, + "logits/chosen": -0.9068955183029175, + "logits/rejected": 0.2122136801481247, + "logps/chosen": -176.02151489257812, + "logps/rejected": -171.41033935546875, + "loss": 0.0102, + "rewards/accuracies": 1.0, + "rewards/chosen": 7.600134372711182, + "rewards/margins": 13.048696517944336, + "rewards/rejected": -5.448561668395996, + "step": 6200 + }, + { + "epoch": 0.68, + "learning_rate": 2.306033095718622e-06, + "logits/chosen": -1.2303380966186523, + "logits/rejected": -0.09389712661504745, + "logps/chosen": -252.9739990234375, + "logps/rejected": -177.40097045898438, + "loss": 0.0318, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 9.339091300964355, + "rewards/margins": 15.034673690795898, + "rewards/rejected": -5.695584297180176, + "step": 6210 + }, + { + "epoch": 0.68, + "learning_rate": 2.291523364512829e-06, + "logits/chosen": -0.8761197328567505, + "logits/rejected": 0.2756437361240387, + "logps/chosen": -170.13662719726562, + "logps/rejected": -184.7697296142578, + "loss": 0.017, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 7.966547966003418, + "rewards/margins": 13.834240913391113, + "rewards/rejected": -5.8676934242248535, + "step": 6220 + }, + { + "epoch": 0.68, + "learning_rate": 2.2770458430793275e-06, + "logits/chosen": -0.9093286395072937, + "logits/rejected": 0.1888500154018402, + "logps/chosen": -183.89134216308594, + "logps/rejected": -174.37554931640625, + "loss": 0.1535, + "rewards/accuracies": 0.9750000238418579, + "rewards/chosen": 7.462967872619629, + "rewards/margins": 12.259900093078613, + "rewards/rejected": -4.796932220458984, + "step": 6230 + }, + { + "epoch": 0.68, + "learning_rate": 2.2626007035878377e-06, + "logits/chosen": -0.7695046663284302, + "logits/rejected": 0.4005756378173828, + "logps/chosen": -169.727294921875, + "logps/rejected": -177.86964416503906, + "loss": 0.0248, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 7.759803771972656, + "rewards/margins": 13.199377059936523, + "rewards/rejected": -5.439573287963867, + "step": 6240 + }, + { + "epoch": 0.69, + "learning_rate": 2.2481881178229907e-06, + "logits/chosen": -1.1645220518112183, + "logits/rejected": -0.06789357960224152, + "logps/chosen": -195.56581115722656, + "logps/rejected": -181.91342163085938, + "loss": 0.0477, + "rewards/accuracies": 0.9750000238418579, + "rewards/chosen": 8.9083251953125, + "rewards/margins": 14.542688369750977, + "rewards/rejected": -5.63436222076416, + "step": 6250 + }, + { + "epoch": 0.69, + "learning_rate": 2.2338082571822822e-06, + "logits/chosen": -1.1157128810882568, + "logits/rejected": 0.10132601112127304, + "logps/chosen": -198.65194702148438, + "logps/rejected": -182.5029754638672, + "loss": 0.0208, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 8.416769027709961, + "rewards/margins": 14.055898666381836, + "rewards/rejected": -5.639128684997559, + "step": 6260 + }, + { + "epoch": 0.69, + "learning_rate": 2.2194612926740285e-06, + "logits/chosen": -0.9742426872253418, + "logits/rejected": -0.08283393085002899, + "logps/chosen": -222.7864227294922, + "logps/rejected": -181.38095092773438, + "loss": 0.0363, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 8.123876571655273, + "rewards/margins": 13.543279647827148, + "rewards/rejected": -5.419403076171875, + "step": 6270 + }, + { + "epoch": 0.69, + "learning_rate": 2.2051473949153386e-06, + "logits/chosen": -0.9475154876708984, + "logits/rejected": -0.014902275986969471, + "logps/chosen": -181.0790252685547, + "logps/rejected": -182.54327392578125, + "loss": 0.0358, + "rewards/accuracies": 0.9750000238418579, + "rewards/chosen": 8.143026351928711, + "rewards/margins": 13.33509635925293, + "rewards/rejected": -5.192070960998535, + "step": 6280 + }, + { + "epoch": 0.69, + "learning_rate": 2.1908667341300923e-06, + "logits/chosen": -0.9155664443969727, + "logits/rejected": 0.4932062029838562, + "logps/chosen": -213.444091796875, + "logps/rejected": -190.84402465820312, + "loss": 0.0393, + "rewards/accuracies": 0.9750000238418579, + "rewards/chosen": 9.100051879882812, + "rewards/margins": 15.362584114074707, + "rewards/rejected": -6.2625322341918945, + "step": 6290 + }, + { + "epoch": 0.69, + "learning_rate": 2.176619480146899e-06, + "logits/chosen": -1.0913949012756348, + "logits/rejected": -0.1356991082429886, + "logps/chosen": -231.8797149658203, + "logps/rejected": -190.54588317871094, + "loss": 0.0095, + "rewards/accuracies": 1.0, + "rewards/chosen": 8.580520629882812, + "rewards/margins": 14.067777633666992, + "rewards/rejected": -5.487257480621338, + "step": 6300 + }, + { + "epoch": 0.69, + "learning_rate": 2.1624058023970945e-06, + "logits/chosen": -1.1264455318450928, + "logits/rejected": -0.016152923926711082, + "logps/chosen": -225.0396728515625, + "logps/rejected": -187.33706665039062, + "loss": 0.0196, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 9.063858985900879, + "rewards/margins": 14.70002269744873, + "rewards/rejected": -5.636163234710693, + "step": 6310 + }, + { + "epoch": 0.69, + "learning_rate": 2.1482258699127133e-06, + "logits/chosen": -1.0974410772323608, + "logits/rejected": -0.16540241241455078, + "logps/chosen": -238.25656127929688, + "logps/rejected": -183.71368408203125, + "loss": 0.0364, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 8.999847412109375, + "rewards/margins": 14.426419258117676, + "rewards/rejected": -5.426571846008301, + "step": 6320 + }, + { + "epoch": 0.69, + "learning_rate": 2.134079851324491e-06, + "logits/chosen": -1.0645891427993774, + "logits/rejected": 0.1286853700876236, + "logps/chosen": -192.16653442382812, + "logps/rejected": -189.41334533691406, + "loss": 0.0335, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 8.62096118927002, + "rewards/margins": 14.932154655456543, + "rewards/rejected": -6.311192512512207, + "step": 6330 + }, + { + "epoch": 0.7, + "learning_rate": 2.1199679148598434e-06, + "logits/chosen": -1.2335244417190552, + "logits/rejected": -0.30533185601234436, + "logps/chosen": -240.8777313232422, + "logps/rejected": -181.75633239746094, + "loss": 0.0316, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 8.351324081420898, + "rewards/margins": 13.978303909301758, + "rewards/rejected": -5.626978874206543, + "step": 6340 + }, + { + "epoch": 0.7, + "learning_rate": 2.105890228340882e-06, + "logits/chosen": -0.9134172201156616, + "logits/rejected": -0.0064248861744999886, + "logps/chosen": -187.78872680664062, + "logps/rejected": -186.11135864257812, + "loss": 0.0579, + "rewards/accuracies": 0.9750000238418579, + "rewards/chosen": 8.586186408996582, + "rewards/margins": 13.922548294067383, + "rewards/rejected": -5.336360454559326, + "step": 6350 + }, + { + "epoch": 0.7, + "learning_rate": 2.091846959182408e-06, + "logits/chosen": -1.2565858364105225, + "logits/rejected": -0.30412963032722473, + "logps/chosen": -221.78042602539062, + "logps/rejected": -177.22105407714844, + "loss": 0.0246, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 7.608975410461426, + "rewards/margins": 13.572161674499512, + "rewards/rejected": -5.963186264038086, + "step": 6360 + }, + { + "epoch": 0.7, + "learning_rate": 2.077838274389924e-06, + "logits/chosen": -1.1190383434295654, + "logits/rejected": -0.06861548125743866, + "logps/chosen": -189.80783081054688, + "logps/rejected": -179.97000122070312, + "loss": 0.0008, + "rewards/accuracies": 1.0, + "rewards/chosen": 7.989137172698975, + "rewards/margins": 13.413408279418945, + "rewards/rejected": -5.424272060394287, + "step": 6370 + }, + { + "epoch": 0.7, + "learning_rate": 2.0638643405576468e-06, + "logits/chosen": -0.7638221383094788, + "logits/rejected": 0.1574622094631195, + "logps/chosen": -211.34158325195312, + "logps/rejected": -184.52516174316406, + "loss": 0.164, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": 7.5503644943237305, + "rewards/margins": 13.215766906738281, + "rewards/rejected": -5.665403366088867, + "step": 6380 + }, + { + "epoch": 0.7, + "learning_rate": 2.0499253238665284e-06, + "logits/chosen": -0.8570092916488647, + "logits/rejected": 0.23685041069984436, + "logps/chosen": -198.95375061035156, + "logps/rejected": -180.033447265625, + "loss": 0.0894, + "rewards/accuracies": 0.9750000238418579, + "rewards/chosen": 8.405570030212402, + "rewards/margins": 14.031819343566895, + "rewards/rejected": -5.626248836517334, + "step": 6390 + }, + { + "epoch": 0.7, + "learning_rate": 2.0360213900822795e-06, + "logits/chosen": -1.176298975944519, + "logits/rejected": -0.17218396067619324, + "logps/chosen": -239.80001831054688, + "logps/rejected": -182.78622436523438, + "loss": 0.0585, + "rewards/accuracies": 0.9624999761581421, + "rewards/chosen": 8.550104141235352, + "rewards/margins": 14.42521858215332, + "rewards/rejected": -5.875112533569336, + "step": 6400 + }, + { + "epoch": 0.7, + "learning_rate": 2.0221527045534e-06, + "logits/chosen": -0.9070030450820923, + "logits/rejected": 0.015762049704790115, + "logps/chosen": -175.78953552246094, + "logps/rejected": -177.07545471191406, + "loss": 0.0133, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 7.764042854309082, + "rewards/margins": 13.49589729309082, + "rewards/rejected": -5.731854438781738, + "step": 6410 + }, + { + "epoch": 0.7, + "learning_rate": 2.008319432209205e-06, + "logits/chosen": -0.9116196632385254, + "logits/rejected": 0.35185912251472473, + "logps/chosen": -179.34474182128906, + "logps/rejected": -188.8530731201172, + "loss": 0.0332, + "rewards/accuracies": 0.9750000238418579, + "rewards/chosen": 8.028730392456055, + "rewards/margins": 14.277336120605469, + "rewards/rejected": -6.248604774475098, + "step": 6420 + }, + { + "epoch": 0.71, + "learning_rate": 1.994521737557869e-06, + "logits/chosen": -0.8864614367485046, + "logits/rejected": -0.0142240971326828, + "logps/chosen": -203.92141723632812, + "logps/rejected": -179.3285369873047, + "loss": 0.0055, + "rewards/accuracies": 1.0, + "rewards/chosen": 8.493464469909668, + "rewards/margins": 14.395231246948242, + "rewards/rejected": -5.901765823364258, + "step": 6430 + }, + { + "epoch": 0.71, + "learning_rate": 1.9807597846844737e-06, + "logits/chosen": -0.7278019189834595, + "logits/rejected": 0.16882732510566711, + "logps/chosen": -198.24404907226562, + "logps/rejected": -179.56248474121094, + "loss": 0.0031, + "rewards/accuracies": 1.0, + "rewards/chosen": 9.059274673461914, + "rewards/margins": 14.529436111450195, + "rewards/rejected": -5.470160007476807, + "step": 6440 + }, + { + "epoch": 0.71, + "learning_rate": 1.9670337372490504e-06, + "logits/chosen": -0.8978595733642578, + "logits/rejected": 0.17058368027210236, + "logps/chosen": -197.95608520507812, + "logps/rejected": -188.17044067382812, + "loss": 0.0015, + "rewards/accuracies": 1.0, + "rewards/chosen": 8.769253730773926, + "rewards/margins": 14.963911056518555, + "rewards/rejected": -6.194657802581787, + "step": 6450 + }, + { + "epoch": 0.71, + "learning_rate": 1.9533437584846317e-06, + "logits/chosen": -0.8699809908866882, + "logits/rejected": 0.19820070266723633, + "logps/chosen": -213.407958984375, + "logps/rejected": -192.2376708984375, + "loss": 0.0501, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 8.489959716796875, + "rewards/margins": 14.298494338989258, + "rewards/rejected": -5.808534622192383, + "step": 6460 + }, + { + "epoch": 0.71, + "learning_rate": 1.939690011195319e-06, + "logits/chosen": -0.8912061452865601, + "logits/rejected": 0.20376217365264893, + "logps/chosen": -244.88687133789062, + "logps/rejected": -177.68331909179688, + "loss": 0.3169, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": 8.518463134765625, + "rewards/margins": 14.28369426727295, + "rewards/rejected": -5.765230655670166, + "step": 6470 + }, + { + "epoch": 0.71, + "learning_rate": 1.9260726577543378e-06, + "logits/chosen": -1.0426092147827148, + "logits/rejected": 0.06061286851763725, + "logps/chosen": -208.2215118408203, + "logps/rejected": -189.3504180908203, + "loss": 0.0283, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 8.105783462524414, + "rewards/margins": 14.031970024108887, + "rewards/rejected": -5.926186561584473, + "step": 6480 + }, + { + "epoch": 0.71, + "learning_rate": 1.9124918601021124e-06, + "logits/chosen": -1.209740161895752, + "logits/rejected": -0.06956999003887177, + "logps/chosen": -242.01235961914062, + "logps/rejected": -201.4385223388672, + "loss": 0.0153, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 9.170475959777832, + "rewards/margins": 16.08095359802246, + "rewards/rejected": -6.910479545593262, + "step": 6490 + }, + { + "epoch": 0.71, + "learning_rate": 1.8989477797443363e-06, + "logits/chosen": -0.8991093635559082, + "logits/rejected": 0.17424564063549042, + "logps/chosen": -199.75894165039062, + "logps/rejected": -199.26998901367188, + "loss": 0.0058, + "rewards/accuracies": 1.0, + "rewards/chosen": 8.661945343017578, + "rewards/margins": 14.76745891571045, + "rewards/rejected": -6.105513572692871, + "step": 6500 + }, + { + "epoch": 0.71, + "learning_rate": 1.8854405777500556e-06, + "logits/chosen": -0.8542581796646118, + "logits/rejected": 0.23183588683605194, + "logps/chosen": -196.17837524414062, + "logps/rejected": -197.6348419189453, + "loss": 0.1439, + "rewards/accuracies": 0.9750000238418579, + "rewards/chosen": 8.242921829223633, + "rewards/margins": 14.213079452514648, + "rewards/rejected": -5.970158100128174, + "step": 6510 + }, + { + "epoch": 0.72, + "learning_rate": 1.8719704147497492e-06, + "logits/chosen": -1.002037763595581, + "logits/rejected": 0.15966366231441498, + "logps/chosen": -218.00228881835938, + "logps/rejected": -184.96510314941406, + "loss": 0.0249, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 8.593465805053711, + "rewards/margins": 14.457681655883789, + "rewards/rejected": -5.8642168045043945, + "step": 6520 + }, + { + "epoch": 0.72, + "learning_rate": 1.8585374509334193e-06, + "logits/chosen": -0.8671720623970032, + "logits/rejected": 0.2303408682346344, + "logps/chosen": -190.29501342773438, + "logps/rejected": -184.4980926513672, + "loss": 0.0242, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 8.49260139465332, + "rewards/margins": 14.452038764953613, + "rewards/rejected": -5.959437370300293, + "step": 6530 + }, + { + "epoch": 0.72, + "learning_rate": 1.845141846048691e-06, + "logits/chosen": -0.8627886772155762, + "logits/rejected": 0.2103911191225052, + "logps/chosen": -228.12161254882812, + "logps/rejected": -186.61599731445312, + "loss": 0.0811, + "rewards/accuracies": 0.9750000238418579, + "rewards/chosen": 8.449845314025879, + "rewards/margins": 14.510551452636719, + "rewards/rejected": -6.060705184936523, + "step": 6540 + }, + { + "epoch": 0.72, + "learning_rate": 1.8317837593989085e-06, + "logits/chosen": -0.7119170427322388, + "logits/rejected": 0.6620062589645386, + "logps/chosen": -176.67666625976562, + "logps/rejected": -193.2861328125, + "loss": 0.0012, + "rewards/accuracies": 1.0, + "rewards/chosen": 7.72995662689209, + "rewards/margins": 14.109159469604492, + "rewards/rejected": -6.379203796386719, + "step": 6550 + }, + { + "epoch": 0.72, + "learning_rate": 1.8184633498412396e-06, + "logits/chosen": -1.1274131536483765, + "logits/rejected": 0.20179924368858337, + "logps/chosen": -218.48245239257812, + "logps/rejected": -199.57827758789062, + "loss": 0.0012, + "rewards/accuracies": 1.0, + "rewards/chosen": 8.991405487060547, + "rewards/margins": 15.496664047241211, + "rewards/rejected": -6.505259037017822, + "step": 6560 + }, + { + "epoch": 0.72, + "learning_rate": 1.8051807757847867e-06, + "logits/chosen": -0.8570119738578796, + "logits/rejected": 0.3766466975212097, + "logps/chosen": -184.0287322998047, + "logps/rejected": -182.9501190185547, + "loss": 0.1758, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": 8.116950035095215, + "rewards/margins": 14.321813583374023, + "rewards/rejected": -6.204864501953125, + "step": 6570 + }, + { + "epoch": 0.72, + "learning_rate": 1.7919361951887082e-06, + "logits/chosen": -0.8077383041381836, + "logits/rejected": 0.2907382845878601, + "logps/chosen": -205.3682403564453, + "logps/rejected": -183.94752502441406, + "loss": 0.0072, + "rewards/accuracies": 1.0, + "rewards/chosen": 8.478684425354004, + "rewards/margins": 14.334627151489258, + "rewards/rejected": -5.855942726135254, + "step": 6580 + }, + { + "epoch": 0.72, + "learning_rate": 1.778729765560337e-06, + "logits/chosen": -0.9524921178817749, + "logits/rejected": 0.46900081634521484, + "logps/chosen": -178.93707275390625, + "logps/rejected": -193.08062744140625, + "loss": 0.0069, + "rewards/accuracies": 1.0, + "rewards/chosen": 7.566506862640381, + "rewards/margins": 14.188827514648438, + "rewards/rejected": -6.622320652008057, + "step": 6590 + }, + { + "epoch": 0.72, + "learning_rate": 1.7655616439533025e-06, + "logits/chosen": -0.8925657272338867, + "logits/rejected": 0.2815835177898407, + "logps/chosen": -179.80181884765625, + "logps/rejected": -187.9346160888672, + "loss": 0.017, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 6.903683662414551, + "rewards/margins": 12.874814987182617, + "rewards/rejected": -5.971131801605225, + "step": 6600 + }, + { + "epoch": 0.73, + "learning_rate": 1.7524319869656675e-06, + "logits/chosen": -1.040313720703125, + "logits/rejected": -0.015298599377274513, + "logps/chosen": -230.0985107421875, + "logps/rejected": -178.57431030273438, + "loss": 0.0121, + "rewards/accuracies": 1.0, + "rewards/chosen": 8.429296493530273, + "rewards/margins": 14.808316230773926, + "rewards/rejected": -6.379019737243652, + "step": 6610 + }, + { + "epoch": 0.73, + "learning_rate": 1.7393409507380688e-06, + "logits/chosen": -0.3277572989463806, + "logits/rejected": 0.6537003517150879, + "logps/chosen": -146.78274536132812, + "logps/rejected": -180.0274200439453, + "loss": 0.0077, + "rewards/accuracies": 1.0, + "rewards/chosen": 6.557616233825684, + "rewards/margins": 12.66218090057373, + "rewards/rejected": -6.104563236236572, + "step": 6620 + }, + { + "epoch": 0.73, + "learning_rate": 1.7262886909518566e-06, + "logits/chosen": -0.893263041973114, + "logits/rejected": 0.41579684615135193, + "logps/chosen": -208.4541473388672, + "logps/rejected": -199.4329376220703, + "loss": 0.057, + "rewards/accuracies": 0.9750000238418579, + "rewards/chosen": 8.48161506652832, + "rewards/margins": 15.338422775268555, + "rewards/rejected": -6.856808662414551, + "step": 6630 + }, + { + "epoch": 0.73, + "learning_rate": 1.7132753628272403e-06, + "logits/chosen": -0.734578549861908, + "logits/rejected": 0.21192821860313416, + "logps/chosen": -200.13687133789062, + "logps/rejected": -193.1783905029297, + "loss": 0.0022, + "rewards/accuracies": 1.0, + "rewards/chosen": 8.08255672454834, + "rewards/margins": 14.069941520690918, + "rewards/rejected": -5.987383842468262, + "step": 6640 + }, + { + "epoch": 0.73, + "learning_rate": 1.7003011211214471e-06, + "logits/chosen": -0.8599523305892944, + "logits/rejected": 0.32559242844581604, + "logps/chosen": -197.9250946044922, + "logps/rejected": -200.5045166015625, + "loss": 0.01, + "rewards/accuracies": 1.0, + "rewards/chosen": 8.308802604675293, + "rewards/margins": 14.8523530960083, + "rewards/rejected": -6.543548583984375, + "step": 6650 + }, + { + "epoch": 0.73, + "learning_rate": 1.6873661201268833e-06, + "logits/chosen": -0.8828017115592957, + "logits/rejected": 0.3141935467720032, + "logps/chosen": -208.2950439453125, + "logps/rejected": -195.88217163085938, + "loss": 0.0054, + "rewards/accuracies": 1.0, + "rewards/chosen": 8.983816146850586, + "rewards/margins": 15.06860637664795, + "rewards/rejected": -6.084790229797363, + "step": 6660 + }, + { + "epoch": 0.73, + "learning_rate": 1.6744705136692906e-06, + "logits/chosen": -0.7972429394721985, + "logits/rejected": 0.4401327073574066, + "logps/chosen": -197.84738159179688, + "logps/rejected": -190.0059814453125, + "loss": 0.0874, + "rewards/accuracies": 0.9624999761581421, + "rewards/chosen": 7.6676788330078125, + "rewards/margins": 14.006418228149414, + "rewards/rejected": -6.33873987197876, + "step": 6670 + }, + { + "epoch": 0.73, + "learning_rate": 1.6616144551059305e-06, + "logits/chosen": -1.0274611711502075, + "logits/rejected": 0.24104909598827362, + "logps/chosen": -193.85702514648438, + "logps/rejected": -192.9866180419922, + "loss": 0.0028, + "rewards/accuracies": 1.0, + "rewards/chosen": 8.414044380187988, + "rewards/margins": 14.701065063476562, + "rewards/rejected": -6.287020683288574, + "step": 6680 + }, + { + "epoch": 0.73, + "learning_rate": 1.6487980973237434e-06, + "logits/chosen": -1.009594202041626, + "logits/rejected": 0.1880350410938263, + "logps/chosen": -216.4499053955078, + "logps/rejected": -197.25399780273438, + "loss": 0.0552, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 8.331182479858398, + "rewards/margins": 14.785545349121094, + "rewards/rejected": -6.4543633460998535, + "step": 6690 + }, + { + "epoch": 0.74, + "learning_rate": 1.6360215927375483e-06, + "logits/chosen": -1.0727919340133667, + "logits/rejected": -0.01006121002137661, + "logps/chosen": -227.48324584960938, + "logps/rejected": -185.60299682617188, + "loss": 0.0042, + "rewards/accuracies": 1.0, + "rewards/chosen": 9.03674602508545, + "rewards/margins": 14.677348136901855, + "rewards/rejected": -5.64060115814209, + "step": 6700 + }, + { + "epoch": 0.74, + "learning_rate": 1.6232850932882155e-06, + "logits/chosen": -0.9123764038085938, + "logits/rejected": 0.4361811578273773, + "logps/chosen": -189.03338623046875, + "logps/rejected": -188.27586364746094, + "loss": 0.041, + "rewards/accuracies": 0.9750000238418579, + "rewards/chosen": 7.3466386795043945, + "rewards/margins": 13.262483596801758, + "rewards/rejected": -5.915842056274414, + "step": 6710 + }, + { + "epoch": 0.74, + "learning_rate": 1.6105887504408679e-06, + "logits/chosen": -0.6901660561561584, + "logits/rejected": 0.48877158761024475, + "logps/chosen": -180.28213500976562, + "logps/rejected": -190.59561157226562, + "loss": 0.0003, + "rewards/accuracies": 1.0, + "rewards/chosen": 8.748319625854492, + "rewards/margins": 15.327142715454102, + "rewards/rejected": -6.578821659088135, + "step": 6720 + }, + { + "epoch": 0.74, + "learning_rate": 1.5979327151830815e-06, + "logits/chosen": -0.7546707987785339, + "logits/rejected": 0.34022727608680725, + "logps/chosen": -186.39620971679688, + "logps/rejected": -187.02423095703125, + "loss": 0.0016, + "rewards/accuracies": 1.0, + "rewards/chosen": 8.231672286987305, + "rewards/margins": 14.30876636505127, + "rewards/rejected": -6.077094078063965, + "step": 6730 + }, + { + "epoch": 0.74, + "learning_rate": 1.5853171380230791e-06, + "logits/chosen": -0.9439939260482788, + "logits/rejected": 0.13540348410606384, + "logps/chosen": -211.34622192382812, + "logps/rejected": -197.50660705566406, + "loss": 0.0001, + "rewards/accuracies": 1.0, + "rewards/chosen": 8.73457145690918, + "rewards/margins": 15.160000801086426, + "rewards/rejected": -6.425428867340088, + "step": 6740 + }, + { + "epoch": 0.74, + "learning_rate": 1.5727421689879518e-06, + "logits/chosen": -1.1217578649520874, + "logits/rejected": 0.18595805764198303, + "logps/chosen": -209.4240264892578, + "logps/rejected": -191.04019165039062, + "loss": 0.0106, + "rewards/accuracies": 1.0, + "rewards/chosen": 8.509339332580566, + "rewards/margins": 14.790704727172852, + "rewards/rejected": -6.281365394592285, + "step": 6750 + }, + { + "epoch": 0.74, + "learning_rate": 1.5602079576218665e-06, + "logits/chosen": -1.176283359527588, + "logits/rejected": -0.15890005230903625, + "logps/chosen": -219.97219848632812, + "logps/rejected": -176.8172607421875, + "loss": 0.0062, + "rewards/accuracies": 1.0, + "rewards/chosen": 8.975140571594238, + "rewards/margins": 14.7715425491333, + "rewards/rejected": -5.796403884887695, + "step": 6760 + }, + { + "epoch": 0.74, + "learning_rate": 1.5477146529842961e-06, + "logits/chosen": -1.0445820093154907, + "logits/rejected": 0.1409655064344406, + "logps/chosen": -212.9378204345703, + "logps/rejected": -189.13320922851562, + "loss": 0.0237, + "rewards/accuracies": 0.9750000238418579, + "rewards/chosen": 8.568227767944336, + "rewards/margins": 14.402395248413086, + "rewards/rejected": -5.834168910980225, + "step": 6770 + }, + { + "epoch": 0.74, + "learning_rate": 1.5352624036482422e-06, + "logits/chosen": -1.0217076539993286, + "logits/rejected": 0.04597047343850136, + "logps/chosen": -230.456298828125, + "logps/rejected": -185.31573486328125, + "loss": 0.0695, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 8.913262367248535, + "rewards/margins": 15.059542655944824, + "rewards/rejected": -6.146279335021973, + "step": 6780 + }, + { + "epoch": 0.75, + "learning_rate": 1.5228513576984633e-06, + "logits/chosen": -1.0757765769958496, + "logits/rejected": 0.13445916771888733, + "logps/chosen": -255.79678344726562, + "logps/rejected": -185.31939697265625, + "loss": 0.0213, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 9.053261756896973, + "rewards/margins": 15.019460678100586, + "rewards/rejected": -5.966198921203613, + "step": 6790 + }, + { + "epoch": 0.75, + "learning_rate": 1.5104816627297203e-06, + "logits/chosen": -1.0749822854995728, + "logits/rejected": 0.2128952443599701, + "logps/chosen": -224.16793823242188, + "logps/rejected": -191.6697235107422, + "loss": 0.0004, + "rewards/accuracies": 1.0, + "rewards/chosen": 8.748997688293457, + "rewards/margins": 14.85130500793457, + "rewards/rejected": -6.102306365966797, + "step": 6800 + }, + { + "epoch": 0.75, + "learning_rate": 1.498153465845022e-06, + "logits/chosen": -1.0690027475357056, + "logits/rejected": 0.10766322910785675, + "logps/chosen": -201.118896484375, + "logps/rejected": -176.67568969726562, + "loss": 0.1024, + "rewards/accuracies": 0.9750000238418579, + "rewards/chosen": 8.735494613647461, + "rewards/margins": 13.862030029296875, + "rewards/rejected": -5.126534461975098, + "step": 6810 + }, + { + "epoch": 0.75, + "learning_rate": 1.4858669136538728e-06, + "logits/chosen": -0.7696990966796875, + "logits/rejected": 0.38921505212783813, + "logps/chosen": -201.67837524414062, + "logps/rejected": -187.2930450439453, + "loss": 0.0843, + "rewards/accuracies": 0.9624999761581421, + "rewards/chosen": 7.945582389831543, + "rewards/margins": 13.929214477539062, + "rewards/rejected": -5.983630657196045, + "step": 6820 + }, + { + "epoch": 0.75, + "learning_rate": 1.4736221522705263e-06, + "logits/chosen": -0.7734932899475098, + "logits/rejected": 0.3267975449562073, + "logps/chosen": -160.51866149902344, + "logps/rejected": -178.46327209472656, + "loss": 0.0383, + "rewards/accuracies": 0.9624999761581421, + "rewards/chosen": 7.400290489196777, + "rewards/margins": 13.402809143066406, + "rewards/rejected": -6.002518653869629, + "step": 6830 + }, + { + "epoch": 0.75, + "learning_rate": 1.4614193273122562e-06, + "logits/chosen": -1.075041651725769, + "logits/rejected": 0.04435647279024124, + "logps/chosen": -246.6208953857422, + "logps/rejected": -187.79441833496094, + "loss": 0.0019, + "rewards/accuracies": 1.0, + "rewards/chosen": 8.809691429138184, + "rewards/margins": 14.558366775512695, + "rewards/rejected": -5.748675346374512, + "step": 6840 + }, + { + "epoch": 0.75, + "learning_rate": 1.4492585838976136e-06, + "logits/chosen": -0.9724672436714172, + "logits/rejected": 0.10981476306915283, + "logps/chosen": -184.01187133789062, + "logps/rejected": -185.1789093017578, + "loss": 0.0322, + "rewards/accuracies": 0.9750000238418579, + "rewards/chosen": 7.92758321762085, + "rewards/margins": 14.278829574584961, + "rewards/rejected": -6.351248264312744, + "step": 6850 + }, + { + "epoch": 0.75, + "learning_rate": 1.4371400666447128e-06, + "logits/chosen": -0.9087244868278503, + "logits/rejected": 0.25366267561912537, + "logps/chosen": -191.01016235351562, + "logps/rejected": -183.60171508789062, + "loss": 0.0682, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 7.3474836349487305, + "rewards/margins": 13.293353080749512, + "rewards/rejected": -5.945868015289307, + "step": 6860 + }, + { + "epoch": 0.75, + "learning_rate": 1.4250639196695e-06, + "logits/chosen": -0.9596236944198608, + "logits/rejected": 0.4607522487640381, + "logps/chosen": -210.98007202148438, + "logps/rejected": -208.0542449951172, + "loss": 0.0031, + "rewards/accuracies": 1.0, + "rewards/chosen": 7.990859031677246, + "rewards/margins": 14.809690475463867, + "rewards/rejected": -6.8188323974609375, + "step": 6870 + }, + { + "epoch": 0.76, + "learning_rate": 1.4130302865840517e-06, + "logits/chosen": -0.946962833404541, + "logits/rejected": 0.37561115622520447, + "logps/chosen": -187.0880584716797, + "logps/rejected": -184.96945190429688, + "loss": 0.0447, + "rewards/accuracies": 0.9750000238418579, + "rewards/chosen": 7.964089870452881, + "rewards/margins": 13.727540969848633, + "rewards/rejected": -5.763451099395752, + "step": 6880 + }, + { + "epoch": 0.76, + "learning_rate": 1.401039310494855e-06, + "logits/chosen": -1.0502557754516602, + "logits/rejected": 0.11881722509860992, + "logps/chosen": -226.1573944091797, + "logps/rejected": -190.77772521972656, + "loss": 0.043, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 8.492795944213867, + "rewards/margins": 14.918710708618164, + "rewards/rejected": -6.4259138107299805, + "step": 6890 + }, + { + "epoch": 0.76, + "learning_rate": 1.3890911340011115e-06, + "logits/chosen": -1.242148756980896, + "logits/rejected": 0.06506754457950592, + "logps/chosen": -267.4064636230469, + "logps/rejected": -183.22940063476562, + "loss": 0.0068, + "rewards/accuracies": 1.0, + "rewards/chosen": 9.72371768951416, + "rewards/margins": 16.367080688476562, + "rewards/rejected": -6.643364906311035, + "step": 6900 + }, + { + "epoch": 0.76, + "learning_rate": 1.3771858991930432e-06, + "logits/chosen": -0.809552013874054, + "logits/rejected": 0.17292878031730652, + "logps/chosen": -188.53353881835938, + "logps/rejected": -172.88441467285156, + "loss": 0.0027, + "rewards/accuracies": 1.0, + "rewards/chosen": 8.176888465881348, + "rewards/margins": 13.844263076782227, + "rewards/rejected": -5.6673760414123535, + "step": 6910 + }, + { + "epoch": 0.76, + "learning_rate": 1.365323747650202e-06, + "logits/chosen": -0.8463281393051147, + "logits/rejected": 0.2564229369163513, + "logps/chosen": -208.6510467529297, + "logps/rejected": -185.48519897460938, + "loss": 0.0378, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 8.73391056060791, + "rewards/margins": 14.568878173828125, + "rewards/rejected": -5.834967136383057, + "step": 6920 + }, + { + "epoch": 0.76, + "learning_rate": 1.3535048204397805e-06, + "logits/chosen": -0.9447125196456909, + "logits/rejected": 0.23670455813407898, + "logps/chosen": -190.20816040039062, + "logps/rejected": -189.29444885253906, + "loss": 0.0045, + "rewards/accuracies": 1.0, + "rewards/chosen": 8.338282585144043, + "rewards/margins": 13.94194221496582, + "rewards/rejected": -5.603659152984619, + "step": 6930 + }, + { + "epoch": 0.76, + "learning_rate": 1.3417292581149388e-06, + "logits/chosen": -0.7695263624191284, + "logits/rejected": 0.4169999957084656, + "logps/chosen": -196.6933135986328, + "logps/rejected": -197.24400329589844, + "loss": 0.0009, + "rewards/accuracies": 1.0, + "rewards/chosen": 8.629964828491211, + "rewards/margins": 15.603785514831543, + "rewards/rejected": -6.973820686340332, + "step": 6940 + }, + { + "epoch": 0.76, + "learning_rate": 1.3299972007131357e-06, + "logits/chosen": -0.9543238878250122, + "logits/rejected": 0.5572659373283386, + "logps/chosen": -177.01568603515625, + "logps/rejected": -189.99713134765625, + "loss": 0.0091, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 8.215497970581055, + "rewards/margins": 14.301098823547363, + "rewards/rejected": -6.085599899291992, + "step": 6950 + }, + { + "epoch": 0.76, + "learning_rate": 1.3183087877544604e-06, + "logits/chosen": -0.9377700090408325, + "logits/rejected": 0.34693416953086853, + "logps/chosen": -207.6963653564453, + "logps/rejected": -188.38156127929688, + "loss": 0.0435, + "rewards/accuracies": 0.9750000238418579, + "rewards/chosen": 8.22341251373291, + "rewards/margins": 14.776949882507324, + "rewards/rejected": -6.553537845611572, + "step": 6960 + }, + { + "epoch": 0.77, + "learning_rate": 1.3066641582399696e-06, + "logits/chosen": -0.881015419960022, + "logits/rejected": 0.2946591377258301, + "logps/chosen": -210.3145751953125, + "logps/rejected": -186.30406188964844, + "loss": 0.0153, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 8.363940238952637, + "rewards/margins": 14.484280586242676, + "rewards/rejected": -6.120340347290039, + "step": 6970 + }, + { + "epoch": 0.77, + "learning_rate": 1.2950634506500392e-06, + "logits/chosen": -0.9658054113388062, + "logits/rejected": 0.257289856672287, + "logps/chosen": -197.36004638671875, + "logps/rejected": -191.48837280273438, + "loss": 0.0661, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 7.946740627288818, + "rewards/margins": 14.595216751098633, + "rewards/rejected": -6.64847469329834, + "step": 6980 + }, + { + "epoch": 0.77, + "learning_rate": 1.2835068029427188e-06, + "logits/chosen": -0.8837069272994995, + "logits/rejected": 0.47144681215286255, + "logps/chosen": -198.55661010742188, + "logps/rejected": -192.7499237060547, + "loss": 0.0006, + "rewards/accuracies": 1.0, + "rewards/chosen": 8.174417495727539, + "rewards/margins": 14.648134231567383, + "rewards/rejected": -6.473714351654053, + "step": 6990 + }, + { + "epoch": 0.77, + "learning_rate": 1.2719943525520884e-06, + "logits/chosen": -1.007331132888794, + "logits/rejected": 0.15776190161705017, + "logps/chosen": -219.5244903564453, + "logps/rejected": -190.54928588867188, + "loss": 0.1765, + "rewards/accuracies": 0.9375, + "rewards/chosen": 8.816061019897461, + "rewards/margins": 14.59998607635498, + "rewards/rejected": -5.7839250564575195, + "step": 7000 + } + ], + "logging_steps": 10, + "max_steps": 9110, + "num_input_tokens_seen": 0, + "num_train_epochs": 1, + "save_steps": 1000, + "total_flos": 8.755170313654764e+18, + "train_batch_size": 2, + "trial_name": null, + "trial_params": null +}