| { |
| "best_metric": null, |
| "best_model_checkpoint": null, |
| "epoch": 0.012329886102677127, |
| "eval_steps": 500, |
| "global_step": 400, |
| "is_hyper_param_search": false, |
| "is_local_process_zero": true, |
| "is_world_process_zero": true, |
| "log_history": [ |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 3.082471525669282e-05, |
| "grad_norm": 0.4096410633951961, |
| "kl": 0.0, |
| "learning_rate": 1e-05, |
| "loss": 0.0, |
| "reward": 0.27293290942907333, |
| "reward_std": 0.0449131247587502, |
| "rewards/clip_reward": 0.27293290942907333, |
| "step": 1 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 6.164943051338563e-05, |
| "grad_norm": 2.7610875519018947, |
| "kl": 0.048065185546875, |
| "learning_rate": 1e-05, |
| "loss": 0.0019, |
| "reward": 0.27704446762800217, |
| "reward_std": 0.04381310846656561, |
| "rewards/clip_reward": 0.27704446762800217, |
| "step": 2 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 9.247414577007844e-05, |
| "grad_norm": 2.6854813853406663, |
| "kl": 0.08209228515625, |
| "learning_rate": 1e-05, |
| "loss": 0.0033, |
| "reward": 0.24470487236976624, |
| "reward_std": 0.039878456853330135, |
| "rewards/clip_reward": 0.24470487236976624, |
| "step": 3 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.00012329886102677127, |
| "grad_norm": 1.844139759481107, |
| "kl": 0.094390869140625, |
| "learning_rate": 1e-05, |
| "loss": 0.0038, |
| "reward": 0.2730746790766716, |
| "reward_std": 0.03929354250431061, |
| "rewards/clip_reward": 0.2730746790766716, |
| "step": 4 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.00015412357628346408, |
| "grad_norm": 2.648057613593892, |
| "kl": 0.149658203125, |
| "learning_rate": 1e-05, |
| "loss": 0.006, |
| "reward": 0.22251639142632484, |
| "reward_std": 0.035196226090192795, |
| "rewards/clip_reward": 0.22251639142632484, |
| "step": 5 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.0001849482915401569, |
| "grad_norm": 2320.3210476556847, |
| "kl": 35.0732421875, |
| "learning_rate": 1e-05, |
| "loss": 1.408, |
| "reward": 0.26333311200141907, |
| "reward_std": 0.043474785052239895, |
| "rewards/clip_reward": 0.26333311200141907, |
| "step": 6 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.00021577300679684973, |
| "grad_norm": 7.940267210806191, |
| "kl": 0.358489990234375, |
| "learning_rate": 1e-05, |
| "loss": 0.0144, |
| "reward": 0.26793035864830017, |
| "reward_std": 0.03859096858650446, |
| "rewards/clip_reward": 0.26793035864830017, |
| "step": 7 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.00024659772205354254, |
| "grad_norm": 1.2528619108757855, |
| "kl": 0.093505859375, |
| "learning_rate": 1e-05, |
| "loss": 0.0037, |
| "reward": 0.25467605516314507, |
| "reward_std": 0.0469845924526453, |
| "rewards/clip_reward": 0.25467605516314507, |
| "step": 8 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.0002774224373102353, |
| "grad_norm": 0.3609296984986853, |
| "kl": 0.03167724609375, |
| "learning_rate": 1e-05, |
| "loss": 0.0013, |
| "reward": 0.2549675516784191, |
| "reward_std": 0.050234788097441196, |
| "rewards/clip_reward": 0.2549675516784191, |
| "step": 9 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.00030824715256692816, |
| "grad_norm": 0.340167108350015, |
| "kl": 0.026275634765625, |
| "learning_rate": 1e-05, |
| "loss": 0.0011, |
| "reward": 0.2656533867120743, |
| "reward_std": 0.041638208553195, |
| "rewards/clip_reward": 0.2656533867120743, |
| "step": 10 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.000339071867823621, |
| "grad_norm": 1.2337998822225458, |
| "kl": 0.038787841796875, |
| "learning_rate": 1e-05, |
| "loss": 0.0016, |
| "reward": 0.23507770523428917, |
| "reward_std": 0.03971440531313419, |
| "rewards/clip_reward": 0.23507770523428917, |
| "step": 11 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.0003698965830803138, |
| "grad_norm": 0.4192393740192319, |
| "kl": 0.027679443359375, |
| "learning_rate": 1e-05, |
| "loss": 0.0011, |
| "reward": 0.28236085921525955, |
| "reward_std": 0.047151658684015274, |
| "rewards/clip_reward": 0.28236085921525955, |
| "step": 12 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.0004007212983370066, |
| "grad_norm": 0.44693100447868084, |
| "kl": 0.04705810546875, |
| "learning_rate": 1e-05, |
| "loss": 0.0019, |
| "reward": 0.2532140128314495, |
| "reward_std": 0.045407955069094896, |
| "rewards/clip_reward": 0.2532140128314495, |
| "step": 13 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.00043154601359369945, |
| "grad_norm": 0.4438582117717643, |
| "kl": 0.031341552734375, |
| "learning_rate": 1e-05, |
| "loss": 0.0013, |
| "reward": 0.2620070278644562, |
| "reward_std": 0.044824035838246346, |
| "rewards/clip_reward": 0.2620070278644562, |
| "step": 14 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.00046237072885039224, |
| "grad_norm": 0.4582616987800618, |
| "kl": 0.029937744140625, |
| "learning_rate": 1e-05, |
| "loss": 0.0012, |
| "reward": 0.2698071673512459, |
| "reward_std": 0.044542488642036915, |
| "rewards/clip_reward": 0.2698071673512459, |
| "step": 15 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.0004931954441070851, |
| "grad_norm": 1.7444707877124468, |
| "kl": 0.171722412109375, |
| "learning_rate": 1e-05, |
| "loss": 0.0069, |
| "reward": 0.25084393844008446, |
| "reward_std": 0.03859854955226183, |
| "rewards/clip_reward": 0.25084393844008446, |
| "step": 16 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.0005240201593637779, |
| "grad_norm": 0.9397492604252154, |
| "kl": 0.03289794921875, |
| "learning_rate": 1e-05, |
| "loss": 0.0013, |
| "reward": 0.2502448856830597, |
| "reward_std": 0.046720145270228386, |
| "rewards/clip_reward": 0.2502448856830597, |
| "step": 17 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.0005548448746204706, |
| "grad_norm": 0.2795480658769616, |
| "kl": 0.02972412109375, |
| "learning_rate": 1e-05, |
| "loss": 0.0012, |
| "reward": 0.25642987713217735, |
| "reward_std": 0.040699112229049206, |
| "rewards/clip_reward": 0.25642987713217735, |
| "step": 18 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.0005856695898771635, |
| "grad_norm": 0.42906609124114997, |
| "kl": 0.030792236328125, |
| "learning_rate": 1e-05, |
| "loss": 0.0012, |
| "reward": 0.26084326952695847, |
| "reward_std": 0.046253882348537445, |
| "rewards/clip_reward": 0.26084326952695847, |
| "step": 19 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.0006164943051338563, |
| "grad_norm": 0.27464154300178234, |
| "kl": 0.034576416015625, |
| "learning_rate": 1e-05, |
| "loss": 0.0014, |
| "reward": 0.2642279714345932, |
| "reward_std": 0.04200063832104206, |
| "rewards/clip_reward": 0.2642279714345932, |
| "step": 20 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.0006473190203905491, |
| "grad_norm": 0.28947614961512397, |
| "kl": 0.030029296875, |
| "learning_rate": 1e-05, |
| "loss": 0.0012, |
| "reward": 0.25506409257650375, |
| "reward_std": 0.0439683748409152, |
| "rewards/clip_reward": 0.25506409257650375, |
| "step": 21 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.000678143735647242, |
| "grad_norm": 0.3388326949584448, |
| "kl": 0.0302734375, |
| "learning_rate": 1e-05, |
| "loss": 0.0012, |
| "reward": 0.2455521523952484, |
| "reward_std": 0.04773388337343931, |
| "rewards/clip_reward": 0.2455521523952484, |
| "step": 22 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.0007089684509039348, |
| "grad_norm": 0.251274728050997, |
| "kl": 0.031219482421875, |
| "learning_rate": 1e-05, |
| "loss": 0.0012, |
| "reward": 0.26682381331920624, |
| "reward_std": 0.045767911709845066, |
| "rewards/clip_reward": 0.26682381331920624, |
| "step": 23 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.0007397931661606276, |
| "grad_norm": 0.3330664131200773, |
| "kl": 0.03167724609375, |
| "learning_rate": 1e-05, |
| "loss": 0.0013, |
| "reward": 0.27093350887298584, |
| "reward_std": 0.04017470218241215, |
| "rewards/clip_reward": 0.27093350887298584, |
| "step": 24 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.0007706178814173204, |
| "grad_norm": 0.2855775494556483, |
| "kl": 0.030181884765625, |
| "learning_rate": 1e-05, |
| "loss": 0.0012, |
| "reward": 0.2470133751630783, |
| "reward_std": 0.045277868397533894, |
| "rewards/clip_reward": 0.2470133751630783, |
| "step": 25 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.0008014425966740132, |
| "grad_norm": 0.23359940849678845, |
| "kl": 0.03143310546875, |
| "learning_rate": 1e-05, |
| "loss": 0.0013, |
| "reward": 0.2605663165450096, |
| "reward_std": 0.036497367545962334, |
| "rewards/clip_reward": 0.2605663165450096, |
| "step": 26 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.000832267311930706, |
| "grad_norm": 0.2565562945436733, |
| "kl": 0.031494140625, |
| "learning_rate": 1e-05, |
| "loss": 0.0013, |
| "reward": 0.25371479243040085, |
| "reward_std": 0.04583714855834842, |
| "rewards/clip_reward": 0.25371479243040085, |
| "step": 27 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.0008630920271873989, |
| "grad_norm": 0.37271054820762656, |
| "kl": 0.0306396484375, |
| "learning_rate": 1e-05, |
| "loss": 0.0012, |
| "reward": 0.2469511665403843, |
| "reward_std": 0.035832284949719906, |
| "rewards/clip_reward": 0.2469511665403843, |
| "step": 28 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.0008939167424440917, |
| "grad_norm": 0.24495908441078176, |
| "kl": 0.03009033203125, |
| "learning_rate": 1e-05, |
| "loss": 0.0012, |
| "reward": 0.2665953040122986, |
| "reward_std": 0.0440530339255929, |
| "rewards/clip_reward": 0.2665953040122986, |
| "step": 29 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.0009247414577007845, |
| "grad_norm": 0.27952316749339523, |
| "kl": 0.029510498046875, |
| "learning_rate": 1e-05, |
| "loss": 0.0012, |
| "reward": 0.2593040131032467, |
| "reward_std": 0.04028139542788267, |
| "rewards/clip_reward": 0.2593040131032467, |
| "step": 30 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.0009555661729574773, |
| "grad_norm": 0.29685109479945, |
| "kl": 0.03131103515625, |
| "learning_rate": 1e-05, |
| "loss": 0.0013, |
| "reward": 0.2597034126520157, |
| "reward_std": 0.045073311775922775, |
| "rewards/clip_reward": 0.2597034126520157, |
| "step": 31 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.0009863908882141701, |
| "grad_norm": 0.2788133739842884, |
| "kl": 0.029815673828125, |
| "learning_rate": 1e-05, |
| "loss": 0.0012, |
| "reward": 0.26862068474292755, |
| "reward_std": 0.04331609793007374, |
| "rewards/clip_reward": 0.26862068474292755, |
| "step": 32 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.001017215603470863, |
| "grad_norm": 0.2793158814370596, |
| "kl": 0.032684326171875, |
| "learning_rate": 1e-05, |
| "loss": 0.0013, |
| "reward": 0.2581031806766987, |
| "reward_std": 0.04781654942780733, |
| "rewards/clip_reward": 0.2581031806766987, |
| "step": 33 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.0010480403187275557, |
| "grad_norm": 0.2410712843439065, |
| "kl": 0.031158447265625, |
| "learning_rate": 1e-05, |
| "loss": 0.0012, |
| "reward": 0.25184522196650505, |
| "reward_std": 0.04620496183633804, |
| "rewards/clip_reward": 0.25184522196650505, |
| "step": 34 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.0010788650339842486, |
| "grad_norm": 0.6426585983726603, |
| "kl": 0.034698486328125, |
| "learning_rate": 1e-05, |
| "loss": 0.0014, |
| "reward": 0.26873790100216866, |
| "reward_std": 0.03604905540123582, |
| "rewards/clip_reward": 0.26873790100216866, |
| "step": 35 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.0011096897492409413, |
| "grad_norm": 0.30049643240824964, |
| "kl": 0.028167724609375, |
| "learning_rate": 1e-05, |
| "loss": 0.0011, |
| "reward": 0.28773902356624603, |
| "reward_std": 0.04903798084706068, |
| "rewards/clip_reward": 0.28773902356624603, |
| "step": 36 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.0011405144644976342, |
| "grad_norm": 0.286651911122606, |
| "kl": 0.03125, |
| "learning_rate": 1e-05, |
| "loss": 0.0013, |
| "reward": 0.2746337428689003, |
| "reward_std": 0.039260104298591614, |
| "rewards/clip_reward": 0.2746337428689003, |
| "step": 37 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.001171339179754327, |
| "grad_norm": 0.23526530529450262, |
| "kl": 0.03240966796875, |
| "learning_rate": 1e-05, |
| "loss": 0.0013, |
| "reward": 0.2732752785086632, |
| "reward_std": 0.03741883207112551, |
| "rewards/clip_reward": 0.2732752785086632, |
| "step": 38 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.0012021638950110197, |
| "grad_norm": 0.23423409222245298, |
| "kl": 0.0341796875, |
| "learning_rate": 1e-05, |
| "loss": 0.0014, |
| "reward": 0.24805432185530663, |
| "reward_std": 0.03564950963482261, |
| "rewards/clip_reward": 0.24805432185530663, |
| "step": 39 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.0012329886102677126, |
| "grad_norm": 0.24360760682907664, |
| "kl": 0.03375244140625, |
| "learning_rate": 1e-05, |
| "loss": 0.0014, |
| "reward": 0.2747276946902275, |
| "reward_std": 0.04160565137863159, |
| "rewards/clip_reward": 0.2747276946902275, |
| "step": 40 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.0012638133255244055, |
| "grad_norm": 0.24537882409018658, |
| "kl": 0.0321044921875, |
| "learning_rate": 1e-05, |
| "loss": 0.0013, |
| "reward": 0.2780498266220093, |
| "reward_std": 0.036946577951312065, |
| "rewards/clip_reward": 0.2780498266220093, |
| "step": 41 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.0012946380407810982, |
| "grad_norm": 0.40977788179371805, |
| "kl": 0.03350830078125, |
| "learning_rate": 1e-05, |
| "loss": 0.0013, |
| "reward": 0.2628883123397827, |
| "reward_std": 0.04087233170866966, |
| "rewards/clip_reward": 0.2628883123397827, |
| "step": 42 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.001325462756037791, |
| "grad_norm": 0.3840340108983825, |
| "kl": 0.0338134765625, |
| "learning_rate": 1e-05, |
| "loss": 0.0014, |
| "reward": 0.27363643795251846, |
| "reward_std": 0.03607825469225645, |
| "rewards/clip_reward": 0.27363643795251846, |
| "step": 43 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.001356287471294484, |
| "grad_norm": 0.2581672136879213, |
| "kl": 0.03271484375, |
| "learning_rate": 1e-05, |
| "loss": 0.0013, |
| "reward": 0.2834562137722969, |
| "reward_std": 0.04080742411315441, |
| "rewards/clip_reward": 0.2834562137722969, |
| "step": 44 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.0013871121865511767, |
| "grad_norm": 0.22063460755244194, |
| "kl": 0.03253173828125, |
| "learning_rate": 1e-05, |
| "loss": 0.0013, |
| "reward": 0.2772428020834923, |
| "reward_std": 0.04003895306959748, |
| "rewards/clip_reward": 0.2772428020834923, |
| "step": 45 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.0014179369018078695, |
| "grad_norm": 0.2565525371064913, |
| "kl": 0.03271484375, |
| "learning_rate": 1e-05, |
| "loss": 0.0013, |
| "reward": 0.2706209644675255, |
| "reward_std": 0.04194391146302223, |
| "rewards/clip_reward": 0.2706209644675255, |
| "step": 46 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.0014487616170645624, |
| "grad_norm": 0.2490150749271912, |
| "kl": 0.03497314453125, |
| "learning_rate": 1e-05, |
| "loss": 0.0014, |
| "reward": 0.2823899984359741, |
| "reward_std": 0.04330089082941413, |
| "rewards/clip_reward": 0.2823899984359741, |
| "step": 47 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.0014795863323212551, |
| "grad_norm": 0.31942389183248127, |
| "kl": 0.033203125, |
| "learning_rate": 1e-05, |
| "loss": 0.0013, |
| "reward": 0.27612780779600143, |
| "reward_std": 0.03909243643283844, |
| "rewards/clip_reward": 0.27612780779600143, |
| "step": 48 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.001510411047577948, |
| "grad_norm": 0.25176125337143185, |
| "kl": 0.0323486328125, |
| "learning_rate": 1e-05, |
| "loss": 0.0013, |
| "reward": 0.27114395797252655, |
| "reward_std": 0.043553344905376434, |
| "rewards/clip_reward": 0.27114395797252655, |
| "step": 49 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.001541235762834641, |
| "grad_norm": 0.25140342833503415, |
| "kl": 0.03302001953125, |
| "learning_rate": 1e-05, |
| "loss": 0.0013, |
| "reward": 0.2741701751947403, |
| "reward_std": 0.04216139670461416, |
| "rewards/clip_reward": 0.2741701751947403, |
| "step": 50 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.0015720604780913336, |
| "grad_norm": 0.2712169111225742, |
| "kl": 0.0338134765625, |
| "learning_rate": 1e-05, |
| "loss": 0.0014, |
| "reward": 0.2718612402677536, |
| "reward_std": 0.0403730683028698, |
| "rewards/clip_reward": 0.2718612402677536, |
| "step": 51 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.0016028851933480265, |
| "grad_norm": 0.24185377611726078, |
| "kl": 0.03265380859375, |
| "learning_rate": 1e-05, |
| "loss": 0.0013, |
| "reward": 0.2797749862074852, |
| "reward_std": 0.035524442326277494, |
| "rewards/clip_reward": 0.2797749862074852, |
| "step": 52 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.0016337099086047194, |
| "grad_norm": 0.2925738372367075, |
| "kl": 0.031890869140625, |
| "learning_rate": 1e-05, |
| "loss": 0.0013, |
| "reward": 0.27509263157844543, |
| "reward_std": 0.03820755332708359, |
| "rewards/clip_reward": 0.27509263157844543, |
| "step": 53 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.001664534623861412, |
| "grad_norm": 0.2243087092506723, |
| "kl": 0.037109375, |
| "learning_rate": 1e-05, |
| "loss": 0.0015, |
| "reward": 0.2797403186559677, |
| "reward_std": 0.03433629125356674, |
| "rewards/clip_reward": 0.2797403186559677, |
| "step": 54 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.001695359339118105, |
| "grad_norm": 0.5392048466375525, |
| "kl": 0.034881591796875, |
| "learning_rate": 1e-05, |
| "loss": 0.0014, |
| "reward": 0.2881343886256218, |
| "reward_std": 0.034057734068483114, |
| "rewards/clip_reward": 0.2881343886256218, |
| "step": 55 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.0017261840543747978, |
| "grad_norm": 0.3005202554746241, |
| "kl": 0.03155517578125, |
| "learning_rate": 1e-05, |
| "loss": 0.0013, |
| "reward": 0.28284603357315063, |
| "reward_std": 0.035686352755874395, |
| "rewards/clip_reward": 0.28284603357315063, |
| "step": 56 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.0017570087696314905, |
| "grad_norm": 0.4427830267310237, |
| "kl": 0.03265380859375, |
| "learning_rate": 1e-05, |
| "loss": 0.0013, |
| "reward": 0.2999447137117386, |
| "reward_std": 0.037478470243513584, |
| "rewards/clip_reward": 0.2999447137117386, |
| "step": 57 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.0017878334848881834, |
| "grad_norm": 0.3046115937307048, |
| "kl": 0.03387451171875, |
| "learning_rate": 1e-05, |
| "loss": 0.0014, |
| "reward": 0.2802000045776367, |
| "reward_std": 0.042125691659748554, |
| "rewards/clip_reward": 0.2802000045776367, |
| "step": 58 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.001818658200144876, |
| "grad_norm": 0.2405611837420283, |
| "kl": 0.03338623046875, |
| "learning_rate": 1e-05, |
| "loss": 0.0013, |
| "reward": 0.29059184342622757, |
| "reward_std": 0.037164853885769844, |
| "rewards/clip_reward": 0.29059184342622757, |
| "step": 59 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.001849482915401569, |
| "grad_norm": 0.24105929492457603, |
| "kl": 0.03192138671875, |
| "learning_rate": 1e-05, |
| "loss": 0.0013, |
| "reward": 0.2885151877999306, |
| "reward_std": 0.03311763470992446, |
| "rewards/clip_reward": 0.2885151877999306, |
| "step": 60 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.0018803076306582618, |
| "grad_norm": 0.3666040943996562, |
| "kl": 0.035308837890625, |
| "learning_rate": 1e-05, |
| "loss": 0.0014, |
| "reward": 0.2791602984070778, |
| "reward_std": 0.04032097943127155, |
| "rewards/clip_reward": 0.2791602984070778, |
| "step": 61 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.0019111323459149545, |
| "grad_norm": 0.30249506470602205, |
| "kl": 0.0343017578125, |
| "learning_rate": 1e-05, |
| "loss": 0.0014, |
| "reward": 0.2850872874259949, |
| "reward_std": 0.04306080937385559, |
| "rewards/clip_reward": 0.2850872874259949, |
| "step": 62 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.0019419570611716474, |
| "grad_norm": 0.2627870943706208, |
| "kl": 0.0323486328125, |
| "learning_rate": 1e-05, |
| "loss": 0.0013, |
| "reward": 0.28769853711128235, |
| "reward_std": 0.03929570922628045, |
| "rewards/clip_reward": 0.28769853711128235, |
| "step": 63 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.0019727817764283403, |
| "grad_norm": 0.302176588700791, |
| "kl": 0.03369140625, |
| "learning_rate": 1e-05, |
| "loss": 0.0013, |
| "reward": 0.27818652242422104, |
| "reward_std": 0.03506180923432112, |
| "rewards/clip_reward": 0.27818652242422104, |
| "step": 64 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.002003606491685033, |
| "grad_norm": 0.24687044072190295, |
| "kl": 0.0352783203125, |
| "learning_rate": 1e-05, |
| "loss": 0.0014, |
| "reward": 0.30166226625442505, |
| "reward_std": 0.03524617711082101, |
| "rewards/clip_reward": 0.30166226625442505, |
| "step": 65 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.002034431206941726, |
| "grad_norm": 0.30419214057266597, |
| "kl": 0.0382080078125, |
| "learning_rate": 1e-05, |
| "loss": 0.0015, |
| "reward": 0.2780046612024307, |
| "reward_std": 0.039600692223757505, |
| "rewards/clip_reward": 0.2780046612024307, |
| "step": 66 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.0020652559221984185, |
| "grad_norm": 0.27119766585139016, |
| "kl": 0.03387451171875, |
| "learning_rate": 1e-05, |
| "loss": 0.0014, |
| "reward": 0.294754721224308, |
| "reward_std": 0.034248299431055784, |
| "rewards/clip_reward": 0.294754721224308, |
| "step": 67 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.0020960806374551114, |
| "grad_norm": 0.24490582535286695, |
| "kl": 0.0340576171875, |
| "learning_rate": 1e-05, |
| "loss": 0.0014, |
| "reward": 0.29758618772029877, |
| "reward_std": 0.03921351861208677, |
| "rewards/clip_reward": 0.29758618772029877, |
| "step": 68 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.0021269053527118043, |
| "grad_norm": 0.2906006315520449, |
| "kl": 0.0374755859375, |
| "learning_rate": 1e-05, |
| "loss": 0.0015, |
| "reward": 0.28643497824668884, |
| "reward_std": 0.03893239703029394, |
| "rewards/clip_reward": 0.28643497824668884, |
| "step": 69 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.002157730067968497, |
| "grad_norm": 0.2861214644137581, |
| "kl": 0.0352783203125, |
| "learning_rate": 1e-05, |
| "loss": 0.0014, |
| "reward": 0.2825808748602867, |
| "reward_std": 0.03427910804748535, |
| "rewards/clip_reward": 0.2825808748602867, |
| "step": 70 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.00218855478322519, |
| "grad_norm": 0.37011498013054983, |
| "kl": 0.03814697265625, |
| "learning_rate": 1e-05, |
| "loss": 0.0015, |
| "reward": 0.2854940667748451, |
| "reward_std": 0.034281593747437, |
| "rewards/clip_reward": 0.2854940667748451, |
| "step": 71 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.0022193794984818826, |
| "grad_norm": 0.24442639828837878, |
| "kl": 0.0372314453125, |
| "learning_rate": 1e-05, |
| "loss": 0.0015, |
| "reward": 0.28134068101644516, |
| "reward_std": 0.03528659883886576, |
| "rewards/clip_reward": 0.28134068101644516, |
| "step": 72 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.0022502042137385755, |
| "grad_norm": 0.4685706452282796, |
| "kl": 0.04559326171875, |
| "learning_rate": 1e-05, |
| "loss": 0.0018, |
| "reward": 0.2905724048614502, |
| "reward_std": 0.040434951428323984, |
| "rewards/clip_reward": 0.2905724048614502, |
| "step": 73 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.0022810289289952683, |
| "grad_norm": 0.2978744286423797, |
| "kl": 0.03948974609375, |
| "learning_rate": 1e-05, |
| "loss": 0.0016, |
| "reward": 0.2837352082133293, |
| "reward_std": 0.037585786543786526, |
| "rewards/clip_reward": 0.2837352082133293, |
| "step": 74 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.0023118536442519612, |
| "grad_norm": 0.2642233216051261, |
| "kl": 0.03631591796875, |
| "learning_rate": 1e-05, |
| "loss": 0.0015, |
| "reward": 0.2846062183380127, |
| "reward_std": 0.032358222641050816, |
| "rewards/clip_reward": 0.2846062183380127, |
| "step": 75 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.002342678359508654, |
| "grad_norm": 0.38580930898434906, |
| "kl": 0.03814697265625, |
| "learning_rate": 1e-05, |
| "loss": 0.0015, |
| "reward": 0.2916061654686928, |
| "reward_std": 0.03999630082398653, |
| "rewards/clip_reward": 0.2916061654686928, |
| "step": 76 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.002373503074765347, |
| "grad_norm": 0.2926547008151926, |
| "kl": 0.03857421875, |
| "learning_rate": 1e-05, |
| "loss": 0.0015, |
| "reward": 0.295952245593071, |
| "reward_std": 0.03780945483595133, |
| "rewards/clip_reward": 0.295952245593071, |
| "step": 77 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.0024043277900220395, |
| "grad_norm": 0.25208665362075805, |
| "kl": 0.0377197265625, |
| "learning_rate": 1e-05, |
| "loss": 0.0015, |
| "reward": 0.2766958400607109, |
| "reward_std": 0.034185357857495546, |
| "rewards/clip_reward": 0.2766958400607109, |
| "step": 78 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.0024351525052787324, |
| "grad_norm": 0.24422171158940864, |
| "kl": 0.0357666015625, |
| "learning_rate": 1e-05, |
| "loss": 0.0014, |
| "reward": 0.2962762862443924, |
| "reward_std": 0.037650241516530514, |
| "rewards/clip_reward": 0.2962762862443924, |
| "step": 79 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.0024659772205354253, |
| "grad_norm": 0.22603888808512623, |
| "kl": 0.03509521484375, |
| "learning_rate": 1e-05, |
| "loss": 0.0014, |
| "reward": 0.30113714188337326, |
| "reward_std": 0.03855917416512966, |
| "rewards/clip_reward": 0.30113714188337326, |
| "step": 80 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.002496801935792118, |
| "grad_norm": 0.37733238977424605, |
| "kl": 0.0335693359375, |
| "learning_rate": 1e-05, |
| "loss": 0.0013, |
| "reward": 0.28931064158678055, |
| "reward_std": 0.041936729568988085, |
| "rewards/clip_reward": 0.28931064158678055, |
| "step": 81 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.002527626651048811, |
| "grad_norm": 2397.9127472616574, |
| "kl": 2.27880859375, |
| "learning_rate": 1e-05, |
| "loss": 0.0912, |
| "reward": 0.2881544306874275, |
| "reward_std": 0.03951821103692055, |
| "rewards/clip_reward": 0.2881544306874275, |
| "step": 82 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.002558451366305504, |
| "grad_norm": 0.2551335788444245, |
| "kl": 0.035888671875, |
| "learning_rate": 1e-05, |
| "loss": 0.0014, |
| "reward": 0.29982026666402817, |
| "reward_std": 0.041388670448213816, |
| "rewards/clip_reward": 0.29982026666402817, |
| "step": 83 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.0025892760815621964, |
| "grad_norm": 0.2368918105634009, |
| "kl": 0.0340576171875, |
| "learning_rate": 1e-05, |
| "loss": 0.0014, |
| "reward": 0.29101932793855667, |
| "reward_std": 0.02901851013302803, |
| "rewards/clip_reward": 0.29101932793855667, |
| "step": 84 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.0026201007968188893, |
| "grad_norm": 0.37688170465955695, |
| "kl": 0.0369873046875, |
| "learning_rate": 1e-05, |
| "loss": 0.0015, |
| "reward": 0.28874391317367554, |
| "reward_std": 0.03887001145631075, |
| "rewards/clip_reward": 0.28874391317367554, |
| "step": 85 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.002650925512075582, |
| "grad_norm": 0.2519378074219347, |
| "kl": 0.033203125, |
| "learning_rate": 1e-05, |
| "loss": 0.0013, |
| "reward": 0.30362261086702347, |
| "reward_std": 0.03999508544802666, |
| "rewards/clip_reward": 0.30362261086702347, |
| "step": 86 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.002681750227332275, |
| "grad_norm": 0.35429225967137656, |
| "kl": 0.03558349609375, |
| "learning_rate": 1e-05, |
| "loss": 0.0014, |
| "reward": 0.28408853709697723, |
| "reward_std": 0.038170519284904, |
| "rewards/clip_reward": 0.28408853709697723, |
| "step": 87 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.002712574942588968, |
| "grad_norm": 0.24826760619770735, |
| "kl": 0.03509521484375, |
| "learning_rate": 1e-05, |
| "loss": 0.0014, |
| "reward": 0.2828948646783829, |
| "reward_std": 0.030402940697968006, |
| "rewards/clip_reward": 0.2828948646783829, |
| "step": 88 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.002743399657845661, |
| "grad_norm": 0.24050960559813894, |
| "kl": 0.0369873046875, |
| "learning_rate": 1e-05, |
| "loss": 0.0015, |
| "reward": 0.28854893147945404, |
| "reward_std": 0.031570473685860634, |
| "rewards/clip_reward": 0.28854893147945404, |
| "step": 89 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.0027742243731023533, |
| "grad_norm": 0.3157405419841437, |
| "kl": 0.03387451171875, |
| "learning_rate": 1e-05, |
| "loss": 0.0014, |
| "reward": 0.3040951117873192, |
| "reward_std": 0.03156033856794238, |
| "rewards/clip_reward": 0.3040951117873192, |
| "step": 90 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.002805049088359046, |
| "grad_norm": 0.255465711049786, |
| "kl": 0.036865234375, |
| "learning_rate": 1e-05, |
| "loss": 0.0015, |
| "reward": 0.28177835047245026, |
| "reward_std": 0.03183179069310427, |
| "rewards/clip_reward": 0.28177835047245026, |
| "step": 91 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.002835873803615739, |
| "grad_norm": 0.2754084224378065, |
| "kl": 0.0357666015625, |
| "learning_rate": 1e-05, |
| "loss": 0.0014, |
| "reward": 0.28247682750225067, |
| "reward_std": 0.04544441122561693, |
| "rewards/clip_reward": 0.28247682750225067, |
| "step": 92 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.002866698518872432, |
| "grad_norm": 0.30917890933321285, |
| "kl": 0.03411865234375, |
| "learning_rate": 1e-05, |
| "loss": 0.0014, |
| "reward": 0.30449671298265457, |
| "reward_std": 0.038043808192014694, |
| "rewards/clip_reward": 0.30449671298265457, |
| "step": 93 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.002897523234129125, |
| "grad_norm": 0.2739772980532729, |
| "kl": 0.0352783203125, |
| "learning_rate": 1e-05, |
| "loss": 0.0014, |
| "reward": 0.2972172349691391, |
| "reward_std": 0.04181887488812208, |
| "rewards/clip_reward": 0.2972172349691391, |
| "step": 94 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.0029283479493858173, |
| "grad_norm": 0.4882887735659522, |
| "kl": 0.0369873046875, |
| "learning_rate": 1e-05, |
| "loss": 0.0015, |
| "reward": 0.30260052531957626, |
| "reward_std": 0.03962193429470062, |
| "rewards/clip_reward": 0.30260052531957626, |
| "step": 95 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.0029591726646425102, |
| "grad_norm": 0.2758869488382661, |
| "kl": 0.03765869140625, |
| "learning_rate": 1e-05, |
| "loss": 0.0015, |
| "reward": 0.273670993745327, |
| "reward_std": 0.034096458461135626, |
| "rewards/clip_reward": 0.273670993745327, |
| "step": 96 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.002989997379899203, |
| "grad_norm": 0.3218139742013513, |
| "kl": 0.0357666015625, |
| "learning_rate": 1e-05, |
| "loss": 0.0014, |
| "reward": 0.2953087314963341, |
| "reward_std": 0.03857262898236513, |
| "rewards/clip_reward": 0.2953087314963341, |
| "step": 97 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.003020822095155896, |
| "grad_norm": 0.27436643426318075, |
| "kl": 0.03802490234375, |
| "learning_rate": 1e-05, |
| "loss": 0.0015, |
| "reward": 0.2942623794078827, |
| "reward_std": 0.04214351158589125, |
| "rewards/clip_reward": 0.2942623794078827, |
| "step": 98 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.003051646810412589, |
| "grad_norm": 0.46160645575216935, |
| "kl": 0.0382080078125, |
| "learning_rate": 1e-05, |
| "loss": 0.0015, |
| "reward": 0.2718002200126648, |
| "reward_std": 0.03470963425934315, |
| "rewards/clip_reward": 0.2718002200126648, |
| "step": 99 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.003082471525669282, |
| "grad_norm": 0.2710130788062584, |
| "kl": 0.0394287109375, |
| "learning_rate": 1e-05, |
| "loss": 0.0016, |
| "reward": 0.30108004808425903, |
| "reward_std": 0.03474955866113305, |
| "rewards/clip_reward": 0.30108004808425903, |
| "step": 100 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.0031132962409259743, |
| "grad_norm": 0.49951830790663787, |
| "kl": 0.0382080078125, |
| "learning_rate": 1e-05, |
| "loss": 0.0015, |
| "reward": 0.2938985303044319, |
| "reward_std": 0.03756955498829484, |
| "rewards/clip_reward": 0.2938985303044319, |
| "step": 101 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.003144120956182667, |
| "grad_norm": 0.309853879858268, |
| "kl": 0.03790283203125, |
| "learning_rate": 1e-05, |
| "loss": 0.0015, |
| "reward": 0.2770190164446831, |
| "reward_std": 0.04118606820702553, |
| "rewards/clip_reward": 0.2770190164446831, |
| "step": 102 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.00317494567143936, |
| "grad_norm": 0.303540839737249, |
| "kl": 0.04150390625, |
| "learning_rate": 1e-05, |
| "loss": 0.0017, |
| "reward": 0.3010944053530693, |
| "reward_std": 0.03948027174919844, |
| "rewards/clip_reward": 0.3010944053530693, |
| "step": 103 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.003205770386696053, |
| "grad_norm": 0.4242493698059439, |
| "kl": 0.04302978515625, |
| "learning_rate": 1e-05, |
| "loss": 0.0017, |
| "reward": 0.30070605874061584, |
| "reward_std": 0.03413457376882434, |
| "rewards/clip_reward": 0.30070605874061584, |
| "step": 104 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.003236595101952746, |
| "grad_norm": 0.2893333008136438, |
| "kl": 0.04876708984375, |
| "learning_rate": 1e-05, |
| "loss": 0.002, |
| "reward": 0.30694443732500076, |
| "reward_std": 0.03970211138948798, |
| "rewards/clip_reward": 0.30694443732500076, |
| "step": 105 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.0032674198172094387, |
| "grad_norm": 0.9533572888059393, |
| "kl": 0.05682373046875, |
| "learning_rate": 1e-05, |
| "loss": 0.0023, |
| "reward": 0.3035873845219612, |
| "reward_std": 0.03094083722680807, |
| "rewards/clip_reward": 0.3035873845219612, |
| "step": 106 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.003298244532466131, |
| "grad_norm": 0.6987792496689551, |
| "kl": 0.0474853515625, |
| "learning_rate": 1e-05, |
| "loss": 0.0019, |
| "reward": 0.26798417791724205, |
| "reward_std": 0.0339839537627995, |
| "rewards/clip_reward": 0.26798417791724205, |
| "step": 107 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.003329069247722824, |
| "grad_norm": 0.2695872192954547, |
| "kl": 0.03973388671875, |
| "learning_rate": 1e-05, |
| "loss": 0.0016, |
| "reward": 0.2790006026625633, |
| "reward_std": 0.040701782796531916, |
| "rewards/clip_reward": 0.2790006026625633, |
| "step": 108 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.003359893962979517, |
| "grad_norm": 2.392119350950025, |
| "kl": 0.04656982421875, |
| "learning_rate": 1e-05, |
| "loss": 0.0019, |
| "reward": 0.29154668748378754, |
| "reward_std": 0.03984204959124327, |
| "rewards/clip_reward": 0.29154668748378754, |
| "step": 109 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.00339071867823621, |
| "grad_norm": 0.30051030904173764, |
| "kl": 0.0389404296875, |
| "learning_rate": 1e-05, |
| "loss": 0.0016, |
| "reward": 0.28251485526561737, |
| "reward_std": 0.03768534865230322, |
| "rewards/clip_reward": 0.28251485526561737, |
| "step": 110 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.0034215433934929027, |
| "grad_norm": 0.33966462863188296, |
| "kl": 0.04290771484375, |
| "learning_rate": 1e-05, |
| "loss": 0.0017, |
| "reward": 0.28882861137390137, |
| "reward_std": 0.030829247552901506, |
| "rewards/clip_reward": 0.28882861137390137, |
| "step": 111 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.0034523681087495956, |
| "grad_norm": 0.27353141186845165, |
| "kl": 0.043212890625, |
| "learning_rate": 1e-05, |
| "loss": 0.0017, |
| "reward": 0.2902967408299446, |
| "reward_std": 0.03842968260869384, |
| "rewards/clip_reward": 0.2902967408299446, |
| "step": 112 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.003483192824006288, |
| "grad_norm": 0.31909208905959036, |
| "kl": 0.039794921875, |
| "learning_rate": 1e-05, |
| "loss": 0.0016, |
| "reward": 0.30095263570547104, |
| "reward_std": 0.03839526232331991, |
| "rewards/clip_reward": 0.30095263570547104, |
| "step": 113 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.003514017539262981, |
| "grad_norm": 0.2628770760125084, |
| "kl": 0.04241943359375, |
| "learning_rate": 1e-05, |
| "loss": 0.0017, |
| "reward": 0.30361922085285187, |
| "reward_std": 0.03204685868695378, |
| "rewards/clip_reward": 0.30361922085285187, |
| "step": 114 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.003544842254519674, |
| "grad_norm": 0.5077902647628189, |
| "kl": 0.04266357421875, |
| "learning_rate": 1e-05, |
| "loss": 0.0017, |
| "reward": 0.28539516031742096, |
| "reward_std": 0.03661313094198704, |
| "rewards/clip_reward": 0.28539516031742096, |
| "step": 115 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.0035756669697763668, |
| "grad_norm": 0.2750571920299247, |
| "kl": 0.04144287109375, |
| "learning_rate": 1e-05, |
| "loss": 0.0017, |
| "reward": 0.28498194366693497, |
| "reward_std": 0.033483162987977266, |
| "rewards/clip_reward": 0.28498194366693497, |
| "step": 116 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.0036064916850330597, |
| "grad_norm": 0.4090948836642478, |
| "kl": 0.05047607421875, |
| "learning_rate": 1e-05, |
| "loss": 0.002, |
| "reward": 0.2884545177221298, |
| "reward_std": 0.03879341948777437, |
| "rewards/clip_reward": 0.2884545177221298, |
| "step": 117 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.003637316400289752, |
| "grad_norm": 0.2747726497219782, |
| "kl": 0.0418701171875, |
| "learning_rate": 1e-05, |
| "loss": 0.0017, |
| "reward": 0.2886419966816902, |
| "reward_std": 0.03201331151649356, |
| "rewards/clip_reward": 0.2886419966816902, |
| "step": 118 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.003668141115546445, |
| "grad_norm": 0.24335488340779768, |
| "kl": 0.041015625, |
| "learning_rate": 1e-05, |
| "loss": 0.0016, |
| "reward": 0.29286789894104004, |
| "reward_std": 0.03851965814828873, |
| "rewards/clip_reward": 0.29286789894104004, |
| "step": 119 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.003698965830803138, |
| "grad_norm": 0.3272604578026692, |
| "kl": 0.0421142578125, |
| "learning_rate": 1e-05, |
| "loss": 0.0017, |
| "reward": 0.3037740960717201, |
| "reward_std": 0.04023708775639534, |
| "rewards/clip_reward": 0.3037740960717201, |
| "step": 120 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.003729790546059831, |
| "grad_norm": 0.2573037344425057, |
| "kl": 0.042724609375, |
| "learning_rate": 1e-05, |
| "loss": 0.0017, |
| "reward": 0.3032229617238045, |
| "reward_std": 0.03995621297508478, |
| "rewards/clip_reward": 0.3032229617238045, |
| "step": 121 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.0037606152613165237, |
| "grad_norm": 0.23884237904332833, |
| "kl": 0.040771484375, |
| "learning_rate": 1e-05, |
| "loss": 0.0016, |
| "reward": 0.2992851510643959, |
| "reward_std": 0.03295802231878042, |
| "rewards/clip_reward": 0.2992851510643959, |
| "step": 122 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.0037914399765732166, |
| "grad_norm": 0.2563902015046439, |
| "kl": 0.03961181640625, |
| "learning_rate": 1e-05, |
| "loss": 0.0016, |
| "reward": 0.29188019037246704, |
| "reward_std": 0.03674457548186183, |
| "rewards/clip_reward": 0.29188019037246704, |
| "step": 123 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.003822264691829909, |
| "grad_norm": 0.2983101831944227, |
| "kl": 0.0421142578125, |
| "learning_rate": 1e-05, |
| "loss": 0.0017, |
| "reward": 0.2967969849705696, |
| "reward_std": 0.035267810337245464, |
| "rewards/clip_reward": 0.2967969849705696, |
| "step": 124 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.003853089407086602, |
| "grad_norm": 0.24729135355433982, |
| "kl": 0.04266357421875, |
| "learning_rate": 1e-05, |
| "loss": 0.0017, |
| "reward": 0.2964273914694786, |
| "reward_std": 0.03762258403003216, |
| "rewards/clip_reward": 0.2964273914694786, |
| "step": 125 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.003883914122343295, |
| "grad_norm": 0.36172960084474337, |
| "kl": 0.0379638671875, |
| "learning_rate": 1e-05, |
| "loss": 0.0015, |
| "reward": 0.2893693521618843, |
| "reward_std": 0.030456844717264175, |
| "rewards/clip_reward": 0.2893693521618843, |
| "step": 126 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.003914738837599988, |
| "grad_norm": 1.3235252455659776, |
| "kl": 0.06842041015625, |
| "learning_rate": 1e-05, |
| "loss": 0.0027, |
| "reward": 0.28936315327882767, |
| "reward_std": 0.032889596186578274, |
| "rewards/clip_reward": 0.28936315327882767, |
| "step": 127 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.003945563552856681, |
| "grad_norm": 0.33282790631669, |
| "kl": 0.04095458984375, |
| "learning_rate": 1e-05, |
| "loss": 0.0016, |
| "reward": 0.3032195121049881, |
| "reward_std": 0.03782738745212555, |
| "rewards/clip_reward": 0.3032195121049881, |
| "step": 128 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.0039763882681133735, |
| "grad_norm": 0.26941621679510896, |
| "kl": 0.0401611328125, |
| "learning_rate": 1e-05, |
| "loss": 0.0016, |
| "reward": 0.29521265625953674, |
| "reward_std": 0.028109443373978138, |
| "rewards/clip_reward": 0.29521265625953674, |
| "step": 129 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.004007212983370066, |
| "grad_norm": 0.6011583420828783, |
| "kl": 0.042236328125, |
| "learning_rate": 1e-05, |
| "loss": 0.0017, |
| "reward": 0.29040753841400146, |
| "reward_std": 0.03746784944087267, |
| "rewards/clip_reward": 0.29040753841400146, |
| "step": 130 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.004038037698626759, |
| "grad_norm": 0.28850122675094336, |
| "kl": 0.03900146484375, |
| "learning_rate": 1e-05, |
| "loss": 0.0016, |
| "reward": 0.298222653567791, |
| "reward_std": 0.032059306744486094, |
| "rewards/clip_reward": 0.298222653567791, |
| "step": 131 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.004068862413883452, |
| "grad_norm": 0.297623960565589, |
| "kl": 0.04571533203125, |
| "learning_rate": 1e-05, |
| "loss": 0.0018, |
| "reward": 0.28365931659936905, |
| "reward_std": 0.03448425652459264, |
| "rewards/clip_reward": 0.28365931659936905, |
| "step": 132 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.004099687129140144, |
| "grad_norm": 0.33237851102895244, |
| "kl": 0.04266357421875, |
| "learning_rate": 1e-05, |
| "loss": 0.0017, |
| "reward": 0.30908743292093277, |
| "reward_std": 0.03715136833488941, |
| "rewards/clip_reward": 0.30908743292093277, |
| "step": 133 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.004130511844396837, |
| "grad_norm": 0.28432833313640343, |
| "kl": 0.04046630859375, |
| "learning_rate": 1e-05, |
| "loss": 0.0016, |
| "reward": 0.28690189868211746, |
| "reward_std": 0.036837459076195955, |
| "rewards/clip_reward": 0.28690189868211746, |
| "step": 134 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.00416133655965353, |
| "grad_norm": 0.289418120507171, |
| "kl": 0.042236328125, |
| "learning_rate": 1e-05, |
| "loss": 0.0017, |
| "reward": 0.27922137826681137, |
| "reward_std": 0.04220917448401451, |
| "rewards/clip_reward": 0.27922137826681137, |
| "step": 135 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.004192161274910223, |
| "grad_norm": 0.443250812892132, |
| "kl": 0.04840087890625, |
| "learning_rate": 1e-05, |
| "loss": 0.0019, |
| "reward": 0.29268399626016617, |
| "reward_std": 0.042575713247060776, |
| "rewards/clip_reward": 0.29268399626016617, |
| "step": 136 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.004222985990166916, |
| "grad_norm": 0.2936447251353246, |
| "kl": 0.0419921875, |
| "learning_rate": 1e-05, |
| "loss": 0.0017, |
| "reward": 0.3000137060880661, |
| "reward_std": 0.03598225861787796, |
| "rewards/clip_reward": 0.3000137060880661, |
| "step": 137 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.004253810705423609, |
| "grad_norm": 0.2638651763481444, |
| "kl": 0.044189453125, |
| "learning_rate": 1e-05, |
| "loss": 0.0018, |
| "reward": 0.29852742701768875, |
| "reward_std": 0.03497701371088624, |
| "rewards/clip_reward": 0.29852742701768875, |
| "step": 138 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.0042846354206803015, |
| "grad_norm": 0.6490276376018281, |
| "kl": 0.04449462890625, |
| "learning_rate": 1e-05, |
| "loss": 0.0018, |
| "reward": 0.2969634085893631, |
| "reward_std": 0.03716578893363476, |
| "rewards/clip_reward": 0.2969634085893631, |
| "step": 139 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.004315460135936994, |
| "grad_norm": 0.4290585504001145, |
| "kl": 0.048583984375, |
| "learning_rate": 1e-05, |
| "loss": 0.0019, |
| "reward": 0.30021680146455765, |
| "reward_std": 0.04055328294634819, |
| "rewards/clip_reward": 0.30021680146455765, |
| "step": 140 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.004346284851193687, |
| "grad_norm": 0.2880701541000658, |
| "kl": 0.046142578125, |
| "learning_rate": 1e-05, |
| "loss": 0.0018, |
| "reward": 0.2995045334100723, |
| "reward_std": 0.03819493483752012, |
| "rewards/clip_reward": 0.2995045334100723, |
| "step": 141 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.00437710956645038, |
| "grad_norm": 0.8593874213184524, |
| "kl": 0.0513916015625, |
| "learning_rate": 1e-05, |
| "loss": 0.0021, |
| "reward": 0.30462899804115295, |
| "reward_std": 0.03424446424469352, |
| "rewards/clip_reward": 0.30462899804115295, |
| "step": 142 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.004407934281707073, |
| "grad_norm": 0.21911976066768077, |
| "kl": 0.04931640625, |
| "learning_rate": 1e-05, |
| "loss": 0.002, |
| "reward": 0.3081146106123924, |
| "reward_std": 0.03195359604433179, |
| "rewards/clip_reward": 0.3081146106123924, |
| "step": 143 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.004438758996963765, |
| "grad_norm": 0.34265255923839133, |
| "kl": 0.0526123046875, |
| "learning_rate": 1e-05, |
| "loss": 0.0021, |
| "reward": 0.2893141508102417, |
| "reward_std": 0.03520893771201372, |
| "rewards/clip_reward": 0.2893141508102417, |
| "step": 144 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.004469583712220458, |
| "grad_norm": 0.3221549984597468, |
| "kl": 0.05474853515625, |
| "learning_rate": 1e-05, |
| "loss": 0.0022, |
| "reward": 0.3002786561846733, |
| "reward_std": 0.033627052791416645, |
| "rewards/clip_reward": 0.3002786561846733, |
| "step": 145 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.004500408427477151, |
| "grad_norm": 0.35731934724740755, |
| "kl": 0.0506591796875, |
| "learning_rate": 1e-05, |
| "loss": 0.002, |
| "reward": 0.2964288890361786, |
| "reward_std": 0.03710630023851991, |
| "rewards/clip_reward": 0.2964288890361786, |
| "step": 146 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.004531233142733844, |
| "grad_norm": 0.498796371247507, |
| "kl": 0.063232421875, |
| "learning_rate": 1e-05, |
| "loss": 0.0025, |
| "reward": 0.2942727655172348, |
| "reward_std": 0.034661782905459404, |
| "rewards/clip_reward": 0.2942727655172348, |
| "step": 147 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.004562057857990537, |
| "grad_norm": 0.35040193973506645, |
| "kl": 0.04986572265625, |
| "learning_rate": 1e-05, |
| "loss": 0.002, |
| "reward": 0.3076755926012993, |
| "reward_std": 0.038345606066286564, |
| "rewards/clip_reward": 0.3076755926012993, |
| "step": 148 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.00459288257324723, |
| "grad_norm": 0.284116312840595, |
| "kl": 0.0498046875, |
| "learning_rate": 1e-05, |
| "loss": 0.002, |
| "reward": 0.2897453159093857, |
| "reward_std": 0.03225108701735735, |
| "rewards/clip_reward": 0.2897453159093857, |
| "step": 149 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.0046237072885039225, |
| "grad_norm": 2.510868241115512, |
| "kl": 0.0657958984375, |
| "learning_rate": 1e-05, |
| "loss": 0.0026, |
| "reward": 0.28433967381715775, |
| "reward_std": 0.03440393693745136, |
| "rewards/clip_reward": 0.28433967381715775, |
| "step": 150 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.004654532003760615, |
| "grad_norm": 0.2919131449174652, |
| "kl": 0.0528564453125, |
| "learning_rate": 1e-05, |
| "loss": 0.0021, |
| "reward": 0.2939991131424904, |
| "reward_std": 0.040119947865605354, |
| "rewards/clip_reward": 0.2939991131424904, |
| "step": 151 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.004685356719017308, |
| "grad_norm": 0.2728005479353481, |
| "kl": 0.0621337890625, |
| "learning_rate": 1e-05, |
| "loss": 0.0025, |
| "reward": 0.31283795088529587, |
| "reward_std": 0.04251454817131162, |
| "rewards/clip_reward": 0.31283795088529587, |
| "step": 152 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.004716181434274001, |
| "grad_norm": 0.3046425258813171, |
| "kl": 0.056884765625, |
| "learning_rate": 1e-05, |
| "loss": 0.0023, |
| "reward": 0.3088100552558899, |
| "reward_std": 0.03973885904997587, |
| "rewards/clip_reward": 0.3088100552558899, |
| "step": 153 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.004747006149530694, |
| "grad_norm": 0.36647907531053076, |
| "kl": 0.06378173828125, |
| "learning_rate": 1e-05, |
| "loss": 0.0026, |
| "reward": 0.2990795224905014, |
| "reward_std": 0.03181237168610096, |
| "rewards/clip_reward": 0.2990795224905014, |
| "step": 154 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.004777830864787387, |
| "grad_norm": 0.2874990294452977, |
| "kl": 0.05413818359375, |
| "learning_rate": 1e-05, |
| "loss": 0.0022, |
| "reward": 0.29652759432792664, |
| "reward_std": 0.034958623349666595, |
| "rewards/clip_reward": 0.29652759432792664, |
| "step": 155 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.004808655580044079, |
| "grad_norm": 0.2571961957720778, |
| "kl": 0.04888916015625, |
| "learning_rate": 1e-05, |
| "loss": 0.002, |
| "reward": 0.30016767233610153, |
| "reward_std": 0.03672470944002271, |
| "rewards/clip_reward": 0.30016767233610153, |
| "step": 156 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.004839480295300772, |
| "grad_norm": 0.2488580090168899, |
| "kl": 0.05389404296875, |
| "learning_rate": 1e-05, |
| "loss": 0.0022, |
| "reward": 0.3036653697490692, |
| "reward_std": 0.03518738830462098, |
| "rewards/clip_reward": 0.3036653697490692, |
| "step": 157 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.004870305010557465, |
| "grad_norm": 0.3740970067478605, |
| "kl": 0.0546875, |
| "learning_rate": 1e-05, |
| "loss": 0.0022, |
| "reward": 0.3022882342338562, |
| "reward_std": 0.040427629835903645, |
| "rewards/clip_reward": 0.3022882342338562, |
| "step": 158 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.004901129725814158, |
| "grad_norm": 0.27811303327109416, |
| "kl": 0.0537109375, |
| "learning_rate": 1e-05, |
| "loss": 0.0021, |
| "reward": 0.29319214075803757, |
| "reward_std": 0.03445435827597976, |
| "rewards/clip_reward": 0.29319214075803757, |
| "step": 159 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.0049319544410708505, |
| "grad_norm": 0.3118292624775214, |
| "kl": 0.0531005859375, |
| "learning_rate": 1e-05, |
| "loss": 0.0021, |
| "reward": 0.29692448675632477, |
| "reward_std": 0.02798191551119089, |
| "rewards/clip_reward": 0.29692448675632477, |
| "step": 160 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.004962779156327543, |
| "grad_norm": 0.24174045123252508, |
| "kl": 0.05987548828125, |
| "learning_rate": 1e-05, |
| "loss": 0.0024, |
| "reward": 0.2794330157339573, |
| "reward_std": 0.033141561318188906, |
| "rewards/clip_reward": 0.2794330157339573, |
| "step": 161 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.004993603871584236, |
| "grad_norm": 0.3249638089165546, |
| "kl": 0.0574951171875, |
| "learning_rate": 1e-05, |
| "loss": 0.0023, |
| "reward": 0.3030798211693764, |
| "reward_std": 0.03338931826874614, |
| "rewards/clip_reward": 0.3030798211693764, |
| "step": 162 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.005024428586840929, |
| "grad_norm": 0.2633110889874991, |
| "kl": 0.05413818359375, |
| "learning_rate": 1e-05, |
| "loss": 0.0022, |
| "reward": 0.29485588520765305, |
| "reward_std": 0.03062314447015524, |
| "rewards/clip_reward": 0.29485588520765305, |
| "step": 163 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.005055253302097622, |
| "grad_norm": 0.2803956547691715, |
| "kl": 0.05316162109375, |
| "learning_rate": 1e-05, |
| "loss": 0.0021, |
| "reward": 0.2970665916800499, |
| "reward_std": 0.036208128556609154, |
| "rewards/clip_reward": 0.2970665916800499, |
| "step": 164 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.005086078017354315, |
| "grad_norm": 0.30407819707590145, |
| "kl": 0.0552978515625, |
| "learning_rate": 1e-05, |
| "loss": 0.0022, |
| "reward": 0.28191495686769485, |
| "reward_std": 0.035574122332036495, |
| "rewards/clip_reward": 0.28191495686769485, |
| "step": 165 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.005116902732611008, |
| "grad_norm": 0.7069970857165138, |
| "kl": 0.05682373046875, |
| "learning_rate": 1e-05, |
| "loss": 0.0023, |
| "reward": 0.3111490234732628, |
| "reward_std": 0.03303992748260498, |
| "rewards/clip_reward": 0.3111490234732628, |
| "step": 166 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.0051477274478677, |
| "grad_norm": 0.5033892459063507, |
| "kl": 0.09039306640625, |
| "learning_rate": 1e-05, |
| "loss": 0.0036, |
| "reward": 0.2889687344431877, |
| "reward_std": 0.030694663524627686, |
| "rewards/clip_reward": 0.2889687344431877, |
| "step": 167 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.005178552163124393, |
| "grad_norm": 0.41528612307548435, |
| "kl": 0.056884765625, |
| "learning_rate": 1e-05, |
| "loss": 0.0023, |
| "reward": 0.3075498342514038, |
| "reward_std": 0.03620789339765906, |
| "rewards/clip_reward": 0.3075498342514038, |
| "step": 168 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.005209376878381086, |
| "grad_norm": 0.32166704594267, |
| "kl": 0.0555419921875, |
| "learning_rate": 1e-05, |
| "loss": 0.0022, |
| "reward": 0.29549096524715424, |
| "reward_std": 0.037705546244978905, |
| "rewards/clip_reward": 0.29549096524715424, |
| "step": 169 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.005240201593637779, |
| "grad_norm": 0.21631365869879202, |
| "kl": 0.05328369140625, |
| "learning_rate": 1e-05, |
| "loss": 0.0021, |
| "reward": 0.2998160049319267, |
| "reward_std": 0.03276947420090437, |
| "rewards/clip_reward": 0.2998160049319267, |
| "step": 170 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.0052710263088944715, |
| "grad_norm": 0.2516096372371072, |
| "kl": 0.05267333984375, |
| "learning_rate": 1e-05, |
| "loss": 0.0021, |
| "reward": 0.31134357303380966, |
| "reward_std": 0.03470080904662609, |
| "rewards/clip_reward": 0.31134357303380966, |
| "step": 171 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.005301851024151164, |
| "grad_norm": 0.29419080361317934, |
| "kl": 0.0538330078125, |
| "learning_rate": 1e-05, |
| "loss": 0.0022, |
| "reward": 0.3061481937766075, |
| "reward_std": 0.03180012432858348, |
| "rewards/clip_reward": 0.3061481937766075, |
| "step": 172 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.005332675739407857, |
| "grad_norm": 0.2498996223533549, |
| "kl": 0.0516357421875, |
| "learning_rate": 1e-05, |
| "loss": 0.0021, |
| "reward": 0.2943173050880432, |
| "reward_std": 0.037327985279262066, |
| "rewards/clip_reward": 0.2943173050880432, |
| "step": 173 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.00536350045466455, |
| "grad_norm": 0.2978051890631816, |
| "kl": 0.05413818359375, |
| "learning_rate": 1e-05, |
| "loss": 0.0022, |
| "reward": 0.3035401850938797, |
| "reward_std": 0.0322981933131814, |
| "rewards/clip_reward": 0.3035401850938797, |
| "step": 174 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.005394325169921243, |
| "grad_norm": 0.24015029890010034, |
| "kl": 0.05206298828125, |
| "learning_rate": 1e-05, |
| "loss": 0.0021, |
| "reward": 0.30325891077518463, |
| "reward_std": 0.04060996416956186, |
| "rewards/clip_reward": 0.30325891077518463, |
| "step": 175 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.005425149885177936, |
| "grad_norm": 0.246629237118055, |
| "kl": 0.05426025390625, |
| "learning_rate": 1e-05, |
| "loss": 0.0022, |
| "reward": 0.296851709485054, |
| "reward_std": 0.03895132802426815, |
| "rewards/clip_reward": 0.296851709485054, |
| "step": 176 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.005455974600434629, |
| "grad_norm": 0.26568277866945256, |
| "kl": 0.05633544921875, |
| "learning_rate": 1e-05, |
| "loss": 0.0023, |
| "reward": 0.3115716502070427, |
| "reward_std": 0.037875589448958635, |
| "rewards/clip_reward": 0.3115716502070427, |
| "step": 177 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.005486799315691322, |
| "grad_norm": 0.2868547511774212, |
| "kl": 0.05279541015625, |
| "learning_rate": 1e-05, |
| "loss": 0.0021, |
| "reward": 0.31741077452898026, |
| "reward_std": 0.03210792690515518, |
| "rewards/clip_reward": 0.31741077452898026, |
| "step": 178 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.005517624030948014, |
| "grad_norm": 0.2559585074400544, |
| "kl": 0.05120849609375, |
| "learning_rate": 1e-05, |
| "loss": 0.002, |
| "reward": 0.3083617463707924, |
| "reward_std": 0.0392221063375473, |
| "rewards/clip_reward": 0.3083617463707924, |
| "step": 179 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.005548448746204707, |
| "grad_norm": 0.24542077699632728, |
| "kl": 0.05108642578125, |
| "learning_rate": 1e-05, |
| "loss": 0.002, |
| "reward": 0.30339157581329346, |
| "reward_std": 0.03507227078080177, |
| "rewards/clip_reward": 0.30339157581329346, |
| "step": 180 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.0055792734614613995, |
| "grad_norm": 0.2540319224053049, |
| "kl": 0.05084228515625, |
| "learning_rate": 1e-05, |
| "loss": 0.002, |
| "reward": 0.30484145134687424, |
| "reward_std": 0.036596270743757486, |
| "rewards/clip_reward": 0.30484145134687424, |
| "step": 181 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.005610098176718092, |
| "grad_norm": 0.37332314299297165, |
| "kl": 0.0526123046875, |
| "learning_rate": 1e-05, |
| "loss": 0.0021, |
| "reward": 0.30233796685934067, |
| "reward_std": 0.030493673402816057, |
| "rewards/clip_reward": 0.30233796685934067, |
| "step": 182 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.005640922891974785, |
| "grad_norm": 0.2303993900918174, |
| "kl": 0.05133056640625, |
| "learning_rate": 1e-05, |
| "loss": 0.0021, |
| "reward": 0.3098205327987671, |
| "reward_std": 0.041695406660437584, |
| "rewards/clip_reward": 0.3098205327987671, |
| "step": 183 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.005671747607231478, |
| "grad_norm": 0.2435717876843093, |
| "kl": 0.05206298828125, |
| "learning_rate": 1e-05, |
| "loss": 0.0021, |
| "reward": 0.29874222725629807, |
| "reward_std": 0.029267210513353348, |
| "rewards/clip_reward": 0.29874222725629807, |
| "step": 184 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.005702572322488171, |
| "grad_norm": 0.31961072153806486, |
| "kl": 0.05194091796875, |
| "learning_rate": 1e-05, |
| "loss": 0.0021, |
| "reward": 0.29779237508773804, |
| "reward_std": 0.035278864204883575, |
| "rewards/clip_reward": 0.29779237508773804, |
| "step": 185 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.005733397037744864, |
| "grad_norm": 0.21282193347830292, |
| "kl": 0.0513916015625, |
| "learning_rate": 1e-05, |
| "loss": 0.0021, |
| "reward": 0.3018389344215393, |
| "reward_std": 0.03779670037329197, |
| "rewards/clip_reward": 0.3018389344215393, |
| "step": 186 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.005764221753001557, |
| "grad_norm": 0.23426135318521582, |
| "kl": 0.052001953125, |
| "learning_rate": 1e-05, |
| "loss": 0.0021, |
| "reward": 0.30514706671237946, |
| "reward_std": 0.03661506250500679, |
| "rewards/clip_reward": 0.30514706671237946, |
| "step": 187 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.00579504646825825, |
| "grad_norm": 0.35531046478807454, |
| "kl": 0.0570068359375, |
| "learning_rate": 1e-05, |
| "loss": 0.0023, |
| "reward": 0.2925054803490639, |
| "reward_std": 0.03318297350779176, |
| "rewards/clip_reward": 0.2925054803490639, |
| "step": 188 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.005825871183514943, |
| "grad_norm": 0.26126815879357707, |
| "kl": 0.05340576171875, |
| "learning_rate": 1e-05, |
| "loss": 0.0021, |
| "reward": 0.30776098370552063, |
| "reward_std": 0.03303914796561003, |
| "rewards/clip_reward": 0.30776098370552063, |
| "step": 189 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.005856695898771635, |
| "grad_norm": 0.3913467133913027, |
| "kl": 0.05584716796875, |
| "learning_rate": 1e-05, |
| "loss": 0.0022, |
| "reward": 0.3134502246975899, |
| "reward_std": 0.033362182322889566, |
| "rewards/clip_reward": 0.3134502246975899, |
| "step": 190 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.005887520614028328, |
| "grad_norm": 0.5481147865863106, |
| "kl": 0.05731201171875, |
| "learning_rate": 1e-05, |
| "loss": 0.0023, |
| "reward": 0.3062956631183624, |
| "reward_std": 0.03714533941820264, |
| "rewards/clip_reward": 0.3062956631183624, |
| "step": 191 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.0059183453292850205, |
| "grad_norm": 0.38319922904187664, |
| "kl": 0.0501708984375, |
| "learning_rate": 1e-05, |
| "loss": 0.002, |
| "reward": 0.29817763715982437, |
| "reward_std": 0.031161442399024963, |
| "rewards/clip_reward": 0.29817763715982437, |
| "step": 192 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.005949170044541713, |
| "grad_norm": 0.7525966257009459, |
| "kl": 0.05181884765625, |
| "learning_rate": 1e-05, |
| "loss": 0.0021, |
| "reward": 0.2918746843934059, |
| "reward_std": 0.04155853856354952, |
| "rewards/clip_reward": 0.2918746843934059, |
| "step": 193 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.005979994759798406, |
| "grad_norm": 0.28730243211205864, |
| "kl": 0.0511474609375, |
| "learning_rate": 1e-05, |
| "loss": 0.002, |
| "reward": 0.2828526049852371, |
| "reward_std": 0.03443083353340626, |
| "rewards/clip_reward": 0.2828526049852371, |
| "step": 194 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.006010819475055099, |
| "grad_norm": 0.25755366719945655, |
| "kl": 0.05511474609375, |
| "learning_rate": 1e-05, |
| "loss": 0.0022, |
| "reward": 0.30451397597789764, |
| "reward_std": 0.030547112692147493, |
| "rewards/clip_reward": 0.30451397597789764, |
| "step": 195 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.006041644190311792, |
| "grad_norm": 0.28409469826599004, |
| "kl": 0.0528564453125, |
| "learning_rate": 1e-05, |
| "loss": 0.0021, |
| "reward": 0.3040374740958214, |
| "reward_std": 0.03506700927391648, |
| "rewards/clip_reward": 0.3040374740958214, |
| "step": 196 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.006072468905568485, |
| "grad_norm": 0.24631301394880073, |
| "kl": 0.05291748046875, |
| "learning_rate": 1e-05, |
| "loss": 0.0021, |
| "reward": 0.3064323738217354, |
| "reward_std": 0.03699143324047327, |
| "rewards/clip_reward": 0.3064323738217354, |
| "step": 197 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.006103293620825178, |
| "grad_norm": 0.4776380300716616, |
| "kl": 0.06048583984375, |
| "learning_rate": 1e-05, |
| "loss": 0.0024, |
| "reward": 0.31653470546007156, |
| "reward_std": 0.03443007543683052, |
| "rewards/clip_reward": 0.31653470546007156, |
| "step": 198 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.006134118336081871, |
| "grad_norm": 0.2717998039709889, |
| "kl": 0.05657958984375, |
| "learning_rate": 1e-05, |
| "loss": 0.0023, |
| "reward": 0.2982766330242157, |
| "reward_std": 0.030803233850747347, |
| "rewards/clip_reward": 0.2982766330242157, |
| "step": 199 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.006164943051338564, |
| "grad_norm": 0.28573048152617825, |
| "kl": 0.05755615234375, |
| "learning_rate": 1e-05, |
| "loss": 0.0023, |
| "reward": 0.2895183190703392, |
| "reward_std": 0.031674451660364866, |
| "rewards/clip_reward": 0.2895183190703392, |
| "step": 200 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.0061957677665952565, |
| "grad_norm": 0.3012206807393158, |
| "kl": 0.055908203125, |
| "learning_rate": 1e-05, |
| "loss": 0.0022, |
| "reward": 0.3014395534992218, |
| "reward_std": 0.03418161487206817, |
| "rewards/clip_reward": 0.3014395534992218, |
| "step": 201 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.0062265924818519485, |
| "grad_norm": 0.24480682812310384, |
| "kl": 0.057373046875, |
| "learning_rate": 1e-05, |
| "loss": 0.0023, |
| "reward": 0.28224068135023117, |
| "reward_std": 0.03003824595361948, |
| "rewards/clip_reward": 0.28224068135023117, |
| "step": 202 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.006257417197108641, |
| "grad_norm": 0.2531535146591285, |
| "kl": 0.05511474609375, |
| "learning_rate": 1e-05, |
| "loss": 0.0022, |
| "reward": 0.30891023576259613, |
| "reward_std": 0.03753681713715196, |
| "rewards/clip_reward": 0.30891023576259613, |
| "step": 203 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.006288241912365334, |
| "grad_norm": 0.23120591809436197, |
| "kl": 0.05841064453125, |
| "learning_rate": 1e-05, |
| "loss": 0.0023, |
| "reward": 0.3163115456700325, |
| "reward_std": 0.02788963308557868, |
| "rewards/clip_reward": 0.3163115456700325, |
| "step": 204 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.006319066627622027, |
| "grad_norm": 0.23022883641873151, |
| "kl": 0.056884765625, |
| "learning_rate": 1e-05, |
| "loss": 0.0023, |
| "reward": 0.2952466085553169, |
| "reward_std": 0.03618847485631704, |
| "rewards/clip_reward": 0.2952466085553169, |
| "step": 205 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.00634989134287872, |
| "grad_norm": 0.2604310735559812, |
| "kl": 0.05767822265625, |
| "learning_rate": 1e-05, |
| "loss": 0.0023, |
| "reward": 0.30279867351055145, |
| "reward_std": 0.03358080657199025, |
| "rewards/clip_reward": 0.30279867351055145, |
| "step": 206 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.006380716058135413, |
| "grad_norm": 0.3065404532343023, |
| "kl": 0.06097412109375, |
| "learning_rate": 1e-05, |
| "loss": 0.0024, |
| "reward": 0.2905031070113182, |
| "reward_std": 0.03163592051714659, |
| "rewards/clip_reward": 0.2905031070113182, |
| "step": 207 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.006411540773392106, |
| "grad_norm": 0.2302581935455144, |
| "kl": 0.06036376953125, |
| "learning_rate": 1e-05, |
| "loss": 0.0024, |
| "reward": 0.3036247417330742, |
| "reward_std": 0.04002736788243055, |
| "rewards/clip_reward": 0.3036247417330742, |
| "step": 208 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.006442365488648799, |
| "grad_norm": 0.22939030410663466, |
| "kl": 0.0615234375, |
| "learning_rate": 1e-05, |
| "loss": 0.0025, |
| "reward": 0.3065522834658623, |
| "reward_std": 0.035838291980326176, |
| "rewards/clip_reward": 0.3065522834658623, |
| "step": 209 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.006473190203905492, |
| "grad_norm": 1.1535119363825117, |
| "kl": 0.171630859375, |
| "learning_rate": 1e-05, |
| "loss": 0.0068, |
| "reward": 0.3034718856215477, |
| "reward_std": 0.028684359975159168, |
| "rewards/clip_reward": 0.3034718856215477, |
| "step": 210 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.0065040149191621845, |
| "grad_norm": 0.475789174353479, |
| "kl": 0.06304931640625, |
| "learning_rate": 1e-05, |
| "loss": 0.0025, |
| "reward": 0.3188106268644333, |
| "reward_std": 0.03359405370429158, |
| "rewards/clip_reward": 0.3188106268644333, |
| "step": 211 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.006534839634418877, |
| "grad_norm": 0.30958280497728724, |
| "kl": 0.06011962890625, |
| "learning_rate": 1e-05, |
| "loss": 0.0024, |
| "reward": 0.31080956757068634, |
| "reward_std": 0.03438552375882864, |
| "rewards/clip_reward": 0.31080956757068634, |
| "step": 212 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.0065656643496755695, |
| "grad_norm": 0.2945478977489792, |
| "kl": 0.06329345703125, |
| "learning_rate": 1e-05, |
| "loss": 0.0025, |
| "reward": 0.29854634404182434, |
| "reward_std": 0.03470074059441686, |
| "rewards/clip_reward": 0.29854634404182434, |
| "step": 213 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.006596489064932262, |
| "grad_norm": 0.311983746378144, |
| "kl": 0.061279296875, |
| "learning_rate": 1e-05, |
| "loss": 0.0025, |
| "reward": 0.310188353061676, |
| "reward_std": 0.04189980635419488, |
| "rewards/clip_reward": 0.310188353061676, |
| "step": 214 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.006627313780188955, |
| "grad_norm": 0.26157966652434694, |
| "kl": 0.06146240234375, |
| "learning_rate": 1e-05, |
| "loss": 0.0025, |
| "reward": 0.31515590101480484, |
| "reward_std": 0.0373789188452065, |
| "rewards/clip_reward": 0.31515590101480484, |
| "step": 215 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.006658138495445648, |
| "grad_norm": 0.3427812618986616, |
| "kl": 0.06329345703125, |
| "learning_rate": 1e-05, |
| "loss": 0.0025, |
| "reward": 0.31115034967660904, |
| "reward_std": 0.03764009568840265, |
| "rewards/clip_reward": 0.31115034967660904, |
| "step": 216 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.006688963210702341, |
| "grad_norm": 0.31645931728689874, |
| "kl": 0.05767822265625, |
| "learning_rate": 1e-05, |
| "loss": 0.0023, |
| "reward": 0.28984929621219635, |
| "reward_std": 0.03542311815544963, |
| "rewards/clip_reward": 0.28984929621219635, |
| "step": 217 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.006719787925959034, |
| "grad_norm": 0.2228608743429633, |
| "kl": 0.05902099609375, |
| "learning_rate": 1e-05, |
| "loss": 0.0024, |
| "reward": 0.3080795630812645, |
| "reward_std": 0.03324572555720806, |
| "rewards/clip_reward": 0.3080795630812645, |
| "step": 218 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.006750612641215727, |
| "grad_norm": 0.6337063521534685, |
| "kl": 0.06610107421875, |
| "learning_rate": 1e-05, |
| "loss": 0.0026, |
| "reward": 0.3004284203052521, |
| "reward_std": 0.030279683880507946, |
| "rewards/clip_reward": 0.3004284203052521, |
| "step": 219 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.00678143735647242, |
| "grad_norm": 0.35877724546468126, |
| "kl": 0.0599365234375, |
| "learning_rate": 1e-05, |
| "loss": 0.0024, |
| "reward": 0.2890964075922966, |
| "reward_std": 0.03512955084443092, |
| "rewards/clip_reward": 0.2890964075922966, |
| "step": 220 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.006812262071729113, |
| "grad_norm": 0.28709775498334184, |
| "kl": 0.06353759765625, |
| "learning_rate": 1e-05, |
| "loss": 0.0025, |
| "reward": 0.3098203092813492, |
| "reward_std": 0.03407229436561465, |
| "rewards/clip_reward": 0.3098203092813492, |
| "step": 221 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.0068430867869858055, |
| "grad_norm": 0.2587966892355202, |
| "kl": 0.0631103515625, |
| "learning_rate": 1e-05, |
| "loss": 0.0025, |
| "reward": 0.2910105660557747, |
| "reward_std": 0.03544299304485321, |
| "rewards/clip_reward": 0.2910105660557747, |
| "step": 222 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.006873911502242498, |
| "grad_norm": 0.20375336506340044, |
| "kl": 0.06231689453125, |
| "learning_rate": 1e-05, |
| "loss": 0.0025, |
| "reward": 0.3012443706393242, |
| "reward_std": 0.03373287199065089, |
| "rewards/clip_reward": 0.3012443706393242, |
| "step": 223 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.006904736217499191, |
| "grad_norm": 0.22796858972291945, |
| "kl": 0.058349609375, |
| "learning_rate": 1e-05, |
| "loss": 0.0023, |
| "reward": 0.27249932289123535, |
| "reward_std": 0.034992088098078966, |
| "rewards/clip_reward": 0.27249932289123535, |
| "step": 224 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.006935560932755883, |
| "grad_norm": 0.23923560288044327, |
| "kl": 0.05914306640625, |
| "learning_rate": 1e-05, |
| "loss": 0.0024, |
| "reward": 0.30393005907535553, |
| "reward_std": 0.04046585503965616, |
| "rewards/clip_reward": 0.30393005907535553, |
| "step": 225 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.006966385648012576, |
| "grad_norm": 0.24781024716109332, |
| "kl": 0.05828857421875, |
| "learning_rate": 1e-05, |
| "loss": 0.0023, |
| "reward": 0.29162784665822983, |
| "reward_std": 0.03501830715686083, |
| "rewards/clip_reward": 0.29162784665822983, |
| "step": 226 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.006997210363269269, |
| "grad_norm": 0.2709603773452262, |
| "kl": 0.0609130859375, |
| "learning_rate": 1e-05, |
| "loss": 0.0024, |
| "reward": 0.2914000228047371, |
| "reward_std": 0.030948857311159372, |
| "rewards/clip_reward": 0.2914000228047371, |
| "step": 227 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.007028035078525962, |
| "grad_norm": 0.21567469316732074, |
| "kl": 0.0567626953125, |
| "learning_rate": 1e-05, |
| "loss": 0.0023, |
| "reward": 0.3019692078232765, |
| "reward_std": 0.038113043643534184, |
| "rewards/clip_reward": 0.3019692078232765, |
| "step": 228 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.007058859793782655, |
| "grad_norm": 0.223865539956382, |
| "kl": 0.05474853515625, |
| "learning_rate": 1e-05, |
| "loss": 0.0022, |
| "reward": 0.276552177965641, |
| "reward_std": 0.03583432175219059, |
| "rewards/clip_reward": 0.276552177965641, |
| "step": 229 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.007089684509039348, |
| "grad_norm": 0.23022368412719968, |
| "kl": 0.05950927734375, |
| "learning_rate": 1e-05, |
| "loss": 0.0024, |
| "reward": 0.30329927057027817, |
| "reward_std": 0.035789184272289276, |
| "rewards/clip_reward": 0.30329927057027817, |
| "step": 230 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.007120509224296041, |
| "grad_norm": 0.2872615168609158, |
| "kl": 0.05938720703125, |
| "learning_rate": 1e-05, |
| "loss": 0.0024, |
| "reward": 0.3115493804216385, |
| "reward_std": 0.03708712290972471, |
| "rewards/clip_reward": 0.3115493804216385, |
| "step": 231 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.0071513339395527335, |
| "grad_norm": 0.34806552807362773, |
| "kl": 0.06298828125, |
| "learning_rate": 1e-05, |
| "loss": 0.0025, |
| "reward": 0.3054001107811928, |
| "reward_std": 0.034296643920242786, |
| "rewards/clip_reward": 0.3054001107811928, |
| "step": 232 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.007182158654809426, |
| "grad_norm": 0.286918714200751, |
| "kl": 0.06072998046875, |
| "learning_rate": 1e-05, |
| "loss": 0.0024, |
| "reward": 0.299840584397316, |
| "reward_std": 0.029763209633529186, |
| "rewards/clip_reward": 0.299840584397316, |
| "step": 233 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.007212983370066119, |
| "grad_norm": 0.2246601984953947, |
| "kl": 0.0570068359375, |
| "learning_rate": 1e-05, |
| "loss": 0.0023, |
| "reward": 0.3116656318306923, |
| "reward_std": 0.038989217951893806, |
| "rewards/clip_reward": 0.3116656318306923, |
| "step": 234 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.007243808085322812, |
| "grad_norm": 0.2300241445857201, |
| "kl": 0.06207275390625, |
| "learning_rate": 1e-05, |
| "loss": 0.0025, |
| "reward": 0.3119165748357773, |
| "reward_std": 0.030884731095284224, |
| "rewards/clip_reward": 0.3119165748357773, |
| "step": 235 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.007274632800579504, |
| "grad_norm": 0.37606313252297296, |
| "kl": 0.06292724609375, |
| "learning_rate": 1e-05, |
| "loss": 0.0025, |
| "reward": 0.2938763126730919, |
| "reward_std": 0.03854981064796448, |
| "rewards/clip_reward": 0.2938763126730919, |
| "step": 236 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.007305457515836197, |
| "grad_norm": 0.27390469202393175, |
| "kl": 0.06268310546875, |
| "learning_rate": 1e-05, |
| "loss": 0.0025, |
| "reward": 0.30878350138664246, |
| "reward_std": 0.0367919635027647, |
| "rewards/clip_reward": 0.30878350138664246, |
| "step": 237 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.00733628223109289, |
| "grad_norm": 0.3923664539501469, |
| "kl": 0.06011962890625, |
| "learning_rate": 1e-05, |
| "loss": 0.0024, |
| "reward": 0.31253431737422943, |
| "reward_std": 0.03306168969720602, |
| "rewards/clip_reward": 0.31253431737422943, |
| "step": 238 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.007367106946349583, |
| "grad_norm": 19.999839999172977, |
| "kl": 0.11572265625, |
| "learning_rate": 1e-05, |
| "loss": 0.0046, |
| "reward": 0.2926497831940651, |
| "reward_std": 0.03439700556918979, |
| "rewards/clip_reward": 0.2926497831940651, |
| "step": 239 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.007397931661606276, |
| "grad_norm": 0.3274619779549125, |
| "kl": 0.0623779296875, |
| "learning_rate": 1e-05, |
| "loss": 0.0025, |
| "reward": 0.31169167160987854, |
| "reward_std": 0.03383385669440031, |
| "rewards/clip_reward": 0.31169167160987854, |
| "step": 240 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.007428756376862969, |
| "grad_norm": 0.3930715184021957, |
| "kl": 0.064453125, |
| "learning_rate": 1e-05, |
| "loss": 0.0026, |
| "reward": 0.3149360120296478, |
| "reward_std": 0.036022236570715904, |
| "rewards/clip_reward": 0.3149360120296478, |
| "step": 241 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.007459581092119662, |
| "grad_norm": 0.23471411051429383, |
| "kl": 0.0631103515625, |
| "learning_rate": 1e-05, |
| "loss": 0.0025, |
| "reward": 0.2915840148925781, |
| "reward_std": 0.04799613729119301, |
| "rewards/clip_reward": 0.2915840148925781, |
| "step": 242 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.0074904058073763545, |
| "grad_norm": 0.25074369126631646, |
| "kl": 0.06268310546875, |
| "learning_rate": 1e-05, |
| "loss": 0.0025, |
| "reward": 0.30563782155513763, |
| "reward_std": 0.036503950599581, |
| "rewards/clip_reward": 0.30563782155513763, |
| "step": 243 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.007521230522633047, |
| "grad_norm": 0.25106966232242506, |
| "kl": 0.06280517578125, |
| "learning_rate": 1e-05, |
| "loss": 0.0025, |
| "reward": 0.31641364842653275, |
| "reward_std": 0.042671964969486, |
| "rewards/clip_reward": 0.31641364842653275, |
| "step": 244 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.00755205523788974, |
| "grad_norm": 0.24863565809024027, |
| "kl": 0.06585693359375, |
| "learning_rate": 1e-05, |
| "loss": 0.0026, |
| "reward": 0.29705505073070526, |
| "reward_std": 0.02980767609551549, |
| "rewards/clip_reward": 0.29705505073070526, |
| "step": 245 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.007582879953146433, |
| "grad_norm": 0.3215906875694069, |
| "kl": 0.0626220703125, |
| "learning_rate": 1e-05, |
| "loss": 0.0025, |
| "reward": 0.2966236397624016, |
| "reward_std": 0.032666172832250595, |
| "rewards/clip_reward": 0.2966236397624016, |
| "step": 246 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.007613704668403126, |
| "grad_norm": 0.25802005781105997, |
| "kl": 0.0655517578125, |
| "learning_rate": 1e-05, |
| "loss": 0.0026, |
| "reward": 0.29460806399583817, |
| "reward_std": 0.03424982167780399, |
| "rewards/clip_reward": 0.29460806399583817, |
| "step": 247 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.007644529383659818, |
| "grad_norm": 0.41480007000983, |
| "kl": 0.06292724609375, |
| "learning_rate": 1e-05, |
| "loss": 0.0025, |
| "reward": 0.30160878598690033, |
| "reward_std": 0.03246263647451997, |
| "rewards/clip_reward": 0.30160878598690033, |
| "step": 248 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.007675354098916511, |
| "grad_norm": 0.38518441595490016, |
| "kl": 0.07415771484375, |
| "learning_rate": 1e-05, |
| "loss": 0.003, |
| "reward": 0.2882128059864044, |
| "reward_std": 0.03637995757162571, |
| "rewards/clip_reward": 0.2882128059864044, |
| "step": 249 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.007706178814173204, |
| "grad_norm": 0.2296837125573533, |
| "kl": 0.06439208984375, |
| "learning_rate": 1e-05, |
| "loss": 0.0026, |
| "reward": 0.31887152045965195, |
| "reward_std": 0.031468767672777176, |
| "rewards/clip_reward": 0.31887152045965195, |
| "step": 250 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.007737003529429897, |
| "grad_norm": 0.2771302981384482, |
| "kl": 0.06256103515625, |
| "learning_rate": 1e-05, |
| "loss": 0.0025, |
| "reward": 0.29794805496931076, |
| "reward_std": 0.03718498535454273, |
| "rewards/clip_reward": 0.29794805496931076, |
| "step": 251 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.00776782824468659, |
| "grad_norm": 0.26425224371180456, |
| "kl": 0.0660400390625, |
| "learning_rate": 1e-05, |
| "loss": 0.0026, |
| "reward": 0.31550634652376175, |
| "reward_std": 0.03392920456826687, |
| "rewards/clip_reward": 0.31550634652376175, |
| "step": 252 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.0077986529599432825, |
| "grad_norm": 0.3039933530405912, |
| "kl": 0.06451416015625, |
| "learning_rate": 1e-05, |
| "loss": 0.0026, |
| "reward": 0.31224576383829117, |
| "reward_std": 0.03579189581796527, |
| "rewards/clip_reward": 0.31224576383829117, |
| "step": 253 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.007829477675199975, |
| "grad_norm": 0.21264552398464287, |
| "kl": 0.066650390625, |
| "learning_rate": 1e-05, |
| "loss": 0.0027, |
| "reward": 0.30356816947460175, |
| "reward_std": 0.03573842905461788, |
| "rewards/clip_reward": 0.30356816947460175, |
| "step": 254 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.007860302390456668, |
| "grad_norm": 0.23510191594709257, |
| "kl": 0.06463623046875, |
| "learning_rate": 1e-05, |
| "loss": 0.0026, |
| "reward": 0.29867023229599, |
| "reward_std": 0.03537313872948289, |
| "rewards/clip_reward": 0.29867023229599, |
| "step": 255 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.007891127105713361, |
| "grad_norm": 0.8798380521542469, |
| "kl": 0.0672607421875, |
| "learning_rate": 1e-05, |
| "loss": 0.0027, |
| "reward": 0.2907034158706665, |
| "reward_std": 0.03144656075164676, |
| "rewards/clip_reward": 0.2907034158706665, |
| "step": 256 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.007921951820970054, |
| "grad_norm": 0.30489147633127095, |
| "kl": 0.06390380859375, |
| "learning_rate": 1e-05, |
| "loss": 0.0026, |
| "reward": 0.2994539961218834, |
| "reward_std": 0.036147382110357285, |
| "rewards/clip_reward": 0.2994539961218834, |
| "step": 257 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.007952776536226747, |
| "grad_norm": 0.2724891171334766, |
| "kl": 0.07373046875, |
| "learning_rate": 1e-05, |
| "loss": 0.003, |
| "reward": 0.29950564354658127, |
| "reward_std": 0.031049554236233234, |
| "rewards/clip_reward": 0.29950564354658127, |
| "step": 258 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.00798360125148344, |
| "grad_norm": 0.2257138587782837, |
| "kl": 0.0606689453125, |
| "learning_rate": 1e-05, |
| "loss": 0.0024, |
| "reward": 0.31028740108013153, |
| "reward_std": 0.03820298006758094, |
| "rewards/clip_reward": 0.31028740108013153, |
| "step": 259 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.008014425966740133, |
| "grad_norm": 0.23904222780230097, |
| "kl": 0.06512451171875, |
| "learning_rate": 1e-05, |
| "loss": 0.0026, |
| "reward": 0.30369560420513153, |
| "reward_std": 0.034888788126409054, |
| "rewards/clip_reward": 0.30369560420513153, |
| "step": 260 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.008045250681996826, |
| "grad_norm": 0.26551280919013587, |
| "kl": 0.069091796875, |
| "learning_rate": 1e-05, |
| "loss": 0.0028, |
| "reward": 0.3005470857024193, |
| "reward_std": 0.03615998663008213, |
| "rewards/clip_reward": 0.3005470857024193, |
| "step": 261 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.008076075397253519, |
| "grad_norm": 0.2641814708454766, |
| "kl": 0.067138671875, |
| "learning_rate": 1e-05, |
| "loss": 0.0027, |
| "reward": 0.30573634058237076, |
| "reward_std": 0.03119312133640051, |
| "rewards/clip_reward": 0.30573634058237076, |
| "step": 262 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.008106900112510211, |
| "grad_norm": 0.2538072823054899, |
| "kl": 0.067626953125, |
| "learning_rate": 1e-05, |
| "loss": 0.0027, |
| "reward": 0.28952275961637497, |
| "reward_std": 0.02620957838371396, |
| "rewards/clip_reward": 0.28952275961637497, |
| "step": 263 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.008137724827766904, |
| "grad_norm": 0.27840725784727466, |
| "kl": 0.0692138671875, |
| "learning_rate": 1e-05, |
| "loss": 0.0028, |
| "reward": 0.31542008370161057, |
| "reward_std": 0.037005496211349964, |
| "rewards/clip_reward": 0.31542008370161057, |
| "step": 264 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.008168549543023595, |
| "grad_norm": 0.3168534549683751, |
| "kl": 0.069091796875, |
| "learning_rate": 1e-05, |
| "loss": 0.0028, |
| "reward": 0.2891938388347626, |
| "reward_std": 0.032484165858477354, |
| "rewards/clip_reward": 0.2891938388347626, |
| "step": 265 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.008199374258280288, |
| "grad_norm": 0.34875471295741794, |
| "kl": 0.071533203125, |
| "learning_rate": 1e-05, |
| "loss": 0.0029, |
| "reward": 0.3205078914761543, |
| "reward_std": 0.042449533008039, |
| "rewards/clip_reward": 0.3205078914761543, |
| "step": 266 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.008230198973536981, |
| "grad_norm": 0.2913604348839751, |
| "kl": 0.0697021484375, |
| "learning_rate": 1e-05, |
| "loss": 0.0028, |
| "reward": 0.3089185729622841, |
| "reward_std": 0.03769417991861701, |
| "rewards/clip_reward": 0.3089185729622841, |
| "step": 267 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.008261023688793674, |
| "grad_norm": 0.2408505354217034, |
| "kl": 0.06768798828125, |
| "learning_rate": 1e-05, |
| "loss": 0.0027, |
| "reward": 0.30584635585546494, |
| "reward_std": 0.03537123091518879, |
| "rewards/clip_reward": 0.30584635585546494, |
| "step": 268 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.008291848404050367, |
| "grad_norm": 0.23962011506896905, |
| "kl": 0.0709228515625, |
| "learning_rate": 1e-05, |
| "loss": 0.0028, |
| "reward": 0.2843191996216774, |
| "reward_std": 0.030163435731083155, |
| "rewards/clip_reward": 0.2843191996216774, |
| "step": 269 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.00832267311930706, |
| "grad_norm": 0.267565227028639, |
| "kl": 0.072021484375, |
| "learning_rate": 1e-05, |
| "loss": 0.0029, |
| "reward": 0.30715377628803253, |
| "reward_std": 0.03210371499881148, |
| "rewards/clip_reward": 0.30715377628803253, |
| "step": 270 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.008353497834563753, |
| "grad_norm": 0.26478125349454257, |
| "kl": 0.071533203125, |
| "learning_rate": 1e-05, |
| "loss": 0.0029, |
| "reward": 0.3105710372328758, |
| "reward_std": 0.03365481458604336, |
| "rewards/clip_reward": 0.3105710372328758, |
| "step": 271 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.008384322549820446, |
| "grad_norm": 0.25310448605992364, |
| "kl": 0.0711669921875, |
| "learning_rate": 1e-05, |
| "loss": 0.0028, |
| "reward": 0.2988099977374077, |
| "reward_std": 0.03356699598953128, |
| "rewards/clip_reward": 0.2988099977374077, |
| "step": 272 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.008415147265077139, |
| "grad_norm": 0.6415597863522857, |
| "kl": 0.0704345703125, |
| "learning_rate": 1e-05, |
| "loss": 0.0028, |
| "reward": 0.3161829188466072, |
| "reward_std": 0.03346774587407708, |
| "rewards/clip_reward": 0.3161829188466072, |
| "step": 273 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.008445971980333832, |
| "grad_norm": 0.2963215633498198, |
| "kl": 0.0755615234375, |
| "learning_rate": 1e-05, |
| "loss": 0.003, |
| "reward": 0.29340942949056625, |
| "reward_std": 0.029817141592502594, |
| "rewards/clip_reward": 0.29340942949056625, |
| "step": 274 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.008476796695590524, |
| "grad_norm": 0.28624937293920455, |
| "kl": 0.07177734375, |
| "learning_rate": 1e-05, |
| "loss": 0.0029, |
| "reward": 0.29413383454084396, |
| "reward_std": 0.03951750136911869, |
| "rewards/clip_reward": 0.29413383454084396, |
| "step": 275 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.008507621410847217, |
| "grad_norm": 0.3584737945543695, |
| "kl": 0.065185546875, |
| "learning_rate": 1e-05, |
| "loss": 0.0026, |
| "reward": 0.30792876332998276, |
| "reward_std": 0.03842892590910196, |
| "rewards/clip_reward": 0.30792876332998276, |
| "step": 276 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.00853844612610391, |
| "grad_norm": 0.41014371409185196, |
| "kl": 0.086669921875, |
| "learning_rate": 1e-05, |
| "loss": 0.0035, |
| "reward": 0.30880793929100037, |
| "reward_std": 0.03384514432400465, |
| "rewards/clip_reward": 0.30880793929100037, |
| "step": 277 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.008569270841360603, |
| "grad_norm": 0.6923907529205436, |
| "kl": 0.07275390625, |
| "learning_rate": 1e-05, |
| "loss": 0.0029, |
| "reward": 0.30738433450460434, |
| "reward_std": 0.030436881817877293, |
| "rewards/clip_reward": 0.30738433450460434, |
| "step": 278 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.008600095556617296, |
| "grad_norm": 0.2508894379559603, |
| "kl": 0.06707763671875, |
| "learning_rate": 1e-05, |
| "loss": 0.0027, |
| "reward": 0.29732104390859604, |
| "reward_std": 0.03281824570149183, |
| "rewards/clip_reward": 0.29732104390859604, |
| "step": 279 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.008630920271873989, |
| "grad_norm": 0.21788541287282853, |
| "kl": 0.066650390625, |
| "learning_rate": 1e-05, |
| "loss": 0.0027, |
| "reward": 0.3087487816810608, |
| "reward_std": 0.03466750681400299, |
| "rewards/clip_reward": 0.3087487816810608, |
| "step": 280 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.008661744987130682, |
| "grad_norm": 0.3677827474941058, |
| "kl": 0.077880859375, |
| "learning_rate": 1e-05, |
| "loss": 0.0031, |
| "reward": 0.3030627593398094, |
| "reward_std": 0.0359897562302649, |
| "rewards/clip_reward": 0.3030627593398094, |
| "step": 281 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.008692569702387375, |
| "grad_norm": 0.314089431725954, |
| "kl": 0.07061767578125, |
| "learning_rate": 1e-05, |
| "loss": 0.0028, |
| "reward": 0.30142877995967865, |
| "reward_std": 0.032457209192216396, |
| "rewards/clip_reward": 0.30142877995967865, |
| "step": 282 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.008723394417644068, |
| "grad_norm": 0.2690848375206651, |
| "kl": 0.06787109375, |
| "learning_rate": 1e-05, |
| "loss": 0.0027, |
| "reward": 0.29926204681396484, |
| "reward_std": 0.046972066164016724, |
| "rewards/clip_reward": 0.29926204681396484, |
| "step": 283 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.00875421913290076, |
| "grad_norm": 0.5301348946984951, |
| "kl": 0.06787109375, |
| "learning_rate": 1e-05, |
| "loss": 0.0027, |
| "reward": 0.3038446083664894, |
| "reward_std": 0.03355177864432335, |
| "rewards/clip_reward": 0.3038446083664894, |
| "step": 284 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.008785043848157453, |
| "grad_norm": 0.3640215089911138, |
| "kl": 0.0740966796875, |
| "learning_rate": 1e-05, |
| "loss": 0.003, |
| "reward": 0.29393448680639267, |
| "reward_std": 0.029715597163885832, |
| "rewards/clip_reward": 0.29393448680639267, |
| "step": 285 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.008815868563414146, |
| "grad_norm": 0.25955791764443836, |
| "kl": 0.0677490234375, |
| "learning_rate": 1e-05, |
| "loss": 0.0027, |
| "reward": 0.29612670093774796, |
| "reward_std": 0.03200107906013727, |
| "rewards/clip_reward": 0.29612670093774796, |
| "step": 286 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.008846693278670839, |
| "grad_norm": 0.2840084484574733, |
| "kl": 0.0736083984375, |
| "learning_rate": 1e-05, |
| "loss": 0.0029, |
| "reward": 0.28986190259456635, |
| "reward_std": 0.03536898456513882, |
| "rewards/clip_reward": 0.28986190259456635, |
| "step": 287 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.00887751799392753, |
| "grad_norm": 0.2504845781925923, |
| "kl": 0.0736083984375, |
| "learning_rate": 1e-05, |
| "loss": 0.0029, |
| "reward": 0.28574925661087036, |
| "reward_std": 0.029350985307246447, |
| "rewards/clip_reward": 0.28574925661087036, |
| "step": 288 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.008908342709184223, |
| "grad_norm": 1.9538978684731851, |
| "kl": 0.073486328125, |
| "learning_rate": 1e-05, |
| "loss": 0.0029, |
| "reward": 0.2989484444260597, |
| "reward_std": 0.037056506145745516, |
| "rewards/clip_reward": 0.2989484444260597, |
| "step": 289 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.008939167424440916, |
| "grad_norm": 0.29595148973162033, |
| "kl": 0.068359375, |
| "learning_rate": 1e-05, |
| "loss": 0.0027, |
| "reward": 0.2984117269515991, |
| "reward_std": 0.04048395995050669, |
| "rewards/clip_reward": 0.2984117269515991, |
| "step": 290 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.008969992139697609, |
| "grad_norm": 0.46786991142607903, |
| "kl": 0.0726318359375, |
| "learning_rate": 1e-05, |
| "loss": 0.0029, |
| "reward": 0.29787522554397583, |
| "reward_std": 0.03701747301965952, |
| "rewards/clip_reward": 0.29787522554397583, |
| "step": 291 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.009000816854954302, |
| "grad_norm": 0.2415371991232022, |
| "kl": 0.0673828125, |
| "learning_rate": 1e-05, |
| "loss": 0.0027, |
| "reward": 0.2959031015634537, |
| "reward_std": 0.03375337179750204, |
| "rewards/clip_reward": 0.2959031015634537, |
| "step": 292 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.009031641570210995, |
| "grad_norm": 0.2509472132536602, |
| "kl": 0.0736083984375, |
| "learning_rate": 1e-05, |
| "loss": 0.003, |
| "reward": 0.3059312403202057, |
| "reward_std": 0.029257553163915873, |
| "rewards/clip_reward": 0.3059312403202057, |
| "step": 293 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.009062466285467688, |
| "grad_norm": 0.27142301376417255, |
| "kl": 0.0682373046875, |
| "learning_rate": 1e-05, |
| "loss": 0.0027, |
| "reward": 0.302873820066452, |
| "reward_std": 0.03212234936654568, |
| "rewards/clip_reward": 0.302873820066452, |
| "step": 294 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.00909329100072438, |
| "grad_norm": 0.6714767650621216, |
| "kl": 0.0770263671875, |
| "learning_rate": 1e-05, |
| "loss": 0.0031, |
| "reward": 0.302773617208004, |
| "reward_std": 0.037844753824174404, |
| "rewards/clip_reward": 0.302773617208004, |
| "step": 295 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.009124115715981073, |
| "grad_norm": 0.3313051439529105, |
| "kl": 0.071533203125, |
| "learning_rate": 1e-05, |
| "loss": 0.0029, |
| "reward": 0.3004399463534355, |
| "reward_std": 0.03778542298823595, |
| "rewards/clip_reward": 0.3004399463534355, |
| "step": 296 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.009154940431237766, |
| "grad_norm": 0.2861755625116893, |
| "kl": 0.0765380859375, |
| "learning_rate": 1e-05, |
| "loss": 0.0031, |
| "reward": 0.2957393676042557, |
| "reward_std": 0.03911779401823878, |
| "rewards/clip_reward": 0.2957393676042557, |
| "step": 297 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.00918576514649446, |
| "grad_norm": 0.3643143782064657, |
| "kl": 0.0721435546875, |
| "learning_rate": 1e-05, |
| "loss": 0.0029, |
| "reward": 0.3204945772886276, |
| "reward_std": 0.04190400242805481, |
| "rewards/clip_reward": 0.3204945772886276, |
| "step": 298 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.009216589861751152, |
| "grad_norm": 0.3506675665500401, |
| "kl": 0.0740966796875, |
| "learning_rate": 1e-05, |
| "loss": 0.003, |
| "reward": 0.30117396265268326, |
| "reward_std": 0.03484671004116535, |
| "rewards/clip_reward": 0.30117396265268326, |
| "step": 299 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.009247414577007845, |
| "grad_norm": 0.4873394391686336, |
| "kl": 0.0721435546875, |
| "learning_rate": 1e-05, |
| "loss": 0.0029, |
| "reward": 0.3100610002875328, |
| "reward_std": 0.03472679574042559, |
| "rewards/clip_reward": 0.3100610002875328, |
| "step": 300 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.009278239292264538, |
| "grad_norm": 0.4157803977878561, |
| "kl": 0.072021484375, |
| "learning_rate": 1e-05, |
| "loss": 0.0029, |
| "reward": 0.30167850106954575, |
| "reward_std": 0.031223418191075325, |
| "rewards/clip_reward": 0.30167850106954575, |
| "step": 301 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.00930906400752123, |
| "grad_norm": 0.3797545814324745, |
| "kl": 0.0731201171875, |
| "learning_rate": 1e-05, |
| "loss": 0.0029, |
| "reward": 0.31539446860551834, |
| "reward_std": 0.041049075312912464, |
| "rewards/clip_reward": 0.31539446860551834, |
| "step": 302 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.009339888722777924, |
| "grad_norm": 0.2743776739836363, |
| "kl": 0.0791015625, |
| "learning_rate": 1e-05, |
| "loss": 0.0032, |
| "reward": 0.30482209473848343, |
| "reward_std": 0.03603475447744131, |
| "rewards/clip_reward": 0.30482209473848343, |
| "step": 303 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.009370713438034617, |
| "grad_norm": 0.3102618921636113, |
| "kl": 0.0723876953125, |
| "learning_rate": 1e-05, |
| "loss": 0.0029, |
| "reward": 0.3093208372592926, |
| "reward_std": 0.03316945396363735, |
| "rewards/clip_reward": 0.3093208372592926, |
| "step": 304 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.00940153815329131, |
| "grad_norm": 0.22184784039477515, |
| "kl": 0.0692138671875, |
| "learning_rate": 1e-05, |
| "loss": 0.0028, |
| "reward": 0.3097628057003021, |
| "reward_std": 0.034655320923775434, |
| "rewards/clip_reward": 0.3097628057003021, |
| "step": 305 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.009432362868548002, |
| "grad_norm": 0.2679648313292432, |
| "kl": 0.0689697265625, |
| "learning_rate": 1e-05, |
| "loss": 0.0028, |
| "reward": 0.31062986701726913, |
| "reward_std": 0.0346312508918345, |
| "rewards/clip_reward": 0.31062986701726913, |
| "step": 306 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.009463187583804695, |
| "grad_norm": 0.2091622067188512, |
| "kl": 0.068115234375, |
| "learning_rate": 1e-05, |
| "loss": 0.0027, |
| "reward": 0.2891889810562134, |
| "reward_std": 0.03427890222519636, |
| "rewards/clip_reward": 0.2891889810562134, |
| "step": 307 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.009494012299061388, |
| "grad_norm": 0.380322887324008, |
| "kl": 0.077392578125, |
| "learning_rate": 1e-05, |
| "loss": 0.0031, |
| "reward": 0.2787666954100132, |
| "reward_std": 0.030210648197680712, |
| "rewards/clip_reward": 0.2787666954100132, |
| "step": 308 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.009524837014318081, |
| "grad_norm": 0.2639105229044439, |
| "kl": 0.06744384765625, |
| "learning_rate": 1e-05, |
| "loss": 0.0027, |
| "reward": 0.30674491077661514, |
| "reward_std": 0.026672531850636005, |
| "rewards/clip_reward": 0.30674491077661514, |
| "step": 309 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.009555661729574774, |
| "grad_norm": 0.24148098115752395, |
| "kl": 0.0694580078125, |
| "learning_rate": 1e-05, |
| "loss": 0.0028, |
| "reward": 0.2937857508659363, |
| "reward_std": 0.03315737470984459, |
| "rewards/clip_reward": 0.2937857508659363, |
| "step": 310 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.009586486444831465, |
| "grad_norm": 0.2751403655433819, |
| "kl": 0.0654296875, |
| "learning_rate": 1e-05, |
| "loss": 0.0026, |
| "reward": 0.3029419779777527, |
| "reward_std": 0.037644670344889164, |
| "rewards/clip_reward": 0.3029419779777527, |
| "step": 311 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.009617311160088158, |
| "grad_norm": 0.43265267199518154, |
| "kl": 0.067626953125, |
| "learning_rate": 1e-05, |
| "loss": 0.0027, |
| "reward": 0.3041282668709755, |
| "reward_std": 0.036585441790521145, |
| "rewards/clip_reward": 0.3041282668709755, |
| "step": 312 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.00964813587534485, |
| "grad_norm": 0.23342772151709762, |
| "kl": 0.0709228515625, |
| "learning_rate": 1e-05, |
| "loss": 0.0028, |
| "reward": 0.281108133494854, |
| "reward_std": 0.032666866201907396, |
| "rewards/clip_reward": 0.281108133494854, |
| "step": 313 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.009678960590601544, |
| "grad_norm": 0.2544379988905811, |
| "kl": 0.071044921875, |
| "learning_rate": 1e-05, |
| "loss": 0.0028, |
| "reward": 0.31978850811719894, |
| "reward_std": 0.035083431750535965, |
| "rewards/clip_reward": 0.31978850811719894, |
| "step": 314 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.009709785305858237, |
| "grad_norm": 0.2804560271375766, |
| "kl": 0.068115234375, |
| "learning_rate": 1e-05, |
| "loss": 0.0027, |
| "reward": 0.3023409992456436, |
| "reward_std": 0.039583622477948666, |
| "rewards/clip_reward": 0.3023409992456436, |
| "step": 315 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.00974061002111493, |
| "grad_norm": 0.3308718322184164, |
| "kl": 0.065673828125, |
| "learning_rate": 1e-05, |
| "loss": 0.0026, |
| "reward": 0.3051457703113556, |
| "reward_std": 0.030992726795375347, |
| "rewards/clip_reward": 0.3051457703113556, |
| "step": 316 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.009771434736371622, |
| "grad_norm": 0.28901404215876114, |
| "kl": 0.0743408203125, |
| "learning_rate": 1e-05, |
| "loss": 0.003, |
| "reward": 0.30402371287345886, |
| "reward_std": 0.029232127591967583, |
| "rewards/clip_reward": 0.30402371287345886, |
| "step": 317 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.009802259451628315, |
| "grad_norm": 0.35413328500218205, |
| "kl": 0.072021484375, |
| "learning_rate": 1e-05, |
| "loss": 0.0029, |
| "reward": 0.30789412558078766, |
| "reward_std": 0.03223106591030955, |
| "rewards/clip_reward": 0.30789412558078766, |
| "step": 318 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.009833084166885008, |
| "grad_norm": 0.2340787790478016, |
| "kl": 0.0662841796875, |
| "learning_rate": 1e-05, |
| "loss": 0.0027, |
| "reward": 0.3076978102326393, |
| "reward_std": 0.026181872468441725, |
| "rewards/clip_reward": 0.3076978102326393, |
| "step": 319 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.009863908882141701, |
| "grad_norm": 0.2160311104140781, |
| "kl": 0.06451416015625, |
| "learning_rate": 1e-05, |
| "loss": 0.0026, |
| "reward": 0.3071356937289238, |
| "reward_std": 0.03830281179398298, |
| "rewards/clip_reward": 0.3071356937289238, |
| "step": 320 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.009894733597398394, |
| "grad_norm": 0.2377449652657432, |
| "kl": 0.069580078125, |
| "learning_rate": 1e-05, |
| "loss": 0.0028, |
| "reward": 0.2980581670999527, |
| "reward_std": 0.04127457272261381, |
| "rewards/clip_reward": 0.2980581670999527, |
| "step": 321 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.009925558312655087, |
| "grad_norm": 0.24180517546337557, |
| "kl": 0.06591796875, |
| "learning_rate": 1e-05, |
| "loss": 0.0026, |
| "reward": 0.3178565129637718, |
| "reward_std": 0.03440323146060109, |
| "rewards/clip_reward": 0.3178565129637718, |
| "step": 322 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.00995638302791178, |
| "grad_norm": 0.2316875481005505, |
| "kl": 0.0665283203125, |
| "learning_rate": 1e-05, |
| "loss": 0.0027, |
| "reward": 0.30603795498609543, |
| "reward_std": 0.03252642601728439, |
| "rewards/clip_reward": 0.30603795498609543, |
| "step": 323 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.009987207743168473, |
| "grad_norm": 0.3112158582767738, |
| "kl": 0.0694580078125, |
| "learning_rate": 1e-05, |
| "loss": 0.0028, |
| "reward": 0.2933913469314575, |
| "reward_std": 0.03322177054360509, |
| "rewards/clip_reward": 0.2933913469314575, |
| "step": 324 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.010018032458425166, |
| "grad_norm": 0.2406318070865998, |
| "kl": 0.06787109375, |
| "learning_rate": 1e-05, |
| "loss": 0.0027, |
| "reward": 0.2959599047899246, |
| "reward_std": 0.04275544732809067, |
| "rewards/clip_reward": 0.2959599047899246, |
| "step": 325 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.010048857173681858, |
| "grad_norm": 0.25585002828007836, |
| "kl": 0.06561279296875, |
| "learning_rate": 1e-05, |
| "loss": 0.0026, |
| "reward": 0.306929275393486, |
| "reward_std": 0.03498923219740391, |
| "rewards/clip_reward": 0.306929275393486, |
| "step": 326 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.010079681888938551, |
| "grad_norm": 0.2860165243450835, |
| "kl": 0.06787109375, |
| "learning_rate": 1e-05, |
| "loss": 0.0027, |
| "reward": 0.3039538636803627, |
| "reward_std": 0.0350054414011538, |
| "rewards/clip_reward": 0.3039538636803627, |
| "step": 327 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.010110506604195244, |
| "grad_norm": 0.2915143244249602, |
| "kl": 0.064697265625, |
| "learning_rate": 1e-05, |
| "loss": 0.0026, |
| "reward": 0.30542658269405365, |
| "reward_std": 0.03288308531045914, |
| "rewards/clip_reward": 0.30542658269405365, |
| "step": 328 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.010141331319451937, |
| "grad_norm": 0.2132282152683129, |
| "kl": 0.06341552734375, |
| "learning_rate": 1e-05, |
| "loss": 0.0025, |
| "reward": 0.30350538343191147, |
| "reward_std": 0.03533684695139527, |
| "rewards/clip_reward": 0.30350538343191147, |
| "step": 329 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.01017215603470863, |
| "grad_norm": 0.2229730055471249, |
| "kl": 0.0667724609375, |
| "learning_rate": 1e-05, |
| "loss": 0.0027, |
| "reward": 0.3181814104318619, |
| "reward_std": 0.04055699147284031, |
| "rewards/clip_reward": 0.3181814104318619, |
| "step": 330 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.010202980749965323, |
| "grad_norm": 0.29467120899645105, |
| "kl": 0.078369140625, |
| "learning_rate": 1e-05, |
| "loss": 0.0031, |
| "reward": 0.3067881911993027, |
| "reward_std": 0.03262139391154051, |
| "rewards/clip_reward": 0.3067881911993027, |
| "step": 331 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.010233805465222016, |
| "grad_norm": 0.2626359604798955, |
| "kl": 0.0689697265625, |
| "learning_rate": 1e-05, |
| "loss": 0.0028, |
| "reward": 0.3152911067008972, |
| "reward_std": 0.033262383192777634, |
| "rewards/clip_reward": 0.3152911067008972, |
| "step": 332 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.010264630180478709, |
| "grad_norm": 0.2557414678170153, |
| "kl": 0.065185546875, |
| "learning_rate": 1e-05, |
| "loss": 0.0026, |
| "reward": 0.2922421917319298, |
| "reward_std": 0.03196657868102193, |
| "rewards/clip_reward": 0.2922421917319298, |
| "step": 333 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.0102954548957354, |
| "grad_norm": 0.46611276904322874, |
| "kl": 0.0941162109375, |
| "learning_rate": 1e-05, |
| "loss": 0.0038, |
| "reward": 0.3090960830450058, |
| "reward_std": 0.03677598666399717, |
| "rewards/clip_reward": 0.3090960830450058, |
| "step": 334 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.010326279610992093, |
| "grad_norm": 0.2309027643372804, |
| "kl": 0.0677490234375, |
| "learning_rate": 1e-05, |
| "loss": 0.0027, |
| "reward": 0.3062281683087349, |
| "reward_std": 0.02914317138493061, |
| "rewards/clip_reward": 0.3062281683087349, |
| "step": 335 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.010357104326248786, |
| "grad_norm": 0.24277503406886997, |
| "kl": 0.064208984375, |
| "learning_rate": 1e-05, |
| "loss": 0.0026, |
| "reward": 0.30345892161130905, |
| "reward_std": 0.03988218680024147, |
| "rewards/clip_reward": 0.30345892161130905, |
| "step": 336 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.010387929041505478, |
| "grad_norm": 0.24331709055222814, |
| "kl": 0.06549072265625, |
| "learning_rate": 1e-05, |
| "loss": 0.0026, |
| "reward": 0.28554844856262207, |
| "reward_std": 0.03036739816889167, |
| "rewards/clip_reward": 0.28554844856262207, |
| "step": 337 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.010418753756762171, |
| "grad_norm": 0.3529980048921207, |
| "kl": 0.06640625, |
| "learning_rate": 1e-05, |
| "loss": 0.0027, |
| "reward": 0.30094072222709656, |
| "reward_std": 0.03639454022049904, |
| "rewards/clip_reward": 0.30094072222709656, |
| "step": 338 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.010449578472018864, |
| "grad_norm": 0.4648925688267997, |
| "kl": 0.070068359375, |
| "learning_rate": 1e-05, |
| "loss": 0.0028, |
| "reward": 0.3127727806568146, |
| "reward_std": 0.037409001495689154, |
| "rewards/clip_reward": 0.3127727806568146, |
| "step": 339 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.010480403187275557, |
| "grad_norm": 0.21891573660402955, |
| "kl": 0.0687255859375, |
| "learning_rate": 1e-05, |
| "loss": 0.0027, |
| "reward": 0.2772018723189831, |
| "reward_std": 0.02981334552168846, |
| "rewards/clip_reward": 0.2772018723189831, |
| "step": 340 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.01051122790253225, |
| "grad_norm": 0.22903622425258144, |
| "kl": 0.0693359375, |
| "learning_rate": 1e-05, |
| "loss": 0.0028, |
| "reward": 0.30165959894657135, |
| "reward_std": 0.03327458165585995, |
| "rewards/clip_reward": 0.30165959894657135, |
| "step": 341 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.010542052617788943, |
| "grad_norm": 0.2563729286323683, |
| "kl": 0.0703125, |
| "learning_rate": 1e-05, |
| "loss": 0.0028, |
| "reward": 0.29880737513303757, |
| "reward_std": 0.03046613559126854, |
| "rewards/clip_reward": 0.29880737513303757, |
| "step": 342 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.010572877333045636, |
| "grad_norm": 0.23890758116942548, |
| "kl": 0.07080078125, |
| "learning_rate": 1e-05, |
| "loss": 0.0028, |
| "reward": 0.30336084961891174, |
| "reward_std": 0.03497588029131293, |
| "rewards/clip_reward": 0.30336084961891174, |
| "step": 343 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.010603702048302329, |
| "grad_norm": 0.3415925254166041, |
| "kl": 0.0689697265625, |
| "learning_rate": 1e-05, |
| "loss": 0.0028, |
| "reward": 0.31711456179618835, |
| "reward_std": 0.03901077900081873, |
| "rewards/clip_reward": 0.31711456179618835, |
| "step": 344 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.010634526763559022, |
| "grad_norm": 0.28715059266697696, |
| "kl": 0.0751953125, |
| "learning_rate": 1e-05, |
| "loss": 0.003, |
| "reward": 0.2894616648554802, |
| "reward_std": 0.03519205283373594, |
| "rewards/clip_reward": 0.2894616648554802, |
| "step": 345 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.010665351478815715, |
| "grad_norm": 0.21252915417396734, |
| "kl": 0.072265625, |
| "learning_rate": 1e-05, |
| "loss": 0.0029, |
| "reward": 0.3071034103631973, |
| "reward_std": 0.03865744452923536, |
| "rewards/clip_reward": 0.3071034103631973, |
| "step": 346 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.010696176194072407, |
| "grad_norm": 0.2213810998809183, |
| "kl": 0.0711669921875, |
| "learning_rate": 1e-05, |
| "loss": 0.0028, |
| "reward": 0.30335263907909393, |
| "reward_std": 0.03155648289248347, |
| "rewards/clip_reward": 0.30335263907909393, |
| "step": 347 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.0107270009093291, |
| "grad_norm": 0.22688649328130808, |
| "kl": 0.0701904296875, |
| "learning_rate": 1e-05, |
| "loss": 0.0028, |
| "reward": 0.30418138951063156, |
| "reward_std": 0.03234159108251333, |
| "rewards/clip_reward": 0.30418138951063156, |
| "step": 348 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.010757825624585793, |
| "grad_norm": 0.29663311793497593, |
| "kl": 0.06951904296875, |
| "learning_rate": 1e-05, |
| "loss": 0.0028, |
| "reward": 0.3119578883051872, |
| "reward_std": 0.035748135298490524, |
| "rewards/clip_reward": 0.3119578883051872, |
| "step": 349 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.010788650339842486, |
| "grad_norm": 0.21987020789836037, |
| "kl": 0.0726318359375, |
| "learning_rate": 1e-05, |
| "loss": 0.0029, |
| "reward": 0.2886430397629738, |
| "reward_std": 0.028050173074007034, |
| "rewards/clip_reward": 0.2886430397629738, |
| "step": 350 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.010819475055099179, |
| "grad_norm": 0.25690691289795353, |
| "kl": 0.0743408203125, |
| "learning_rate": 1e-05, |
| "loss": 0.003, |
| "reward": 0.3163754343986511, |
| "reward_std": 0.03696467122063041, |
| "rewards/clip_reward": 0.3163754343986511, |
| "step": 351 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.010850299770355872, |
| "grad_norm": 0.20820549385701986, |
| "kl": 0.068603515625, |
| "learning_rate": 1e-05, |
| "loss": 0.0027, |
| "reward": 0.2914758771657944, |
| "reward_std": 0.036506949458271265, |
| "rewards/clip_reward": 0.2914758771657944, |
| "step": 352 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.010881124485612565, |
| "grad_norm": 0.23881025300895642, |
| "kl": 0.0703125, |
| "learning_rate": 1e-05, |
| "loss": 0.0028, |
| "reward": 0.305471271276474, |
| "reward_std": 0.03957742918282747, |
| "rewards/clip_reward": 0.305471271276474, |
| "step": 353 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.010911949200869258, |
| "grad_norm": 0.23403139952741436, |
| "kl": 0.073974609375, |
| "learning_rate": 1e-05, |
| "loss": 0.003, |
| "reward": 0.29349175840616226, |
| "reward_std": 0.029270636849105358, |
| "rewards/clip_reward": 0.29349175840616226, |
| "step": 354 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.01094277391612595, |
| "grad_norm": 0.29623954478847825, |
| "kl": 0.0679931640625, |
| "learning_rate": 1e-05, |
| "loss": 0.0027, |
| "reward": 0.3115154132246971, |
| "reward_std": 0.03451331192627549, |
| "rewards/clip_reward": 0.3115154132246971, |
| "step": 355 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.010973598631382643, |
| "grad_norm": 0.22254248972166946, |
| "kl": 0.07177734375, |
| "learning_rate": 1e-05, |
| "loss": 0.0029, |
| "reward": 0.2945594787597656, |
| "reward_std": 0.037838808726519346, |
| "rewards/clip_reward": 0.2945594787597656, |
| "step": 356 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.011004423346639335, |
| "grad_norm": 0.31078887072255357, |
| "kl": 0.069580078125, |
| "learning_rate": 1e-05, |
| "loss": 0.0028, |
| "reward": 0.2983166500926018, |
| "reward_std": 0.038911592215299606, |
| "rewards/clip_reward": 0.2983166500926018, |
| "step": 357 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.011035248061896027, |
| "grad_norm": 0.21415415508214233, |
| "kl": 0.0740966796875, |
| "learning_rate": 1e-05, |
| "loss": 0.003, |
| "reward": 0.2967897802591324, |
| "reward_std": 0.034322294406592846, |
| "rewards/clip_reward": 0.2967897802591324, |
| "step": 358 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.01106607277715272, |
| "grad_norm": 0.22868327148604967, |
| "kl": 0.073486328125, |
| "learning_rate": 1e-05, |
| "loss": 0.0029, |
| "reward": 0.30835631489753723, |
| "reward_std": 0.030591858085244894, |
| "rewards/clip_reward": 0.30835631489753723, |
| "step": 359 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.011096897492409413, |
| "grad_norm": 0.2536865522371228, |
| "kl": 0.0755615234375, |
| "learning_rate": 1e-05, |
| "loss": 0.003, |
| "reward": 0.3081829249858856, |
| "reward_std": 0.03304952848702669, |
| "rewards/clip_reward": 0.3081829249858856, |
| "step": 360 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.011127722207666106, |
| "grad_norm": 0.28703592507172904, |
| "kl": 0.0833740234375, |
| "learning_rate": 1e-05, |
| "loss": 0.0033, |
| "reward": 0.31137142330408096, |
| "reward_std": 0.036431248765438795, |
| "rewards/clip_reward": 0.31137142330408096, |
| "step": 361 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.011158546922922799, |
| "grad_norm": 0.25966729830990576, |
| "kl": 0.072265625, |
| "learning_rate": 1e-05, |
| "loss": 0.0029, |
| "reward": 0.32117273658514023, |
| "reward_std": 0.03095503943040967, |
| "rewards/clip_reward": 0.32117273658514023, |
| "step": 362 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.011189371638179492, |
| "grad_norm": 0.25023628654317165, |
| "kl": 0.072509765625, |
| "learning_rate": 1e-05, |
| "loss": 0.0029, |
| "reward": 0.31916752457618713, |
| "reward_std": 0.03393755853176117, |
| "rewards/clip_reward": 0.31916752457618713, |
| "step": 363 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.011220196353436185, |
| "grad_norm": 0.23509232771567978, |
| "kl": 0.0703125, |
| "learning_rate": 1e-05, |
| "loss": 0.0028, |
| "reward": 0.3117026165127754, |
| "reward_std": 0.02881152043119073, |
| "rewards/clip_reward": 0.3117026165127754, |
| "step": 364 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.011251021068692878, |
| "grad_norm": 0.25193424249683816, |
| "kl": 0.0743408203125, |
| "learning_rate": 1e-05, |
| "loss": 0.003, |
| "reward": 0.3166244179010391, |
| "reward_std": 0.03036302560940385, |
| "rewards/clip_reward": 0.3166244179010391, |
| "step": 365 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.01128184578394957, |
| "grad_norm": 0.33920942950993677, |
| "kl": 0.078369140625, |
| "learning_rate": 1e-05, |
| "loss": 0.0031, |
| "reward": 0.30771925300359726, |
| "reward_std": 0.03633692301809788, |
| "rewards/clip_reward": 0.30771925300359726, |
| "step": 366 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.011312670499206263, |
| "grad_norm": 0.21836890161975292, |
| "kl": 0.0755615234375, |
| "learning_rate": 1e-05, |
| "loss": 0.003, |
| "reward": 0.2963472232222557, |
| "reward_std": 0.039488581009209156, |
| "rewards/clip_reward": 0.2963472232222557, |
| "step": 367 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.011343495214462956, |
| "grad_norm": 0.5069167126879947, |
| "kl": 0.0767822265625, |
| "learning_rate": 1e-05, |
| "loss": 0.0031, |
| "reward": 0.2941269427537918, |
| "reward_std": 0.0324277700856328, |
| "rewards/clip_reward": 0.2941269427537918, |
| "step": 368 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.01137431992971965, |
| "grad_norm": 0.23131621578167488, |
| "kl": 0.072509765625, |
| "learning_rate": 1e-05, |
| "loss": 0.0029, |
| "reward": 0.3116966411471367, |
| "reward_std": 0.035368080250918865, |
| "rewards/clip_reward": 0.3116966411471367, |
| "step": 369 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.011405144644976342, |
| "grad_norm": 0.2722113704455896, |
| "kl": 0.071533203125, |
| "learning_rate": 1e-05, |
| "loss": 0.0029, |
| "reward": 0.3093196973204613, |
| "reward_std": 0.0321941040456295, |
| "rewards/clip_reward": 0.3093196973204613, |
| "step": 370 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.011435969360233035, |
| "grad_norm": 0.28730230026079845, |
| "kl": 0.0716552734375, |
| "learning_rate": 1e-05, |
| "loss": 0.0029, |
| "reward": 0.30720267444849014, |
| "reward_std": 0.03148469375446439, |
| "rewards/clip_reward": 0.30720267444849014, |
| "step": 371 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.011466794075489728, |
| "grad_norm": 0.2195587993801826, |
| "kl": 0.0721435546875, |
| "learning_rate": 1e-05, |
| "loss": 0.0029, |
| "reward": 0.31224188208580017, |
| "reward_std": 0.03507778514176607, |
| "rewards/clip_reward": 0.31224188208580017, |
| "step": 372 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.01149761879074642, |
| "grad_norm": 0.22359313280470233, |
| "kl": 0.07666015625, |
| "learning_rate": 1e-05, |
| "loss": 0.0031, |
| "reward": 0.3193345367908478, |
| "reward_std": 0.03153304476290941, |
| "rewards/clip_reward": 0.3193345367908478, |
| "step": 373 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.011528443506003114, |
| "grad_norm": 0.24587352736332965, |
| "kl": 0.07958984375, |
| "learning_rate": 1e-05, |
| "loss": 0.0032, |
| "reward": 0.3059887960553169, |
| "reward_std": 0.03858480043709278, |
| "rewards/clip_reward": 0.3059887960553169, |
| "step": 374 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.011559268221259807, |
| "grad_norm": 0.20861176096611583, |
| "kl": 0.0777587890625, |
| "learning_rate": 1e-05, |
| "loss": 0.0031, |
| "reward": 0.31156501173973083, |
| "reward_std": 0.03071795403957367, |
| "rewards/clip_reward": 0.31156501173973083, |
| "step": 375 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.0115900929365165, |
| "grad_norm": 0.3384083028926716, |
| "kl": 0.0843505859375, |
| "learning_rate": 1e-05, |
| "loss": 0.0034, |
| "reward": 0.2961001992225647, |
| "reward_std": 0.03549221996217966, |
| "rewards/clip_reward": 0.2961001992225647, |
| "step": 376 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.011620917651773192, |
| "grad_norm": 0.25505451962723275, |
| "kl": 0.0787353515625, |
| "learning_rate": 1e-05, |
| "loss": 0.0032, |
| "reward": 0.3040069565176964, |
| "reward_std": 0.030104033648967743, |
| "rewards/clip_reward": 0.3040069565176964, |
| "step": 377 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.011651742367029885, |
| "grad_norm": 0.28066405899218805, |
| "kl": 0.0821533203125, |
| "learning_rate": 1e-05, |
| "loss": 0.0033, |
| "reward": 0.3125178590416908, |
| "reward_std": 0.03651127126067877, |
| "rewards/clip_reward": 0.3125178590416908, |
| "step": 378 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.011682567082286578, |
| "grad_norm": 0.27665197939767366, |
| "kl": 0.079345703125, |
| "learning_rate": 1e-05, |
| "loss": 0.0032, |
| "reward": 0.3094138279557228, |
| "reward_std": 0.038227169774472713, |
| "rewards/clip_reward": 0.3094138279557228, |
| "step": 379 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.01171339179754327, |
| "grad_norm": 0.2597327901842019, |
| "kl": 0.080810546875, |
| "learning_rate": 1e-05, |
| "loss": 0.0032, |
| "reward": 0.3057212755084038, |
| "reward_std": 0.03503256104886532, |
| "rewards/clip_reward": 0.3057212755084038, |
| "step": 380 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.011744216512799962, |
| "grad_norm": 0.32226220932101085, |
| "kl": 0.0845947265625, |
| "learning_rate": 1e-05, |
| "loss": 0.0034, |
| "reward": 0.3279699385166168, |
| "reward_std": 0.0381966782733798, |
| "rewards/clip_reward": 0.3279699385166168, |
| "step": 381 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.011775041228056655, |
| "grad_norm": 0.2956679464457214, |
| "kl": 0.081787109375, |
| "learning_rate": 1e-05, |
| "loss": 0.0033, |
| "reward": 0.30723875761032104, |
| "reward_std": 0.03168717911466956, |
| "rewards/clip_reward": 0.30723875761032104, |
| "step": 382 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.011805865943313348, |
| "grad_norm": 0.5191055044770609, |
| "kl": 0.091552734375, |
| "learning_rate": 1e-05, |
| "loss": 0.0037, |
| "reward": 0.31639552116394043, |
| "reward_std": 0.03826092462986708, |
| "rewards/clip_reward": 0.31639552116394043, |
| "step": 383 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.011836690658570041, |
| "grad_norm": 0.3776553446735074, |
| "kl": 0.091552734375, |
| "learning_rate": 1e-05, |
| "loss": 0.0037, |
| "reward": 0.3041198328137398, |
| "reward_std": 0.026013361755758524, |
| "rewards/clip_reward": 0.3041198328137398, |
| "step": 384 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.011867515373826734, |
| "grad_norm": 0.2572694780557513, |
| "kl": 0.08837890625, |
| "learning_rate": 1e-05, |
| "loss": 0.0035, |
| "reward": 0.2923019975423813, |
| "reward_std": 0.03231561556458473, |
| "rewards/clip_reward": 0.2923019975423813, |
| "step": 385 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.011898340089083427, |
| "grad_norm": 15.002263434692885, |
| "kl": 0.1844482421875, |
| "learning_rate": 1e-05, |
| "loss": 0.0074, |
| "reward": 0.31261809915304184, |
| "reward_std": 0.033471152652055025, |
| "rewards/clip_reward": 0.31261809915304184, |
| "step": 386 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.01192916480434012, |
| "grad_norm": 34.74334790635388, |
| "kl": 0.585205078125, |
| "learning_rate": 1e-05, |
| "loss": 0.0234, |
| "reward": 0.31164050102233887, |
| "reward_std": 0.03602212620899081, |
| "rewards/clip_reward": 0.31164050102233887, |
| "step": 387 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.011959989519596812, |
| "grad_norm": 0.2788251418304219, |
| "kl": 0.082275390625, |
| "learning_rate": 1e-05, |
| "loss": 0.0033, |
| "reward": 0.2998761534690857, |
| "reward_std": 0.03641665726900101, |
| "rewards/clip_reward": 0.2998761534690857, |
| "step": 388 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.011990814234853505, |
| "grad_norm": 0.23027728154942576, |
| "kl": 0.086669921875, |
| "learning_rate": 1e-05, |
| "loss": 0.0035, |
| "reward": 0.30939532816410065, |
| "reward_std": 0.033424354158341885, |
| "rewards/clip_reward": 0.30939532816410065, |
| "step": 389 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.012021638950110198, |
| "grad_norm": 0.25021193487381366, |
| "kl": 0.091552734375, |
| "learning_rate": 1e-05, |
| "loss": 0.0037, |
| "reward": 0.3133769854903221, |
| "reward_std": 0.034206162206828594, |
| "rewards/clip_reward": 0.3133769854903221, |
| "step": 390 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.012052463665366891, |
| "grad_norm": 0.23903302345332506, |
| "kl": 0.0802001953125, |
| "learning_rate": 1e-05, |
| "loss": 0.0032, |
| "reward": 0.3138531744480133, |
| "reward_std": 0.03734522592276335, |
| "rewards/clip_reward": 0.3138531744480133, |
| "step": 391 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.012083288380623584, |
| "grad_norm": 0.2598108600282988, |
| "kl": 0.083251953125, |
| "learning_rate": 1e-05, |
| "loss": 0.0033, |
| "reward": 0.32199686020612717, |
| "reward_std": 0.03402840159833431, |
| "rewards/clip_reward": 0.32199686020612717, |
| "step": 392 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.012114113095880277, |
| "grad_norm": 0.23642883730734804, |
| "kl": 0.08056640625, |
| "learning_rate": 1e-05, |
| "loss": 0.0032, |
| "reward": 0.3051422908902168, |
| "reward_std": 0.03256607661023736, |
| "rewards/clip_reward": 0.3051422908902168, |
| "step": 393 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.01214493781113697, |
| "grad_norm": 0.22592478949720543, |
| "kl": 0.0802001953125, |
| "learning_rate": 1e-05, |
| "loss": 0.0032, |
| "reward": 0.3141862004995346, |
| "reward_std": 0.03128534881398082, |
| "rewards/clip_reward": 0.3141862004995346, |
| "step": 394 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.012175762526393663, |
| "grad_norm": 0.27814956809020164, |
| "kl": 0.080810546875, |
| "learning_rate": 1e-05, |
| "loss": 0.0032, |
| "reward": 0.30130794644355774, |
| "reward_std": 0.028409887570887804, |
| "rewards/clip_reward": 0.30130794644355774, |
| "step": 395 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.012206587241650356, |
| "grad_norm": 0.21400571540908192, |
| "kl": 0.086669921875, |
| "learning_rate": 1e-05, |
| "loss": 0.0035, |
| "reward": 0.30789605528116226, |
| "reward_std": 0.034056794829666615, |
| "rewards/clip_reward": 0.30789605528116226, |
| "step": 396 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.012237411956907049, |
| "grad_norm": 0.21349466740102746, |
| "kl": 0.080810546875, |
| "learning_rate": 1e-05, |
| "loss": 0.0032, |
| "reward": 0.3021450638771057, |
| "reward_std": 0.035551701206713915, |
| "rewards/clip_reward": 0.3021450638771057, |
| "step": 397 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.012268236672163741, |
| "grad_norm": 0.21264064002989824, |
| "kl": 0.080322265625, |
| "learning_rate": 1e-05, |
| "loss": 0.0032, |
| "reward": 0.3078059256076813, |
| "reward_std": 0.03562043095007539, |
| "rewards/clip_reward": 0.3078059256076813, |
| "step": 398 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.012299061387420434, |
| "grad_norm": 0.2898591178898953, |
| "kl": 0.08642578125, |
| "learning_rate": 1e-05, |
| "loss": 0.0035, |
| "reward": 0.3178284019231796, |
| "reward_std": 0.03499773098155856, |
| "rewards/clip_reward": 0.3178284019231796, |
| "step": 399 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 4096.0, |
| "epoch": 0.012329886102677127, |
| "grad_norm": 0.5180005174036416, |
| "kl": 0.08935546875, |
| "learning_rate": 1e-05, |
| "loss": 0.0036, |
| "reward": 0.32223907113075256, |
| "reward_std": 0.0368791688233614, |
| "rewards/clip_reward": 0.32223907113075256, |
| "step": 400 |
| } |
| ], |
| "logging_steps": 1, |
| "max_steps": 32441, |
| "num_input_tokens_seen": 0, |
| "num_train_epochs": 1, |
| "save_steps": 10, |
| "stateful_callbacks": { |
| "TrainerControl": { |
| "args": { |
| "should_epoch_stop": false, |
| "should_evaluate": false, |
| "should_log": false, |
| "should_save": true, |
| "should_training_stop": false |
| }, |
| "attributes": {} |
| } |
| }, |
| "total_flos": 0.0, |
| "train_batch_size": 6, |
| "trial_name": null, |
| "trial_params": null |
| } |
|
|
