| { |
| "best_metric": null, |
| "best_model_checkpoint": null, |
| "epoch": 1.0, |
| "eval_steps": 500, |
| "global_step": 335, |
| "is_hyper_param_search": false, |
| "is_local_process_zero": true, |
| "is_world_process_zero": true, |
| "log_history": [ |
| { |
| "completion_length": 285.25, |
| "epoch": 0.0029850746268656717, |
| "grad_norm": 0.0, |
| "kl": 0.0, |
| "learning_rate": 1.4705882352941178e-07, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 1 |
| }, |
| { |
| "completion_length": 427.875, |
| "epoch": 0.005970149253731343, |
| "grad_norm": 0.22855468094348907, |
| "kl": 0.0, |
| "learning_rate": 2.9411764705882356e-07, |
| "loss": -0.0, |
| "reward": -0.24074998497962952, |
| "reward_std": 0.6006079912185669, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": -0.2407499998807907, |
| "step": 2 |
| }, |
| { |
| "completion_length": 296.5, |
| "epoch": 0.008955223880597015, |
| "grad_norm": 8.133344090310857e-05, |
| "kl": 4.377714503789321e-06, |
| "learning_rate": 4.4117647058823536e-07, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 3 |
| }, |
| { |
| "completion_length": 238.375, |
| "epoch": 0.011940298507462687, |
| "grad_norm": 0.0003321934782434255, |
| "kl": 1.140108270192286e-05, |
| "learning_rate": 5.882352941176471e-07, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 4 |
| }, |
| { |
| "completion_length": 182.5, |
| "epoch": 0.014925373134328358, |
| "grad_norm": 0.3859254717826843, |
| "kl": 1.0751177796919364e-05, |
| "learning_rate": 7.352941176470589e-07, |
| "loss": 0.0, |
| "reward": -0.07850000262260437, |
| "reward_std": 0.22203154861927032, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": -0.07850000262260437, |
| "step": 5 |
| }, |
| { |
| "completion_length": 311.75, |
| "epoch": 0.01791044776119403, |
| "grad_norm": 0.34084582328796387, |
| "kl": 8.012263606360648e-06, |
| "learning_rate": 8.823529411764707e-07, |
| "loss": 0.0, |
| "reward": -0.015250000171363354, |
| "reward_std": 0.04313351586461067, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": -0.015250000171363354, |
| "step": 6 |
| }, |
| { |
| "completion_length": 190.75, |
| "epoch": 0.020895522388059702, |
| "grad_norm": 0.5919244885444641, |
| "kl": 9.190077435050625e-06, |
| "learning_rate": 1.0294117647058825e-06, |
| "loss": 0.0, |
| "reward": -0.013125000521540642, |
| "reward_std": 0.04896482080221176, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": -0.013125000521540642, |
| "step": 7 |
| }, |
| { |
| "completion_length": 226.75, |
| "epoch": 0.023880597014925373, |
| "grad_norm": 0.31808775663375854, |
| "kl": 4.68343068860122e-06, |
| "learning_rate": 1.1764705882352942e-06, |
| "loss": 0.0, |
| "reward": -0.03175000101327896, |
| "reward_std": 0.0898025631904602, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": -0.03175000101327896, |
| "step": 8 |
| }, |
| { |
| "completion_length": 418.75, |
| "epoch": 0.026865671641791045, |
| "grad_norm": 0.0001131733224610798, |
| "kl": 8.002484719327185e-06, |
| "learning_rate": 1.323529411764706e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 9 |
| }, |
| { |
| "completion_length": 579.625, |
| "epoch": 0.029850746268656716, |
| "grad_norm": 0.16198158264160156, |
| "kl": 6.150351055111969e-06, |
| "learning_rate": 1.4705882352941177e-06, |
| "loss": 0.0, |
| "reward": 0.015625, |
| "reward_std": 0.04419417306780815, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.015625, |
| "step": 10 |
| }, |
| { |
| "completion_length": 514.0, |
| "epoch": 0.03283582089552239, |
| "grad_norm": 0.1703915297985077, |
| "kl": 6.072848009353038e-06, |
| "learning_rate": 1.6176470588235297e-06, |
| "loss": 0.0, |
| "reward": -0.21412500739097595, |
| "reward_std": 0.6056370139122009, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": -0.21412500739097595, |
| "step": 11 |
| }, |
| { |
| "completion_length": 459.875, |
| "epoch": 0.03582089552238806, |
| "grad_norm": 6.846042379038408e-05, |
| "kl": 4.143657861277461e-06, |
| "learning_rate": 1.7647058823529414e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 12 |
| }, |
| { |
| "completion_length": 530.375, |
| "epoch": 0.03880597014925373, |
| "grad_norm": 7.420629845000803e-05, |
| "kl": 5.574378064920893e-06, |
| "learning_rate": 1.9117647058823528e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 13 |
| }, |
| { |
| "completion_length": 493.875, |
| "epoch": 0.041791044776119404, |
| "grad_norm": 0.0001040869319695048, |
| "kl": 8.489936590194702e-06, |
| "learning_rate": 2.058823529411765e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 14 |
| }, |
| { |
| "completion_length": 614.125, |
| "epoch": 0.04477611940298507, |
| "grad_norm": 0.20984017848968506, |
| "kl": 1.4391247532330453e-05, |
| "learning_rate": 2.2058823529411767e-06, |
| "loss": 0.0, |
| "reward": -0.26112499833106995, |
| "reward_std": 0.5091128349304199, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": -0.26112499833106995, |
| "step": 15 |
| }, |
| { |
| "completion_length": 512.375, |
| "epoch": 0.04776119402985075, |
| "grad_norm": 0.19785551726818085, |
| "kl": 8.07260767032858e-06, |
| "learning_rate": 2.3529411764705885e-06, |
| "loss": 0.0, |
| "reward": -0.2941250205039978, |
| "reward_std": 0.5475807785987854, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": -0.2941250205039978, |
| "step": 16 |
| }, |
| { |
| "completion_length": 308.375, |
| "epoch": 0.050746268656716415, |
| "grad_norm": 0.33640098571777344, |
| "kl": 2.1692663722205907e-05, |
| "learning_rate": 2.5e-06, |
| "loss": 0.0, |
| "reward": 0.18700000643730164, |
| "reward_std": 0.7908055782318115, |
| "rewards/correctness_reward_func": 0.25, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": -0.06300000101327896, |
| "step": 17 |
| }, |
| { |
| "completion_length": 663.125, |
| "epoch": 0.05373134328358209, |
| "grad_norm": 0.17784741520881653, |
| "kl": 1.2350877113931347e-05, |
| "learning_rate": 2.647058823529412e-06, |
| "loss": 0.0, |
| "reward": -0.1081250011920929, |
| "reward_std": 0.35900595784187317, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": -0.1081250011920929, |
| "step": 18 |
| }, |
| { |
| "completion_length": 245.5, |
| "epoch": 0.056716417910447764, |
| "grad_norm": 0.0005393940955400467, |
| "kl": 1.7745833247317933e-05, |
| "learning_rate": 2.7941176470588237e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 19 |
| }, |
| { |
| "completion_length": 203.25, |
| "epoch": 0.05970149253731343, |
| "grad_norm": 0.35458770394325256, |
| "kl": 8.00752459326759e-05, |
| "learning_rate": 2.9411764705882355e-06, |
| "loss": 0.0, |
| "reward": -0.12062501162290573, |
| "reward_std": 0.5963314175605774, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": -0.12062500417232513, |
| "step": 20 |
| }, |
| { |
| "completion_length": 381.25, |
| "epoch": 0.0626865671641791, |
| "grad_norm": 0.22898265719413757, |
| "kl": 3.1856085115578026e-05, |
| "learning_rate": 3.0882352941176476e-06, |
| "loss": 0.0, |
| "reward": -0.0364999994635582, |
| "reward_std": 0.1032375916838646, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": -0.0364999994635582, |
| "step": 21 |
| }, |
| { |
| "completion_length": 369.5, |
| "epoch": 0.06567164179104477, |
| "grad_norm": 0.00023389812849927694, |
| "kl": 1.4598858797398861e-05, |
| "learning_rate": 3.2352941176470594e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 22 |
| }, |
| { |
| "completion_length": 350.625, |
| "epoch": 0.06865671641791045, |
| "grad_norm": 0.21955104172229767, |
| "kl": 7.147570431698114e-05, |
| "learning_rate": 3.382352941176471e-06, |
| "loss": 0.0, |
| "reward": -0.2084999978542328, |
| "reward_std": 0.5897271037101746, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": -0.2084999978542328, |
| "step": 23 |
| }, |
| { |
| "completion_length": 418.25, |
| "epoch": 0.07164179104477612, |
| "grad_norm": 0.27754807472229004, |
| "kl": 6.455584662035108e-05, |
| "learning_rate": 3.529411764705883e-06, |
| "loss": 0.0, |
| "reward": -0.046875, |
| "reward_std": 0.48620080947875977, |
| "rewards/correctness_reward_func": 0.25, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": -0.296875, |
| "step": 24 |
| }, |
| { |
| "completion_length": 342.625, |
| "epoch": 0.07462686567164178, |
| "grad_norm": 0.33675655722618103, |
| "kl": 0.000209408113732934, |
| "learning_rate": 3.6764705882352946e-06, |
| "loss": 0.0, |
| "reward": -0.242374986410141, |
| "reward_std": 0.45189568400382996, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": -0.24237500131130219, |
| "step": 25 |
| }, |
| { |
| "completion_length": 249.25, |
| "epoch": 0.07761194029850746, |
| "grad_norm": 0.000760515860747546, |
| "kl": 6.706830026814714e-05, |
| "learning_rate": 3.8235294117647055e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 26 |
| }, |
| { |
| "completion_length": 261.125, |
| "epoch": 0.08059701492537313, |
| "grad_norm": 0.5024670362472534, |
| "kl": 0.00022520618222188205, |
| "learning_rate": 3.970588235294118e-06, |
| "loss": 0.0, |
| "reward": -0.016625000163912773, |
| "reward_std": 0.04702260345220566, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": -0.016625000163912773, |
| "step": 27 |
| }, |
| { |
| "completion_length": 551.375, |
| "epoch": 0.08358208955223881, |
| "grad_norm": 0.20812274515628815, |
| "kl": 0.00023469090228900313, |
| "learning_rate": 4.11764705882353e-06, |
| "loss": 0.0, |
| "reward": -0.296750009059906, |
| "reward_std": 0.557201087474823, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": -0.296750009059906, |
| "step": 28 |
| }, |
| { |
| "completion_length": 286.625, |
| "epoch": 0.08656716417910448, |
| "grad_norm": 0.48242834210395813, |
| "kl": 0.00043666691635735333, |
| "learning_rate": 4.264705882352942e-06, |
| "loss": 0.0, |
| "reward": 0.015625, |
| "reward_std": 0.04419417306780815, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.015625, |
| "step": 29 |
| }, |
| { |
| "completion_length": 320.25, |
| "epoch": 0.08955223880597014, |
| "grad_norm": 0.2443850338459015, |
| "kl": 0.001531625515781343, |
| "learning_rate": 4.411764705882353e-06, |
| "loss": 0.0001, |
| "reward": -0.11225000023841858, |
| "reward_std": 0.31749093532562256, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": -0.11225000023841858, |
| "step": 30 |
| }, |
| { |
| "completion_length": 370.125, |
| "epoch": 0.09253731343283582, |
| "grad_norm": 0.28154098987579346, |
| "kl": 0.0002926041779574007, |
| "learning_rate": 4.558823529411765e-06, |
| "loss": 0.0, |
| "reward": 0.03137499839067459, |
| "reward_std": 0.05809582397341728, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.03137499839067459, |
| "step": 31 |
| }, |
| { |
| "completion_length": 249.625, |
| "epoch": 0.0955223880597015, |
| "grad_norm": 0.0027966005727648735, |
| "kl": 0.0003874501271639019, |
| "learning_rate": 4.705882352941177e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 32 |
| }, |
| { |
| "completion_length": 282.625, |
| "epoch": 0.09850746268656717, |
| "grad_norm": 0.0010501514188945293, |
| "kl": 0.0001470946444896981, |
| "learning_rate": 4.852941176470589e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 33 |
| }, |
| { |
| "completion_length": 439.625, |
| "epoch": 0.10149253731343283, |
| "grad_norm": 0.0016689057229086757, |
| "kl": 0.00023773338762111962, |
| "learning_rate": 5e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 34 |
| }, |
| { |
| "completion_length": 677.375, |
| "epoch": 0.1044776119402985, |
| "grad_norm": 0.0020174484234303236, |
| "kl": 0.00034356946707703173, |
| "learning_rate": 4.999863832700438e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 35 |
| }, |
| { |
| "completion_length": 524.25, |
| "epoch": 0.10746268656716418, |
| "grad_norm": 0.24175743758678436, |
| "kl": 0.0005913617205806077, |
| "learning_rate": 4.9994553456349785e-06, |
| "loss": 0.0, |
| "reward": -0.1913750022649765, |
| "reward_std": 0.541290283203125, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": -0.1913750022649765, |
| "step": 36 |
| }, |
| { |
| "completion_length": 498.5, |
| "epoch": 0.11044776119402985, |
| "grad_norm": 0.007789173629134893, |
| "kl": 0.0014796899631619453, |
| "learning_rate": 4.998774583301685e-06, |
| "loss": 0.0001, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 37 |
| }, |
| { |
| "completion_length": 402.5, |
| "epoch": 0.11343283582089553, |
| "grad_norm": 0.2350546419620514, |
| "kl": 0.001118505373597145, |
| "learning_rate": 4.997821619858614e-06, |
| "loss": 0.0, |
| "reward": 0.04012499749660492, |
| "reward_std": 0.11349063366651535, |
| "rewards/correctness_reward_func": 0.25, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": -0.20987500250339508, |
| "step": 38 |
| }, |
| { |
| "completion_length": 212.625, |
| "epoch": 0.11641791044776119, |
| "grad_norm": 0.594977855682373, |
| "kl": 0.0025897223968058825, |
| "learning_rate": 4.9965965591157314e-06, |
| "loss": 0.0001, |
| "reward": 0.21799999475479126, |
| "reward_std": 0.7819932699203491, |
| "rewards/correctness_reward_func": 0.25, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": -0.03200000151991844, |
| "step": 39 |
| }, |
| { |
| "completion_length": 171.125, |
| "epoch": 0.11940298507462686, |
| "grad_norm": 0.3670198321342468, |
| "kl": 0.002329748822376132, |
| "learning_rate": 4.995099534523608e-06, |
| "loss": 0.0001, |
| "reward": -0.00037499889731407166, |
| "reward_std": 0.08900070190429688, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": -0.00037499889731407166, |
| "step": 40 |
| }, |
| { |
| "completion_length": 363.75, |
| "epoch": 0.12238805970149254, |
| "grad_norm": 0.162104994058609, |
| "kl": 0.0005531070055440068, |
| "learning_rate": 4.993330709158879e-06, |
| "loss": 0.0, |
| "reward": 0.03125, |
| "reward_std": 0.0883883461356163, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.03125, |
| "step": 41 |
| }, |
| { |
| "completion_length": 370.75, |
| "epoch": 0.1253731343283582, |
| "grad_norm": 0.22303998470306396, |
| "kl": 0.0011743077775463462, |
| "learning_rate": 4.991290275706486e-06, |
| "loss": 0.0, |
| "reward": 0.17225000262260437, |
| "reward_std": 0.8312016725540161, |
| "rewards/correctness_reward_func": 0.25, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": -0.07774999737739563, |
| "step": 42 |
| }, |
| { |
| "completion_length": 585.75, |
| "epoch": 0.12835820895522387, |
| "grad_norm": 0.0013642405392602086, |
| "kl": 0.00035193481016904116, |
| "learning_rate": 4.988978456438678e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 43 |
| }, |
| { |
| "completion_length": 299.75, |
| "epoch": 0.13134328358208955, |
| "grad_norm": 0.0038114753551781178, |
| "kl": 0.0007628994644619524, |
| "learning_rate": 4.986395503190805e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 44 |
| }, |
| { |
| "completion_length": 483.875, |
| "epoch": 0.13432835820895522, |
| "grad_norm": 0.0009761088294908404, |
| "kl": 0.000256923318374902, |
| "learning_rate": 4.9835416973338815e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 45 |
| }, |
| { |
| "completion_length": 625.75, |
| "epoch": 0.1373134328358209, |
| "grad_norm": 0.0015355540672317147, |
| "kl": 0.0003070808306802064, |
| "learning_rate": 4.980417349743936e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 46 |
| }, |
| { |
| "completion_length": 371.0, |
| "epoch": 0.14029850746268657, |
| "grad_norm": 0.29057756066322327, |
| "kl": 0.0009468809003010392, |
| "learning_rate": 4.97702280076815e-06, |
| "loss": 0.0, |
| "reward": -0.05824999883770943, |
| "reward_std": 0.16475588083267212, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": -0.05824999883770943, |
| "step": 47 |
| }, |
| { |
| "completion_length": 352.375, |
| "epoch": 0.14328358208955225, |
| "grad_norm": 0.2544783651828766, |
| "kl": 0.0009582972852513194, |
| "learning_rate": 4.973358420187776e-06, |
| "loss": 0.0, |
| "reward": 0.015625, |
| "reward_std": 0.04419417306780815, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.015625, |
| "step": 48 |
| }, |
| { |
| "completion_length": 366.375, |
| "epoch": 0.14626865671641792, |
| "grad_norm": 0.38926711678504944, |
| "kl": 0.001495029078796506, |
| "learning_rate": 4.969424607177861e-06, |
| "loss": 0.0001, |
| "reward": 0.03125, |
| "reward_std": 0.0883883461356163, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.03125, |
| "step": 49 |
| }, |
| { |
| "completion_length": 339.0, |
| "epoch": 0.14925373134328357, |
| "grad_norm": 0.0019049796974286437, |
| "kl": 0.00044422244536690414, |
| "learning_rate": 4.96522179026376e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 50 |
| }, |
| { |
| "completion_length": 277.875, |
| "epoch": 0.15223880597014924, |
| "grad_norm": 0.28892993927001953, |
| "kl": 0.0017293869750574231, |
| "learning_rate": 4.960750427274458e-06, |
| "loss": 0.0001, |
| "reward": -0.054374996572732925, |
| "reward_std": 0.3191067576408386, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": -0.054374996572732925, |
| "step": 51 |
| }, |
| { |
| "completion_length": 292.75, |
| "epoch": 0.15522388059701492, |
| "grad_norm": 0.0033381686080247164, |
| "kl": 0.0008934027864597738, |
| "learning_rate": 4.956011005292693e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 52 |
| }, |
| { |
| "completion_length": 356.875, |
| "epoch": 0.1582089552238806, |
| "grad_norm": 0.0015895323595032096, |
| "kl": 0.000459035363746807, |
| "learning_rate": 4.951004040601898e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 53 |
| }, |
| { |
| "completion_length": 355.0, |
| "epoch": 0.16119402985074627, |
| "grad_norm": 0.24429979920387268, |
| "kl": 0.0013310480862855911, |
| "learning_rate": 4.945730078629965e-06, |
| "loss": 0.0001, |
| "reward": -0.31862500309944153, |
| "reward_std": 0.4402158558368683, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": -0.31862500309944153, |
| "step": 54 |
| }, |
| { |
| "completion_length": 471.125, |
| "epoch": 0.16417910447761194, |
| "grad_norm": 0.17072314023971558, |
| "kl": 0.0008681550971232355, |
| "learning_rate": 4.940189693889819e-06, |
| "loss": 0.0, |
| "reward": 0.01549999974668026, |
| "reward_std": 0.10450837016105652, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.01549999974668026, |
| "step": 55 |
| }, |
| { |
| "completion_length": 393.75, |
| "epoch": 0.16716417910447762, |
| "grad_norm": 0.0035993217024952173, |
| "kl": 0.00099479453638196, |
| "learning_rate": 4.934383489916843e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 56 |
| }, |
| { |
| "completion_length": 335.375, |
| "epoch": 0.1701492537313433, |
| "grad_norm": 0.004349116701632738, |
| "kl": 0.00097746797837317, |
| "learning_rate": 4.928312099203131e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 57 |
| }, |
| { |
| "completion_length": 476.0, |
| "epoch": 0.17313432835820897, |
| "grad_norm": 0.002000964479520917, |
| "kl": 0.0002213095867773518, |
| "learning_rate": 4.921976183128585e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 58 |
| }, |
| { |
| "completion_length": 160.75, |
| "epoch": 0.1761194029850746, |
| "grad_norm": 0.00427111005410552, |
| "kl": 0.001069595105946064, |
| "learning_rate": 4.915376431888871e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 59 |
| }, |
| { |
| "completion_length": 300.125, |
| "epoch": 0.1791044776119403, |
| "grad_norm": 0.003434703918173909, |
| "kl": 0.000937220233026892, |
| "learning_rate": 4.908513564420231e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 60 |
| }, |
| { |
| "completion_length": 369.875, |
| "epoch": 0.18208955223880596, |
| "grad_norm": 0.27929064631462097, |
| "kl": 0.0012021824950352311, |
| "learning_rate": 4.9013883283211705e-06, |
| "loss": 0.0, |
| "reward": -0.019999999552965164, |
| "reward_std": 0.11566577106714249, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": -0.019999999552965164, |
| "step": 61 |
| }, |
| { |
| "completion_length": 362.875, |
| "epoch": 0.18507462686567164, |
| "grad_norm": 0.00122266192920506, |
| "kl": 0.0004295996914152056, |
| "learning_rate": 4.894001499771015e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 62 |
| }, |
| { |
| "completion_length": 325.125, |
| "epoch": 0.1880597014925373, |
| "grad_norm": 0.0036570935044437647, |
| "kl": 0.0008783735684119165, |
| "learning_rate": 4.886353883445363e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 63 |
| }, |
| { |
| "completion_length": 423.625, |
| "epoch": 0.191044776119403, |
| "grad_norm": 0.0054810927249491215, |
| "kl": 0.0009283218532800674, |
| "learning_rate": 4.878446312428424e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 64 |
| }, |
| { |
| "completion_length": 342.625, |
| "epoch": 0.19402985074626866, |
| "grad_norm": 0.19578829407691956, |
| "kl": 0.0010514282621443272, |
| "learning_rate": 4.870279648122271e-06, |
| "loss": 0.0, |
| "reward": -0.1782499998807907, |
| "reward_std": 0.5041671395301819, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": -0.1782499998807907, |
| "step": 65 |
| }, |
| { |
| "completion_length": 334.875, |
| "epoch": 0.19701492537313434, |
| "grad_norm": 0.00260003749281168, |
| "kl": 0.0005953626241534948, |
| "learning_rate": 4.8618547801530045e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 66 |
| }, |
| { |
| "completion_length": 365.5, |
| "epoch": 0.2, |
| "grad_norm": 0.3674154579639435, |
| "kl": 0.0011260712053626776, |
| "learning_rate": 4.853172626273841e-06, |
| "loss": 0.0, |
| "reward": -0.21437500417232513, |
| "reward_std": 0.4378893971443176, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": -0.21437500417232513, |
| "step": 67 |
| }, |
| { |
| "completion_length": 432.875, |
| "epoch": 0.20298507462686566, |
| "grad_norm": 0.001397120882757008, |
| "kl": 0.00025881017791107297, |
| "learning_rate": 4.844234132265139e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 68 |
| }, |
| { |
| "completion_length": 364.875, |
| "epoch": 0.20597014925373133, |
| "grad_norm": 0.0018441714346408844, |
| "kl": 0.00031896165455691516, |
| "learning_rate": 4.835040271831371e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 69 |
| }, |
| { |
| "completion_length": 584.875, |
| "epoch": 0.208955223880597, |
| "grad_norm": 0.001404198701493442, |
| "kl": 0.0003437511913944036, |
| "learning_rate": 4.8255920464950545e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 70 |
| }, |
| { |
| "completion_length": 287.875, |
| "epoch": 0.21194029850746268, |
| "grad_norm": 0.23131783306598663, |
| "kl": 0.00048740627244114876, |
| "learning_rate": 4.8158904854876555e-06, |
| "loss": 0.0, |
| "reward": 0.015625, |
| "reward_std": 0.04419417306780815, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.015625, |
| "step": 71 |
| }, |
| { |
| "completion_length": 481.875, |
| "epoch": 0.21492537313432836, |
| "grad_norm": 0.27868691086769104, |
| "kl": 0.0005523943109437823, |
| "learning_rate": 4.805936645637463e-06, |
| "loss": 0.0, |
| "reward": 0.015625, |
| "reward_std": 0.04419417306780815, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.015625, |
| "step": 72 |
| }, |
| { |
| "completion_length": 498.375, |
| "epoch": 0.21791044776119403, |
| "grad_norm": 0.0016176890349015594, |
| "kl": 0.0005396915948949754, |
| "learning_rate": 4.795731611254473e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 73 |
| }, |
| { |
| "completion_length": 300.875, |
| "epoch": 0.2208955223880597, |
| "grad_norm": 0.24887824058532715, |
| "kl": 0.0005686509539373219, |
| "learning_rate": 4.7852764940122636e-06, |
| "loss": 0.0, |
| "reward": 0.22587499022483826, |
| "reward_std": 0.5445812344551086, |
| "rewards/correctness_reward_func": 0.25, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": -0.024125002324581146, |
| "step": 74 |
| }, |
| { |
| "completion_length": 177.75, |
| "epoch": 0.22388059701492538, |
| "grad_norm": 0.0012561667244881392, |
| "kl": 0.00028040556935593486, |
| "learning_rate": 4.7745724328269e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 75 |
| }, |
| { |
| "completion_length": 410.625, |
| "epoch": 0.22686567164179106, |
| "grad_norm": 0.2470964640378952, |
| "kl": 0.000733887602109462, |
| "learning_rate": 4.763620593732867e-06, |
| "loss": 0.0, |
| "reward": 0.015625, |
| "reward_std": 0.04419417306780815, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.015625, |
| "step": 76 |
| }, |
| { |
| "completion_length": 278.125, |
| "epoch": 0.2298507462686567, |
| "grad_norm": 0.2767794728279114, |
| "kl": 0.0013002330670133233, |
| "learning_rate": 4.752422169756048e-06, |
| "loss": 0.0001, |
| "reward": -0.025499999523162842, |
| "reward_std": 0.0721248909831047, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": -0.025499999523162842, |
| "step": 77 |
| }, |
| { |
| "completion_length": 296.625, |
| "epoch": 0.23283582089552238, |
| "grad_norm": 0.0011332810390740633, |
| "kl": 0.0002715845184866339, |
| "learning_rate": 4.7409783807837654e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 78 |
| }, |
| { |
| "completion_length": 248.625, |
| "epoch": 0.23582089552238805, |
| "grad_norm": 0.001428711460903287, |
| "kl": 0.0004016221791971475, |
| "learning_rate": 4.729290473431892e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 79 |
| }, |
| { |
| "completion_length": 492.875, |
| "epoch": 0.23880597014925373, |
| "grad_norm": 0.24282296001911163, |
| "kl": 0.0003622036019805819, |
| "learning_rate": 4.717359720909053e-06, |
| "loss": 0.0, |
| "reward": 0.03125, |
| "reward_std": 0.0883883461356163, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.03125, |
| "step": 80 |
| }, |
| { |
| "completion_length": 241.875, |
| "epoch": 0.2417910447761194, |
| "grad_norm": 0.2795076370239258, |
| "kl": 0.0006243584793992341, |
| "learning_rate": 4.705187422877931e-06, |
| "loss": 0.0, |
| "reward": 0.01575000025331974, |
| "reward_std": 0.044547729194164276, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.01575000025331974, |
| "step": 81 |
| }, |
| { |
| "completion_length": 405.0, |
| "epoch": 0.24477611940298508, |
| "grad_norm": 0.3399125039577484, |
| "kl": 0.00068903889041394, |
| "learning_rate": 4.692774905313687e-06, |
| "loss": 0.0, |
| "reward": -0.051500000059604645, |
| "reward_std": 0.2009889781475067, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": -0.051500000059604645, |
| "step": 82 |
| }, |
| { |
| "completion_length": 334.5, |
| "epoch": 0.24776119402985075, |
| "grad_norm": 0.0014138659462332726, |
| "kl": 0.0003496205317787826, |
| "learning_rate": 4.68012352035952e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 83 |
| }, |
| { |
| "completion_length": 446.0, |
| "epoch": 0.2507462686567164, |
| "grad_norm": 0.0010413933778181672, |
| "kl": 0.0003284848644398153, |
| "learning_rate": 4.667234646179368e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 84 |
| }, |
| { |
| "completion_length": 487.0, |
| "epoch": 0.2537313432835821, |
| "grad_norm": 0.001018830225802958, |
| "kl": 0.0002749576815403998, |
| "learning_rate": 4.654109686807787e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 85 |
| }, |
| { |
| "completion_length": 295.5, |
| "epoch": 0.25671641791044775, |
| "grad_norm": 0.0019163364777341485, |
| "kl": 0.0005629758234135807, |
| "learning_rate": 4.640750071996995e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 86 |
| }, |
| { |
| "completion_length": 318.25, |
| "epoch": 0.25970149253731345, |
| "grad_norm": 0.5596569180488586, |
| "kl": 0.0005682025803253055, |
| "learning_rate": 4.62715725706113e-06, |
| "loss": 0.0, |
| "reward": 0.015625, |
| "reward_std": 0.04419417306780815, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.015625, |
| "step": 87 |
| }, |
| { |
| "completion_length": 361.0, |
| "epoch": 0.2626865671641791, |
| "grad_norm": 0.0013579176738858223, |
| "kl": 0.0004695644020102918, |
| "learning_rate": 4.613332722717714e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 88 |
| }, |
| { |
| "completion_length": 555.625, |
| "epoch": 0.2656716417910448, |
| "grad_norm": 0.22543174028396606, |
| "kl": 0.0003525623178575188, |
| "learning_rate": 4.599277974926355e-06, |
| "loss": 0.0, |
| "reward": -0.05037499964237213, |
| "reward_std": 0.14248201251029968, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": -0.05037499964237213, |
| "step": 89 |
| }, |
| { |
| "completion_length": 300.625, |
| "epoch": 0.26865671641791045, |
| "grad_norm": 0.21516487002372742, |
| "kl": 0.0005444185808300972, |
| "learning_rate": 4.584994544724695e-06, |
| "loss": 0.0, |
| "reward": 0.19550000131130219, |
| "reward_std": 0.5529574751853943, |
| "rewards/correctness_reward_func": 0.25, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": -0.054499998688697815, |
| "step": 90 |
| }, |
| { |
| "completion_length": 495.25, |
| "epoch": 0.2716417910447761, |
| "grad_norm": 0.22942061722278595, |
| "kl": 0.0016139632789418101, |
| "learning_rate": 4.57048398806163e-06, |
| "loss": 0.0001, |
| "reward": -0.07262499630451202, |
| "reward_std": 0.3230404257774353, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": -0.07262499630451202, |
| "step": 91 |
| }, |
| { |
| "completion_length": 333.375, |
| "epoch": 0.2746268656716418, |
| "grad_norm": 0.001300108153373003, |
| "kl": 0.0003806241147685796, |
| "learning_rate": 4.555747885627812e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 92 |
| }, |
| { |
| "completion_length": 285.125, |
| "epoch": 0.27761194029850744, |
| "grad_norm": 0.0014808428240939975, |
| "kl": 0.00030873267678543925, |
| "learning_rate": 4.540787842683459e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 93 |
| }, |
| { |
| "completion_length": 148.375, |
| "epoch": 0.28059701492537314, |
| "grad_norm": 0.005105325020849705, |
| "kl": 0.0015376665396615863, |
| "learning_rate": 4.525605488883493e-06, |
| "loss": 0.0001, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 94 |
| }, |
| { |
| "completion_length": 302.125, |
| "epoch": 0.2835820895522388, |
| "grad_norm": 0.00199362775310874, |
| "kl": 0.00041642598807811737, |
| "learning_rate": 4.510202478100008e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 95 |
| }, |
| { |
| "completion_length": 182.625, |
| "epoch": 0.2865671641791045, |
| "grad_norm": 0.0016307829646393657, |
| "kl": 0.0004398175806272775, |
| "learning_rate": 4.494580488242109e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 96 |
| }, |
| { |
| "completion_length": 226.875, |
| "epoch": 0.28955223880597014, |
| "grad_norm": 0.41053909063339233, |
| "kl": 0.0007566440617665648, |
| "learning_rate": 4.478741221073136e-06, |
| "loss": 0.0, |
| "reward": 0.015625, |
| "reward_std": 0.04419417306780815, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.015625, |
| "step": 97 |
| }, |
| { |
| "completion_length": 375.25, |
| "epoch": 0.29253731343283584, |
| "grad_norm": 0.17745234072208405, |
| "kl": 0.00043587430263869464, |
| "learning_rate": 4.462686402025277e-06, |
| "loss": 0.0, |
| "reward": 0.015625, |
| "reward_std": 0.04419417306780815, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.015625, |
| "step": 98 |
| }, |
| { |
| "completion_length": 546.25, |
| "epoch": 0.2955223880597015, |
| "grad_norm": 0.1550239473581314, |
| "kl": 0.00024276912154164165, |
| "learning_rate": 4.446417780011618e-06, |
| "loss": 0.0, |
| "reward": 0.015625, |
| "reward_std": 0.04419417306780815, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.015625, |
| "step": 99 |
| }, |
| { |
| "completion_length": 316.25, |
| "epoch": 0.29850746268656714, |
| "grad_norm": 0.22868771851062775, |
| "kl": 0.000838124135043472, |
| "learning_rate": 4.42993712723562e-06, |
| "loss": 0.0, |
| "reward": 0.04699999839067459, |
| "reward_std": 0.13293607532978058, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.04699999839067459, |
| "step": 100 |
| }, |
| { |
| "completion_length": 645.0, |
| "epoch": 0.30149253731343284, |
| "grad_norm": 0.1801333874464035, |
| "kl": 0.00038152255001477897, |
| "learning_rate": 4.413246238998069e-06, |
| "loss": 0.0, |
| "reward": -0.06024999916553497, |
| "reward_std": 0.17041273415088654, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": -0.06024999916553497, |
| "step": 101 |
| }, |
| { |
| "completion_length": 220.875, |
| "epoch": 0.3044776119402985, |
| "grad_norm": 0.0017630718648433685, |
| "kl": 0.00038715871050953865, |
| "learning_rate": 4.396346933501508e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 102 |
| }, |
| { |
| "completion_length": 508.125, |
| "epoch": 0.3074626865671642, |
| "grad_norm": 0.20094433426856995, |
| "kl": 0.0006657220656052232, |
| "learning_rate": 4.379241051652174e-06, |
| "loss": 0.0, |
| "reward": 0.03125, |
| "reward_std": 0.0883883461356163, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.03125, |
| "step": 103 |
| }, |
| { |
| "completion_length": 501.5, |
| "epoch": 0.31044776119402984, |
| "grad_norm": 0.17015448212623596, |
| "kl": 0.000687247549649328, |
| "learning_rate": 4.361930456859455e-06, |
| "loss": 0.0, |
| "reward": 0.04699999839067459, |
| "reward_std": 0.13293607532978058, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.04699999839067459, |
| "step": 104 |
| }, |
| { |
| "completion_length": 414.875, |
| "epoch": 0.31343283582089554, |
| "grad_norm": 0.0016345038311555982, |
| "kl": 0.00038708740612491965, |
| "learning_rate": 4.3444170348329095e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 105 |
| }, |
| { |
| "completion_length": 371.25, |
| "epoch": 0.3164179104477612, |
| "grad_norm": 0.00202854722738266, |
| "kl": 0.0006857087719254196, |
| "learning_rate": 4.326702693376844e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 106 |
| }, |
| { |
| "completion_length": 615.875, |
| "epoch": 0.3194029850746269, |
| "grad_norm": 0.1853620409965515, |
| "kl": 0.0010476625757291913, |
| "learning_rate": 4.308789362182492e-06, |
| "loss": 0.0, |
| "reward": -0.25699999928474426, |
| "reward_std": 0.7269057631492615, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": -0.25699999928474426, |
| "step": 107 |
| }, |
| { |
| "completion_length": 626.625, |
| "epoch": 0.32238805970149254, |
| "grad_norm": 0.22608496248722076, |
| "kl": 0.0009553819545544684, |
| "learning_rate": 4.290678992617797e-06, |
| "loss": 0.0, |
| "reward": 0.03137499839067459, |
| "reward_std": 0.05809582397341728, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.03137499839067459, |
| "step": 108 |
| }, |
| { |
| "completion_length": 374.0, |
| "epoch": 0.3253731343283582, |
| "grad_norm": 0.003469746559858322, |
| "kl": 0.001379699446260929, |
| "learning_rate": 4.2723735575148585e-06, |
| "loss": 0.0001, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 109 |
| }, |
| { |
| "completion_length": 412.875, |
| "epoch": 0.3283582089552239, |
| "grad_norm": 0.19724370539188385, |
| "kl": 0.0006244105170480907, |
| "learning_rate": 4.253875050955005e-06, |
| "loss": 0.0, |
| "reward": 0.015625, |
| "reward_std": 0.04419417306780815, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.015625, |
| "step": 110 |
| }, |
| { |
| "completion_length": 362.375, |
| "epoch": 0.33134328358208953, |
| "grad_norm": 0.17809787392616272, |
| "kl": 0.0009810776682570577, |
| "learning_rate": 4.2351854880515856e-06, |
| "loss": 0.0, |
| "reward": -0.34437501430511475, |
| "reward_std": 0.9740396738052368, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": -0.34437501430511475, |
| "step": 111 |
| }, |
| { |
| "completion_length": 254.375, |
| "epoch": 0.33432835820895523, |
| "grad_norm": 0.28602710366249084, |
| "kl": 0.0038207019679248333, |
| "learning_rate": 4.216306904730448e-06, |
| "loss": 0.0002, |
| "reward": 0.015625, |
| "reward_std": 0.04419417306780815, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.015625, |
| "step": 112 |
| }, |
| { |
| "completion_length": 304.25, |
| "epoch": 0.3373134328358209, |
| "grad_norm": 0.005425428505986929, |
| "kl": 0.0019627241417765617, |
| "learning_rate": 4.197241357508159e-06, |
| "loss": 0.0001, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 113 |
| }, |
| { |
| "completion_length": 281.0, |
| "epoch": 0.3402985074626866, |
| "grad_norm": 0.47344425320625305, |
| "kl": 0.010544579476118088, |
| "learning_rate": 4.177990923267986e-06, |
| "loss": 0.0004, |
| "reward": -0.07524999976158142, |
| "reward_std": 0.13945992290973663, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": -0.07524999976158142, |
| "step": 114 |
| }, |
| { |
| "completion_length": 486.5, |
| "epoch": 0.34328358208955223, |
| "grad_norm": 0.1647060066461563, |
| "kl": 0.003495145123451948, |
| "learning_rate": 4.158557699033644e-06, |
| "loss": 0.0001, |
| "reward": -0.11837500333786011, |
| "reward_std": 0.33481505513191223, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": -0.11837500333786011, |
| "step": 115 |
| }, |
| { |
| "completion_length": 316.125, |
| "epoch": 0.34626865671641793, |
| "grad_norm": 0.00556787708774209, |
| "kl": 0.002718576230108738, |
| "learning_rate": 4.138943801740865e-06, |
| "loss": 0.0001, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 116 |
| }, |
| { |
| "completion_length": 307.25, |
| "epoch": 0.3492537313432836, |
| "grad_norm": 0.3354603350162506, |
| "kl": 0.0044554159976542, |
| "learning_rate": 4.119151368006793e-06, |
| "loss": 0.0002, |
| "reward": 0.039750002324581146, |
| "reward_std": 0.07568685710430145, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.039750002324581146, |
| "step": 117 |
| }, |
| { |
| "completion_length": 527.75, |
| "epoch": 0.3522388059701492, |
| "grad_norm": 0.18804031610488892, |
| "kl": 0.0013875120785087347, |
| "learning_rate": 4.099182553897228e-06, |
| "loss": 0.0001, |
| "reward": 0.03125, |
| "reward_std": 0.0883883461356163, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.03125, |
| "step": 118 |
| }, |
| { |
| "completion_length": 233.625, |
| "epoch": 0.35522388059701493, |
| "grad_norm": 0.3928685188293457, |
| "kl": 0.004655573982745409, |
| "learning_rate": 4.0790395346917674e-06, |
| "loss": 0.0002, |
| "reward": 0.06274999678134918, |
| "reward_std": 0.11619041860103607, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.06274999678134918, |
| "step": 119 |
| }, |
| { |
| "completion_length": 455.0, |
| "epoch": 0.3582089552238806, |
| "grad_norm": 0.24913738667964935, |
| "kl": 0.0021684349048882723, |
| "learning_rate": 4.058724504646834e-06, |
| "loss": 0.0001, |
| "reward": -0.5518749952316284, |
| "reward_std": 0.8652141690254211, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": -0.5518749952316284, |
| "step": 120 |
| }, |
| { |
| "completion_length": 504.75, |
| "epoch": 0.3611940298507463, |
| "grad_norm": 0.25603148341178894, |
| "kl": 0.0012253073509782553, |
| "learning_rate": 4.038239676756654e-06, |
| "loss": 0.0, |
| "reward": -0.05650000274181366, |
| "reward_std": 0.2148142158985138, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": -0.05650000274181366, |
| "step": 121 |
| }, |
| { |
| "completion_length": 479.125, |
| "epoch": 0.3641791044776119, |
| "grad_norm": 0.2229258269071579, |
| "kl": 0.0028934157453477383, |
| "learning_rate": 4.017587282512181e-06, |
| "loss": 0.0001, |
| "reward": -0.1338750123977661, |
| "reward_std": 0.5096223950386047, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": -0.13387499749660492, |
| "step": 122 |
| }, |
| { |
| "completion_length": 372.875, |
| "epoch": 0.36716417910447763, |
| "grad_norm": 0.27619239687919617, |
| "kl": 0.002216364722698927, |
| "learning_rate": 3.996769571658022e-06, |
| "loss": 0.0001, |
| "reward": -0.226624995470047, |
| "reward_std": 0.6928819417953491, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": -0.226624995470047, |
| "step": 123 |
| }, |
| { |
| "completion_length": 461.5, |
| "epoch": 0.3701492537313433, |
| "grad_norm": 0.19344386458396912, |
| "kl": 0.002842097310349345, |
| "learning_rate": 3.975788811947351e-06, |
| "loss": 0.0001, |
| "reward": 0.281125009059906, |
| "reward_std": 0.745917558670044, |
| "rewards/correctness_reward_func": 0.25, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.03112499974668026, |
| "step": 124 |
| }, |
| { |
| "completion_length": 309.75, |
| "epoch": 0.373134328358209, |
| "grad_norm": 0.2874149978160858, |
| "kl": 0.0032137392554432154, |
| "learning_rate": 3.9546472888948825e-06, |
| "loss": 0.0001, |
| "reward": 0.015625, |
| "reward_std": 0.04419417306780815, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.015625, |
| "step": 125 |
| }, |
| { |
| "completion_length": 221.375, |
| "epoch": 0.3761194029850746, |
| "grad_norm": 0.009057161398231983, |
| "kl": 0.005167881492525339, |
| "learning_rate": 3.933347305527898e-06, |
| "loss": 0.0002, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 126 |
| }, |
| { |
| "completion_length": 435.875, |
| "epoch": 0.37910447761194027, |
| "grad_norm": 0.002526797354221344, |
| "kl": 0.0017403013771399856, |
| "learning_rate": 3.911891182135371e-06, |
| "loss": 0.0001, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 127 |
| }, |
| { |
| "completion_length": 449.75, |
| "epoch": 0.382089552238806, |
| "grad_norm": 0.0034000773448497057, |
| "kl": 0.00144351611379534, |
| "learning_rate": 3.890281256015207e-06, |
| "loss": 0.0001, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 128 |
| }, |
| { |
| "completion_length": 330.875, |
| "epoch": 0.3850746268656716, |
| "grad_norm": 0.18436795473098755, |
| "kl": 0.0032451425213366747, |
| "learning_rate": 3.868519881219631e-06, |
| "loss": 0.0001, |
| "reward": -0.11350000649690628, |
| "reward_std": 0.43147987127304077, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": -0.11350000649690628, |
| "step": 129 |
| }, |
| { |
| "completion_length": 265.625, |
| "epoch": 0.3880597014925373, |
| "grad_norm": 0.011368674226105213, |
| "kl": 0.005110865458846092, |
| "learning_rate": 3.8466094282987575e-06, |
| "loss": 0.0002, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 130 |
| }, |
| { |
| "completion_length": 261.625, |
| "epoch": 0.39104477611940297, |
| "grad_norm": 0.3966436982154846, |
| "kl": 0.004083710256963968, |
| "learning_rate": 3.824552284042351e-06, |
| "loss": 0.0002, |
| "reward": 0.01575000025331974, |
| "reward_std": 0.044547729194164276, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.01575000025331974, |
| "step": 131 |
| }, |
| { |
| "completion_length": 371.125, |
| "epoch": 0.3940298507462687, |
| "grad_norm": 0.0057938722893595695, |
| "kl": 0.003004050347954035, |
| "learning_rate": 3.802350851219826e-06, |
| "loss": 0.0001, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 132 |
| }, |
| { |
| "completion_length": 496.75, |
| "epoch": 0.3970149253731343, |
| "grad_norm": 0.24294711649417877, |
| "kl": 0.004811118356883526, |
| "learning_rate": 3.7800075483185073e-06, |
| "loss": 0.0002, |
| "reward": 0.016750000417232513, |
| "reward_std": 0.1915021389722824, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.016750000417232513, |
| "step": 133 |
| }, |
| { |
| "completion_length": 292.5, |
| "epoch": 0.4, |
| "grad_norm": 0.004627264104783535, |
| "kl": 0.002543792361393571, |
| "learning_rate": 3.7575248092801686e-06, |
| "loss": 0.0001, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 134 |
| }, |
| { |
| "completion_length": 304.125, |
| "epoch": 0.40298507462686567, |
| "grad_norm": 0.007394778076559305, |
| "kl": 0.003581845434382558, |
| "learning_rate": 3.734905083235901e-06, |
| "loss": 0.0001, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 135 |
| }, |
| { |
| "completion_length": 365.375, |
| "epoch": 0.4059701492537313, |
| "grad_norm": 0.2609359323978424, |
| "kl": 0.004685729276388884, |
| "learning_rate": 3.712150834239313e-06, |
| "loss": 0.0002, |
| "reward": 0.015625, |
| "reward_std": 0.04419417306780815, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.015625, |
| "step": 136 |
| }, |
| { |
| "completion_length": 521.25, |
| "epoch": 0.408955223880597, |
| "grad_norm": 0.31001636385917664, |
| "kl": 0.002258291933685541, |
| "learning_rate": 3.6892645409981166e-06, |
| "loss": 0.0001, |
| "reward": 0.046875, |
| "reward_std": 0.09300297498703003, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.046875, |
| "step": 137 |
| }, |
| { |
| "completion_length": 237.875, |
| "epoch": 0.41194029850746267, |
| "grad_norm": 0.24746236205101013, |
| "kl": 0.007873887196183205, |
| "learning_rate": 3.6662486966041104e-06, |
| "loss": 0.0003, |
| "reward": -0.011750001460313797, |
| "reward_std": 0.14610245823860168, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": -0.011749999597668648, |
| "step": 138 |
| }, |
| { |
| "completion_length": 510.875, |
| "epoch": 0.41492537313432837, |
| "grad_norm": 0.2037159949541092, |
| "kl": 0.0031850896775722504, |
| "learning_rate": 3.6431058082615966e-06, |
| "loss": 0.0001, |
| "reward": 0.015625, |
| "reward_std": 0.04419417306780815, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.015625, |
| "step": 139 |
| }, |
| { |
| "completion_length": 371.125, |
| "epoch": 0.417910447761194, |
| "grad_norm": 0.25025588274002075, |
| "kl": 0.0022993905004113913, |
| "learning_rate": 3.619838397014263e-06, |
| "loss": 0.0001, |
| "reward": 0.03137499839067459, |
| "reward_std": 0.08874189853668213, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.03137499839067459, |
| "step": 140 |
| }, |
| { |
| "completion_length": 451.0, |
| "epoch": 0.4208955223880597, |
| "grad_norm": 0.0034099018666893244, |
| "kl": 0.0028660029638558626, |
| "learning_rate": 3.5964489974705553e-06, |
| "loss": 0.0001, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 141 |
| }, |
| { |
| "completion_length": 497.75, |
| "epoch": 0.42388059701492536, |
| "grad_norm": 0.4137725234031677, |
| "kl": 0.0026251752860844135, |
| "learning_rate": 3.5729401575275724e-06, |
| "loss": 0.0001, |
| "reward": 0.015625, |
| "reward_std": 0.04419417306780815, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.015625, |
| "step": 142 |
| }, |
| { |
| "completion_length": 320.625, |
| "epoch": 0.42686567164179107, |
| "grad_norm": 0.004924552049487829, |
| "kl": 0.0024821085389703512, |
| "learning_rate": 3.5493144380935155e-06, |
| "loss": 0.0001, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 143 |
| }, |
| { |
| "completion_length": 297.125, |
| "epoch": 0.4298507462686567, |
| "grad_norm": 0.006190278101712465, |
| "kl": 0.0024942473974078894, |
| "learning_rate": 3.5255744128087175e-06, |
| "loss": 0.0001, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 144 |
| }, |
| { |
| "completion_length": 484.0, |
| "epoch": 0.43283582089552236, |
| "grad_norm": 0.32550495862960815, |
| "kl": 0.0038871050346642733, |
| "learning_rate": 3.501722667765286e-06, |
| "loss": 0.0002, |
| "reward": 0.046875, |
| "reward_std": 0.0646936446428299, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.046875, |
| "step": 145 |
| }, |
| { |
| "completion_length": 325.5, |
| "epoch": 0.43582089552238806, |
| "grad_norm": 0.39831942319869995, |
| "kl": 0.00664663827046752, |
| "learning_rate": 3.47776180122539e-06, |
| "loss": 0.0003, |
| "reward": 0.007000000216066837, |
| "reward_std": 0.019798990339040756, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.007000000216066837, |
| "step": 146 |
| }, |
| { |
| "completion_length": 716.875, |
| "epoch": 0.4388059701492537, |
| "grad_norm": 0.0020847241394221783, |
| "kl": 0.0007909125415608287, |
| "learning_rate": 3.4536944233382248e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 147 |
| }, |
| { |
| "completion_length": 581.875, |
| "epoch": 0.4417910447761194, |
| "grad_norm": 0.21988965570926666, |
| "kl": 0.0023626910988241434, |
| "learning_rate": 3.429523155855672e-06, |
| "loss": 0.0001, |
| "reward": 0.046875, |
| "reward_std": 0.0646936446428299, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.046875, |
| "step": 148 |
| }, |
| { |
| "completion_length": 259.625, |
| "epoch": 0.44477611940298506, |
| "grad_norm": 0.26226532459259033, |
| "kl": 0.0048947930335998535, |
| "learning_rate": 3.405250631846708e-06, |
| "loss": 0.0002, |
| "reward": 0.06274999678134918, |
| "reward_std": 0.0948694571852684, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.06274999678134918, |
| "step": 149 |
| }, |
| { |
| "completion_length": 632.375, |
| "epoch": 0.44776119402985076, |
| "grad_norm": 0.20669730007648468, |
| "kl": 0.0018882754957303405, |
| "learning_rate": 3.3808794954105716e-06, |
| "loss": 0.0001, |
| "reward": 0.03125, |
| "reward_std": 0.0578637570142746, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.03125, |
| "step": 150 |
| }, |
| { |
| "completion_length": 290.875, |
| "epoch": 0.4507462686567164, |
| "grad_norm": 0.003552640788257122, |
| "kl": 0.0016134099569171667, |
| "learning_rate": 3.3564124013887324e-06, |
| "loss": 0.0001, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 151 |
| }, |
| { |
| "completion_length": 177.75, |
| "epoch": 0.4537313432835821, |
| "grad_norm": 0.009745140559971333, |
| "kl": 0.004595119506120682, |
| "learning_rate": 3.331852015075685e-06, |
| "loss": 0.0002, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 152 |
| }, |
| { |
| "completion_length": 479.5, |
| "epoch": 0.45671641791044776, |
| "grad_norm": 0.27110740542411804, |
| "kl": 0.0047483891248703, |
| "learning_rate": 3.3072010119286156e-06, |
| "loss": 0.0002, |
| "reward": 0.02812499925494194, |
| "reward_std": 0.05250425264239311, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.02812499925494194, |
| "step": 153 |
| }, |
| { |
| "completion_length": 220.5, |
| "epoch": 0.4597014925373134, |
| "grad_norm": 0.4211854636669159, |
| "kl": 0.006590801756829023, |
| "learning_rate": 3.2824620772759475e-06, |
| "loss": 0.0003, |
| "reward": 0.015625, |
| "reward_std": 0.04419417306780815, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.015625, |
| "step": 154 |
| }, |
| { |
| "completion_length": 241.625, |
| "epoch": 0.4626865671641791, |
| "grad_norm": 0.3244410753250122, |
| "kl": 0.003724359441548586, |
| "learning_rate": 3.257637906024822e-06, |
| "loss": 0.0001, |
| "reward": 0.015625, |
| "reward_std": 0.04419417306780815, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.015625, |
| "step": 155 |
| }, |
| { |
| "completion_length": 414.5, |
| "epoch": 0.46567164179104475, |
| "grad_norm": 0.2715875506401062, |
| "kl": 0.0029947988223284483, |
| "learning_rate": 3.2327312023675287e-06, |
| "loss": 0.0001, |
| "reward": 0.03125, |
| "reward_std": 0.0883883461356163, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.03125, |
| "step": 156 |
| }, |
| { |
| "completion_length": 196.375, |
| "epoch": 0.46865671641791046, |
| "grad_norm": 0.007158320862799883, |
| "kl": 0.0028687817975878716, |
| "learning_rate": 3.20774467948693e-06, |
| "loss": 0.0001, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 157 |
| }, |
| { |
| "completion_length": 275.375, |
| "epoch": 0.4716417910447761, |
| "grad_norm": 0.3656024932861328, |
| "kl": 0.007250001188367605, |
| "learning_rate": 3.1826810592609036e-06, |
| "loss": 0.0003, |
| "reward": 0.046875, |
| "reward_std": 0.0646936446428299, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.046875, |
| "step": 158 |
| }, |
| { |
| "completion_length": 285.875, |
| "epoch": 0.4746268656716418, |
| "grad_norm": 0.4725809693336487, |
| "kl": 0.003180581144988537, |
| "learning_rate": 3.157543071965835e-06, |
| "loss": 0.0001, |
| "reward": 0.03125, |
| "reward_std": 0.0883883461356163, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.03125, |
| "step": 159 |
| }, |
| { |
| "completion_length": 580.125, |
| "epoch": 0.47761194029850745, |
| "grad_norm": 0.30390796065330505, |
| "kl": 0.004015565849840641, |
| "learning_rate": 3.132333455979202e-06, |
| "loss": 0.0002, |
| "reward": -0.017625000327825546, |
| "reward_std": 0.17501260340213776, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": -0.017625000327825546, |
| "step": 160 |
| }, |
| { |
| "completion_length": 292.75, |
| "epoch": 0.48059701492537316, |
| "grad_norm": 0.010895282961428165, |
| "kl": 0.004954541102051735, |
| "learning_rate": 3.107054957481271e-06, |
| "loss": 0.0002, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 161 |
| }, |
| { |
| "completion_length": 260.25, |
| "epoch": 0.4835820895522388, |
| "grad_norm": 0.29293543100357056, |
| "kl": 0.006690036039799452, |
| "learning_rate": 3.0817103301559422e-06, |
| "loss": 0.0003, |
| "reward": 0.06262499839067459, |
| "reward_std": 0.13396474719047546, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.06262499839067459, |
| "step": 162 |
| }, |
| { |
| "completion_length": 329.25, |
| "epoch": 0.48656716417910445, |
| "grad_norm": 0.28982600569725037, |
| "kl": 0.004020814783871174, |
| "learning_rate": 3.056302334890786e-06, |
| "loss": 0.0002, |
| "reward": 0.046875, |
| "reward_std": 0.09300297498703003, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.046875, |
| "step": 163 |
| }, |
| { |
| "completion_length": 211.25, |
| "epoch": 0.48955223880597015, |
| "grad_norm": 0.5855104327201843, |
| "kl": 0.017538880929350853, |
| "learning_rate": 3.030833739476285e-06, |
| "loss": 0.0007, |
| "reward": 0.9138749837875366, |
| "reward_std": 1.1292022466659546, |
| "rewards/correctness_reward_func": 0.75, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.16387498378753662, |
| "step": 164 |
| }, |
| { |
| "completion_length": 488.75, |
| "epoch": 0.4925373134328358, |
| "grad_norm": 0.21862053871154785, |
| "kl": 0.0072796959429979324, |
| "learning_rate": 3.0053073183043257e-06, |
| "loss": 0.0003, |
| "reward": 0.0298750102519989, |
| "reward_std": 1.1403151750564575, |
| "rewards/correctness_reward_func": 0.25, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": -0.2201249897480011, |
| "step": 165 |
| }, |
| { |
| "completion_length": 499.75, |
| "epoch": 0.4955223880597015, |
| "grad_norm": 0.2587985396385193, |
| "kl": 0.006187543738633394, |
| "learning_rate": 2.979725852065981e-06, |
| "loss": 0.0002, |
| "reward": -0.5041249990463257, |
| "reward_std": 1.0025380849838257, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": -0.5041249990463257, |
| "step": 166 |
| }, |
| { |
| "completion_length": 473.375, |
| "epoch": 0.49850746268656715, |
| "grad_norm": 0.25732216238975525, |
| "kl": 0.0039559113793075085, |
| "learning_rate": 2.9540921274485913e-06, |
| "loss": 0.0002, |
| "reward": -0.11225000023841858, |
| "reward_std": 0.370589017868042, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": -0.11225000023841858, |
| "step": 167 |
| }, |
| { |
| "completion_length": 487.625, |
| "epoch": 0.5014925373134328, |
| "grad_norm": 0.2064179629087448, |
| "kl": 0.003909274935722351, |
| "learning_rate": 2.9284089368322044e-06, |
| "loss": 0.0002, |
| "reward": 0.04699999839067459, |
| "reward_std": 0.13293607532978058, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.04699999839067459, |
| "step": 168 |
| }, |
| { |
| "completion_length": 692.75, |
| "epoch": 0.5044776119402985, |
| "grad_norm": 0.17778423428535461, |
| "kl": 0.0032577835954725742, |
| "learning_rate": 2.9026790779853877e-06, |
| "loss": 0.0001, |
| "reward": 0.015625, |
| "reward_std": 0.04419417306780815, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.015625, |
| "step": 169 |
| }, |
| { |
| "completion_length": 338.625, |
| "epoch": 0.5074626865671642, |
| "grad_norm": 0.4213738739490509, |
| "kl": 0.004577150102704763, |
| "learning_rate": 2.876905353760459e-06, |
| "loss": 0.0002, |
| "reward": -0.00612499937415123, |
| "reward_std": 0.19183358550071716, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": -0.00612499937415123, |
| "step": 170 |
| }, |
| { |
| "completion_length": 142.875, |
| "epoch": 0.5104477611940299, |
| "grad_norm": 0.38132330775260925, |
| "kl": 0.02254224196076393, |
| "learning_rate": 2.8510905717881615e-06, |
| "loss": 0.0009, |
| "reward": 0.31575000286102295, |
| "reward_std": 0.783475935459137, |
| "rewards/correctness_reward_func": 0.25, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.06575000286102295, |
| "step": 171 |
| }, |
| { |
| "completion_length": 542.0, |
| "epoch": 0.5134328358208955, |
| "grad_norm": 0.28400880098342896, |
| "kl": 0.006651477422565222, |
| "learning_rate": 2.8252375441718137e-06, |
| "loss": 0.0003, |
| "reward": 0.07837499678134918, |
| "reward_std": 0.09333952516317368, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.07837499678134918, |
| "step": 172 |
| }, |
| { |
| "completion_length": 460.5, |
| "epoch": 0.5164179104477612, |
| "grad_norm": 0.21609985828399658, |
| "kl": 0.007514914497733116, |
| "learning_rate": 2.7993490871809808e-06, |
| "loss": 0.0003, |
| "reward": -0.07925000041723251, |
| "reward_std": 0.27812162041664124, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": -0.07925000041723251, |
| "step": 173 |
| }, |
| { |
| "completion_length": 187.5, |
| "epoch": 0.5194029850746269, |
| "grad_norm": 0.42248091101646423, |
| "kl": 0.007775603327900171, |
| "learning_rate": 2.773428020944687e-06, |
| "loss": 0.0003, |
| "reward": 0.015625, |
| "reward_std": 0.04419417306780815, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.015625, |
| "step": 174 |
| }, |
| { |
| "completion_length": 276.75, |
| "epoch": 0.5223880597014925, |
| "grad_norm": 0.3561972677707672, |
| "kl": 0.013817128725349903, |
| "learning_rate": 2.747477169144202e-06, |
| "loss": 0.0006, |
| "reward": 0.140625, |
| "reward_std": 0.12387890368700027, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.140625, |
| "step": 175 |
| }, |
| { |
| "completion_length": 217.875, |
| "epoch": 0.5253731343283582, |
| "grad_norm": 0.5078591108322144, |
| "kl": 0.01669302023947239, |
| "learning_rate": 2.721499358705458e-06, |
| "loss": 0.0007, |
| "reward": 0.046875, |
| "reward_std": 0.0646936446428299, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.046875, |
| "step": 176 |
| }, |
| { |
| "completion_length": 516.125, |
| "epoch": 0.5283582089552239, |
| "grad_norm": 0.2574947476387024, |
| "kl": 0.005081856623291969, |
| "learning_rate": 2.695497419491089e-06, |
| "loss": 0.0002, |
| "reward": -0.08912499994039536, |
| "reward_std": 0.572532594203949, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": -0.08912500739097595, |
| "step": 177 |
| }, |
| { |
| "completion_length": 460.75, |
| "epoch": 0.5313432835820896, |
| "grad_norm": 0.22305476665496826, |
| "kl": 0.015017062425613403, |
| "learning_rate": 2.6694741839921734e-06, |
| "loss": 0.0006, |
| "reward": 0.34375, |
| "reward_std": 0.7756046056747437, |
| "rewards/correctness_reward_func": 0.25, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.09375, |
| "step": 178 |
| }, |
| { |
| "completion_length": 358.875, |
| "epoch": 0.5343283582089552, |
| "grad_norm": 0.351471871137619, |
| "kl": 0.01629817485809326, |
| "learning_rate": 2.6434324870196746e-06, |
| "loss": 0.0007, |
| "reward": 0.007375001907348633, |
| "reward_std": 0.4031447470188141, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.007374994456768036, |
| "step": 179 |
| }, |
| { |
| "completion_length": 592.0, |
| "epoch": 0.5373134328358209, |
| "grad_norm": 0.2376508116722107, |
| "kl": 0.0033545636106282473, |
| "learning_rate": 2.617375165395634e-06, |
| "loss": 0.0001, |
| "reward": 0.07824999839067459, |
| "reward_std": 0.0932672768831253, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.07824999839067459, |
| "step": 180 |
| }, |
| { |
| "completion_length": 243.375, |
| "epoch": 0.5402985074626866, |
| "grad_norm": 0.38199833035469055, |
| "kl": 0.013456111773848534, |
| "learning_rate": 2.591305057644148e-06, |
| "loss": 0.0005, |
| "reward": 0.625124990940094, |
| "reward_std": 0.9728089570999146, |
| "rewards/correctness_reward_func": 0.5, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.125124990940094, |
| "step": 181 |
| }, |
| { |
| "completion_length": 369.75, |
| "epoch": 0.5432835820895522, |
| "grad_norm": 0.38759690523147583, |
| "kl": 0.035215843468904495, |
| "learning_rate": 2.5652250036821522e-06, |
| "loss": 0.0014, |
| "reward": 0.03612499684095383, |
| "reward_std": 0.2709351181983948, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.036125000566244125, |
| "step": 182 |
| }, |
| { |
| "completion_length": 232.25, |
| "epoch": 0.5462686567164179, |
| "grad_norm": 0.37896519899368286, |
| "kl": 0.021390574052929878, |
| "learning_rate": 2.5391378445100646e-06, |
| "loss": 0.0009, |
| "reward": 0.17350000143051147, |
| "reward_std": 0.14292055368423462, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.17350000143051147, |
| "step": 183 |
| }, |
| { |
| "completion_length": 409.125, |
| "epoch": 0.5492537313432836, |
| "grad_norm": 0.3088514506816864, |
| "kl": 0.014571096748113632, |
| "learning_rate": 2.5130464219022994e-06, |
| "loss": 0.0006, |
| "reward": 0.1146249994635582, |
| "reward_std": 0.1522197127342224, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.1146249994635582, |
| "step": 184 |
| }, |
| { |
| "completion_length": 242.25, |
| "epoch": 0.5522388059701493, |
| "grad_norm": 0.3677271604537964, |
| "kl": 0.012131447903811932, |
| "learning_rate": 2.4869535780977023e-06, |
| "loss": 0.0005, |
| "reward": 0.18474999070167542, |
| "reward_std": 0.0707283467054367, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.18474999070167542, |
| "step": 185 |
| }, |
| { |
| "completion_length": 421.875, |
| "epoch": 0.5552238805970149, |
| "grad_norm": 0.19892147183418274, |
| "kl": 0.009639453142881393, |
| "learning_rate": 2.460862155489936e-06, |
| "loss": 0.0004, |
| "reward": 0.125, |
| "reward_std": 0.09449111670255661, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.125, |
| "step": 186 |
| }, |
| { |
| "completion_length": 615.375, |
| "epoch": 0.5582089552238806, |
| "grad_norm": 0.18018686771392822, |
| "kl": 0.005218564532697201, |
| "learning_rate": 2.4347749963178486e-06, |
| "loss": 0.0002, |
| "reward": 0.01262500137090683, |
| "reward_std": 0.3315926492214203, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.01262500137090683, |
| "step": 187 |
| }, |
| { |
| "completion_length": 409.875, |
| "epoch": 0.5611940298507463, |
| "grad_norm": 0.27012187242507935, |
| "kl": 0.02046685479581356, |
| "learning_rate": 2.408694942355853e-06, |
| "loss": 0.0008, |
| "reward": -0.04087501019239426, |
| "reward_std": 0.6781936883926392, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": -0.04087500274181366, |
| "step": 188 |
| }, |
| { |
| "completion_length": 394.125, |
| "epoch": 0.564179104477612, |
| "grad_norm": 0.30240562558174133, |
| "kl": 0.013552379794418812, |
| "learning_rate": 2.3826248346043664e-06, |
| "loss": 0.0005, |
| "reward": 0.3344999849796295, |
| "reward_std": 0.8115267157554626, |
| "rewards/correctness_reward_func": 0.25, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.08449999988079071, |
| "step": 189 |
| }, |
| { |
| "completion_length": 498.125, |
| "epoch": 0.5671641791044776, |
| "grad_norm": 0.22516800463199615, |
| "kl": 0.007915965281426907, |
| "learning_rate": 2.356567512980326e-06, |
| "loss": 0.0003, |
| "reward": -0.17287498712539673, |
| "reward_std": 0.37088099122047424, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": -0.17287500202655792, |
| "step": 190 |
| }, |
| { |
| "completion_length": 437.375, |
| "epoch": 0.5701492537313433, |
| "grad_norm": 0.26963454484939575, |
| "kl": 0.013868818990886211, |
| "learning_rate": 2.3305258160078274e-06, |
| "loss": 0.0006, |
| "reward": 0.671999990940094, |
| "reward_std": 0.979500949382782, |
| "rewards/correctness_reward_func": 0.5, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.171999990940094, |
| "step": 191 |
| }, |
| { |
| "completion_length": 486.75, |
| "epoch": 0.573134328358209, |
| "grad_norm": 0.2408941686153412, |
| "kl": 0.007514089345932007, |
| "learning_rate": 2.304502580508912e-06, |
| "loss": 0.0003, |
| "reward": 0.29612499475479126, |
| "reward_std": 0.5014508962631226, |
| "rewards/correctness_reward_func": 0.25, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.04612499475479126, |
| "step": 192 |
| }, |
| { |
| "completion_length": 664.25, |
| "epoch": 0.5761194029850746, |
| "grad_norm": 0.7232910394668579, |
| "kl": 0.009346898645162582, |
| "learning_rate": 2.278500641294543e-06, |
| "loss": 0.0004, |
| "reward": 0.046875, |
| "reward_std": 0.09300297498703003, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.046875, |
| "step": 193 |
| }, |
| { |
| "completion_length": 361.5, |
| "epoch": 0.5791044776119403, |
| "grad_norm": 0.2798665165901184, |
| "kl": 0.01410368550568819, |
| "learning_rate": 2.252522830855798e-06, |
| "loss": 0.0006, |
| "reward": 0.10837499797344208, |
| "reward_std": 0.14315219223499298, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.10837499797344208, |
| "step": 194 |
| }, |
| { |
| "completion_length": 366.75, |
| "epoch": 0.582089552238806, |
| "grad_norm": 0.4407714009284973, |
| "kl": 0.016179384663701057, |
| "learning_rate": 2.2265719790553147e-06, |
| "loss": 0.0006, |
| "reward": -0.09425000846385956, |
| "reward_std": 0.33279111981391907, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": -0.09425000101327896, |
| "step": 195 |
| }, |
| { |
| "completion_length": 483.625, |
| "epoch": 0.5850746268656717, |
| "grad_norm": 0.23635739088058472, |
| "kl": 0.009301292710006237, |
| "learning_rate": 2.2006509128190196e-06, |
| "loss": 0.0004, |
| "reward": -0.10637500137090683, |
| "reward_std": 0.4708363115787506, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": -0.10637500137090683, |
| "step": 196 |
| }, |
| { |
| "completion_length": 275.5, |
| "epoch": 0.5880597014925373, |
| "grad_norm": 0.30557289719581604, |
| "kl": 0.026175374165177345, |
| "learning_rate": 2.1747624558281867e-06, |
| "loss": 0.001, |
| "reward": 0.20949998497962952, |
| "reward_std": 0.15469324588775635, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.2094999998807907, |
| "step": 197 |
| }, |
| { |
| "completion_length": 318.25, |
| "epoch": 0.591044776119403, |
| "grad_norm": 0.426329642534256, |
| "kl": 0.019156118854880333, |
| "learning_rate": 2.1489094282118393e-06, |
| "loss": 0.0008, |
| "reward": 0.28349998593330383, |
| "reward_std": 0.6506469249725342, |
| "rewards/correctness_reward_func": 0.25, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.03349999710917473, |
| "step": 198 |
| }, |
| { |
| "completion_length": 658.0, |
| "epoch": 0.5940298507462687, |
| "grad_norm": 0.15060272812843323, |
| "kl": 0.004683774430304766, |
| "learning_rate": 2.1230946462395412e-06, |
| "loss": 0.0002, |
| "reward": 0.125, |
| "reward_std": 0.06681530922651291, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.125, |
| "step": 199 |
| }, |
| { |
| "completion_length": 305.0, |
| "epoch": 0.5970149253731343, |
| "grad_norm": 0.40997669100761414, |
| "kl": 0.01804080791771412, |
| "learning_rate": 2.0973209220146135e-06, |
| "loss": 0.0007, |
| "reward": 0.33412498235702515, |
| "reward_std": 0.7774099111557007, |
| "rewards/correctness_reward_func": 0.25, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.08412499725818634, |
| "step": 200 |
| }, |
| { |
| "completion_length": 590.0, |
| "epoch": 0.6, |
| "grad_norm": 0.2662385106086731, |
| "kl": 0.006966865621507168, |
| "learning_rate": 2.071591063167797e-06, |
| "loss": 0.0003, |
| "reward": -0.23762500286102295, |
| "reward_std": 0.4790639281272888, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": -0.23762500286102295, |
| "step": 201 |
| }, |
| { |
| "completion_length": 240.75, |
| "epoch": 0.6029850746268657, |
| "grad_norm": 0.43724602460861206, |
| "kl": 0.027211960405111313, |
| "learning_rate": 2.045907872551409e-06, |
| "loss": 0.0011, |
| "reward": 0.0846249908208847, |
| "reward_std": 0.24422587454319, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0846249982714653, |
| "step": 202 |
| }, |
| { |
| "completion_length": 463.0, |
| "epoch": 0.6059701492537314, |
| "grad_norm": 0.33706098794937134, |
| "kl": 0.012281076051294804, |
| "learning_rate": 2.0202741479340193e-06, |
| "loss": 0.0005, |
| "reward": 0.23412498831748962, |
| "reward_std": 0.1411113291978836, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.23412498831748962, |
| "step": 203 |
| }, |
| { |
| "completion_length": 223.625, |
| "epoch": 0.608955223880597, |
| "grad_norm": 0.32199856638908386, |
| "kl": 0.02061256766319275, |
| "learning_rate": 1.9946926816956743e-06, |
| "loss": 0.0008, |
| "reward": 0.15049999952316284, |
| "reward_std": 0.21179774403572083, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.15049999952316284, |
| "step": 204 |
| }, |
| { |
| "completion_length": 433.75, |
| "epoch": 0.6119402985074627, |
| "grad_norm": 0.16691480576992035, |
| "kl": 0.015048968605697155, |
| "learning_rate": 1.969166260523717e-06, |
| "loss": 0.0006, |
| "reward": 0.13649998605251312, |
| "reward_std": 0.33145737648010254, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.13650000095367432, |
| "step": 205 |
| }, |
| { |
| "completion_length": 363.125, |
| "epoch": 0.6149253731343284, |
| "grad_norm": 0.24160078167915344, |
| "kl": 0.010224804282188416, |
| "learning_rate": 1.9436976651092143e-06, |
| "loss": 0.0004, |
| "reward": 0.13512499630451202, |
| "reward_std": 0.08309278637170792, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.13512499630451202, |
| "step": 206 |
| }, |
| { |
| "completion_length": 616.625, |
| "epoch": 0.6179104477611941, |
| "grad_norm": 0.24529774487018585, |
| "kl": 0.009581487625837326, |
| "learning_rate": 1.918289669844058e-06, |
| "loss": 0.0004, |
| "reward": 0.12787500023841858, |
| "reward_std": 0.11601285636425018, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.12787500023841858, |
| "step": 207 |
| }, |
| { |
| "completion_length": 285.25, |
| "epoch": 0.6208955223880597, |
| "grad_norm": 0.4515547454357147, |
| "kl": 0.03280240669846535, |
| "learning_rate": 1.8929450425187298e-06, |
| "loss": 0.0013, |
| "reward": 0.13300000131130219, |
| "reward_std": 0.052966292947530746, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.13300000131130219, |
| "step": 208 |
| }, |
| { |
| "completion_length": 143.125, |
| "epoch": 0.6238805970149254, |
| "grad_norm": 0.4566623866558075, |
| "kl": 0.028078753501176834, |
| "learning_rate": 1.8676665440207982e-06, |
| "loss": 0.0011, |
| "reward": 0.8535000085830688, |
| "reward_std": 1.0311086177825928, |
| "rewards/correctness_reward_func": 0.75, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.10350000858306885, |
| "step": 209 |
| }, |
| { |
| "completion_length": 301.375, |
| "epoch": 0.6268656716417911, |
| "grad_norm": 0.35615095496177673, |
| "kl": 0.019768187776207924, |
| "learning_rate": 1.8424569280341653e-06, |
| "loss": 0.0008, |
| "reward": 0.5614999532699585, |
| "reward_std": 0.9637962579727173, |
| "rewards/correctness_reward_func": 0.5, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.06149999797344208, |
| "step": 210 |
| }, |
| { |
| "completion_length": 205.625, |
| "epoch": 0.6298507462686567, |
| "grad_norm": 0.572158932685852, |
| "kl": 0.03914448618888855, |
| "learning_rate": 1.817318940739098e-06, |
| "loss": 0.0016, |
| "reward": 1.1281249523162842, |
| "reward_std": 1.0498979091644287, |
| "rewards/correctness_reward_func": 1.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.12812499701976776, |
| "step": 211 |
| }, |
| { |
| "completion_length": 202.625, |
| "epoch": 0.6328358208955224, |
| "grad_norm": 0.41864466667175293, |
| "kl": 0.033789440989494324, |
| "learning_rate": 1.7922553205130708e-06, |
| "loss": 0.0014, |
| "reward": 0.453249990940094, |
| "reward_std": 0.7317192554473877, |
| "rewards/correctness_reward_func": 0.25, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.203249990940094, |
| "step": 212 |
| }, |
| { |
| "completion_length": 479.375, |
| "epoch": 0.6358208955223881, |
| "grad_norm": 0.19674842059612274, |
| "kl": 0.01116740982979536, |
| "learning_rate": 1.767268797632472e-06, |
| "loss": 0.0004, |
| "reward": 0.028875000774860382, |
| "reward_std": 0.4791833460330963, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.028875000774860382, |
| "step": 213 |
| }, |
| { |
| "completion_length": 228.375, |
| "epoch": 0.6388059701492538, |
| "grad_norm": 0.38439103960990906, |
| "kl": 0.026051824912428856, |
| "learning_rate": 1.7423620939751787e-06, |
| "loss": 0.001, |
| "reward": 0.029624998569488525, |
| "reward_std": 0.3076816201210022, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.029624998569488525, |
| "step": 214 |
| }, |
| { |
| "completion_length": 267.625, |
| "epoch": 0.6417910447761194, |
| "grad_norm": 0.42961663007736206, |
| "kl": 0.015496465377509594, |
| "learning_rate": 1.7175379227240524e-06, |
| "loss": 0.0006, |
| "reward": 0.28037500381469727, |
| "reward_std": 0.08738410472869873, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.28037500381469727, |
| "step": 215 |
| }, |
| { |
| "completion_length": 480.875, |
| "epoch": 0.6447761194029851, |
| "grad_norm": 0.2381271868944168, |
| "kl": 0.0213899165391922, |
| "learning_rate": 1.6927989880713852e-06, |
| "loss": 0.0009, |
| "reward": -0.02775000035762787, |
| "reward_std": 0.5771975517272949, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": -0.02775000035762787, |
| "step": 216 |
| }, |
| { |
| "completion_length": 406.75, |
| "epoch": 0.6477611940298508, |
| "grad_norm": 0.2900620102882385, |
| "kl": 0.015803145244717598, |
| "learning_rate": 1.6681479849243153e-06, |
| "loss": 0.0006, |
| "reward": 0.218874990940094, |
| "reward_std": 0.05794192850589752, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.218874990940094, |
| "step": 217 |
| }, |
| { |
| "completion_length": 457.5, |
| "epoch": 0.6507462686567164, |
| "grad_norm": 0.22088056802749634, |
| "kl": 0.014149850234389305, |
| "learning_rate": 1.6435875986112685e-06, |
| "loss": 0.0006, |
| "reward": 0.08675000071525574, |
| "reward_std": 0.2752093970775604, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.08675000071525574, |
| "step": 218 |
| }, |
| { |
| "completion_length": 466.0, |
| "epoch": 0.6537313432835821, |
| "grad_norm": 0.2606593668460846, |
| "kl": 0.010915166698396206, |
| "learning_rate": 1.6191205045894283e-06, |
| "loss": 0.0004, |
| "reward": 0.08262500166893005, |
| "reward_std": 0.21830511093139648, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.08262500166893005, |
| "step": 219 |
| }, |
| { |
| "completion_length": 549.875, |
| "epoch": 0.6567164179104478, |
| "grad_norm": 0.2776827812194824, |
| "kl": 0.01001064758747816, |
| "learning_rate": 1.594749368153292e-06, |
| "loss": 0.0004, |
| "reward": 0.10012499988079071, |
| "reward_std": 0.11426840722560883, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.10012499988079071, |
| "step": 220 |
| }, |
| { |
| "completion_length": 383.75, |
| "epoch": 0.6597014925373135, |
| "grad_norm": 0.3466421961784363, |
| "kl": 0.011461976915597916, |
| "learning_rate": 1.570476844144329e-06, |
| "loss": 0.0005, |
| "reward": 0.1875, |
| "reward_std": 0.06681530922651291, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.1875, |
| "step": 221 |
| }, |
| { |
| "completion_length": 349.625, |
| "epoch": 0.6626865671641791, |
| "grad_norm": 0.34904956817626953, |
| "kl": 0.023631775751709938, |
| "learning_rate": 1.5463055766617763e-06, |
| "loss": 0.0009, |
| "reward": 0.11737500131130219, |
| "reward_std": 0.25096383690834045, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.11737500131130219, |
| "step": 222 |
| }, |
| { |
| "completion_length": 427.75, |
| "epoch": 0.6656716417910448, |
| "grad_norm": 0.28804051876068115, |
| "kl": 0.02174968458712101, |
| "learning_rate": 1.5222381987746104e-06, |
| "loss": 0.0009, |
| "reward": 0.656374990940094, |
| "reward_std": 0.910563051700592, |
| "rewards/correctness_reward_func": 0.5, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.156374990940094, |
| "step": 223 |
| }, |
| { |
| "completion_length": 303.625, |
| "epoch": 0.6686567164179105, |
| "grad_norm": 0.24906383454799652, |
| "kl": 0.017897039651870728, |
| "learning_rate": 1.4982773322347144e-06, |
| "loss": 0.0007, |
| "reward": 0.453249990940094, |
| "reward_std": 0.7347634434700012, |
| "rewards/correctness_reward_func": 0.25, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.203249990940094, |
| "step": 224 |
| }, |
| { |
| "completion_length": 492.25, |
| "epoch": 0.6716417910447762, |
| "grad_norm": 0.24582958221435547, |
| "kl": 0.013822609558701515, |
| "learning_rate": 1.4744255871912825e-06, |
| "loss": 0.0006, |
| "reward": -0.2888749837875366, |
| "reward_std": 0.9461946487426758, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": -0.2888749837875366, |
| "step": 225 |
| }, |
| { |
| "completion_length": 240.5, |
| "epoch": 0.6746268656716418, |
| "grad_norm": 0.380571573972702, |
| "kl": 0.020389374345541, |
| "learning_rate": 1.4506855619064847e-06, |
| "loss": 0.0008, |
| "reward": 0.6603749990463257, |
| "reward_std": 0.9848659038543701, |
| "rewards/correctness_reward_func": 0.5, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.16037499904632568, |
| "step": 226 |
| }, |
| { |
| "completion_length": 142.75, |
| "epoch": 0.6776119402985075, |
| "grad_norm": 0.4420141875743866, |
| "kl": 0.029349079355597496, |
| "learning_rate": 1.4270598424724291e-06, |
| "loss": 0.0012, |
| "reward": 0.9844999313354492, |
| "reward_std": 0.9861009120941162, |
| "rewards/correctness_reward_func": 0.75, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.234499990940094, |
| "step": 227 |
| }, |
| { |
| "completion_length": 311.375, |
| "epoch": 0.6805970149253732, |
| "grad_norm": 0.35497111082077026, |
| "kl": 0.01766645722091198, |
| "learning_rate": 1.4035510025294463e-06, |
| "loss": 0.0007, |
| "reward": 0.140625, |
| "reward_std": 0.12387890368700027, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.140625, |
| "step": 228 |
| }, |
| { |
| "completion_length": 233.875, |
| "epoch": 0.6835820895522388, |
| "grad_norm": 0.30006691813468933, |
| "kl": 0.02256627008318901, |
| "learning_rate": 1.380161602985738e-06, |
| "loss": 0.0009, |
| "reward": 0.05274999886751175, |
| "reward_std": 0.35022878646850586, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.05274999886751175, |
| "step": 229 |
| }, |
| { |
| "completion_length": 427.625, |
| "epoch": 0.6865671641791045, |
| "grad_norm": 0.22094956040382385, |
| "kl": 0.014623177237808704, |
| "learning_rate": 1.3568941917384038e-06, |
| "loss": 0.0006, |
| "reward": -0.6439999938011169, |
| "reward_std": 1.0302884578704834, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": -0.6440000534057617, |
| "step": 230 |
| }, |
| { |
| "completion_length": 151.0, |
| "epoch": 0.6895522388059702, |
| "grad_norm": 0.669421374797821, |
| "kl": 0.046193912625312805, |
| "learning_rate": 1.3337513033958904e-06, |
| "loss": 0.0018, |
| "reward": 1.355125069618225, |
| "reward_std": 0.9819782376289368, |
| "rewards/correctness_reward_func": 1.25, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.10512500256299973, |
| "step": 231 |
| }, |
| { |
| "completion_length": 524.875, |
| "epoch": 0.6925373134328359, |
| "grad_norm": 0.1570989489555359, |
| "kl": 0.016627641394734383, |
| "learning_rate": 1.310735459001884e-06, |
| "loss": 0.0007, |
| "reward": 0.31312498450279236, |
| "reward_std": 1.1351596117019653, |
| "rewards/correctness_reward_func": 0.5, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": -0.18687501549720764, |
| "step": 232 |
| }, |
| { |
| "completion_length": 471.875, |
| "epoch": 0.6955223880597015, |
| "grad_norm": 0.32607302069664, |
| "kl": 0.0247778482735157, |
| "learning_rate": 1.2878491657606874e-06, |
| "loss": 0.001, |
| "reward": 0.187624990940094, |
| "reward_std": 0.1159592717885971, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.187624990940094, |
| "step": 233 |
| }, |
| { |
| "completion_length": 376.875, |
| "epoch": 0.6985074626865672, |
| "grad_norm": 0.5452307462692261, |
| "kl": 0.018254172056913376, |
| "learning_rate": 1.2650949167640997e-06, |
| "loss": 0.0007, |
| "reward": 0.171875, |
| "reward_std": 0.09300297498703003, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.171875, |
| "step": 234 |
| }, |
| { |
| "completion_length": 281.25, |
| "epoch": 0.7014925373134329, |
| "grad_norm": 0.276838093996048, |
| "kl": 0.0188542939722538, |
| "learning_rate": 1.2424751907198312e-06, |
| "loss": 0.0008, |
| "reward": 0.750124990940094, |
| "reward_std": 0.9281703233718872, |
| "rewards/correctness_reward_func": 0.5, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.250124990940094, |
| "step": 235 |
| }, |
| { |
| "completion_length": 550.625, |
| "epoch": 0.7044776119402985, |
| "grad_norm": 0.29794007539749146, |
| "kl": 0.015997443348169327, |
| "learning_rate": 1.219992451681494e-06, |
| "loss": 0.0006, |
| "reward": 0.171999990940094, |
| "reward_std": 0.09312357008457184, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.171999990940094, |
| "step": 236 |
| }, |
| { |
| "completion_length": 263.625, |
| "epoch": 0.7074626865671642, |
| "grad_norm": 0.4172457158565521, |
| "kl": 0.021728359162807465, |
| "learning_rate": 1.1976491487801747e-06, |
| "loss": 0.0009, |
| "reward": 0.140749990940094, |
| "reward_std": 0.1409870833158493, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.140749990940094, |
| "step": 237 |
| }, |
| { |
| "completion_length": 365.0, |
| "epoch": 0.7104477611940299, |
| "grad_norm": 0.2539860010147095, |
| "kl": 0.015010641887784004, |
| "learning_rate": 1.17544771595765e-06, |
| "loss": 0.0006, |
| "reward": 0.2433750182390213, |
| "reward_std": 0.9216052293777466, |
| "rewards/correctness_reward_func": 0.25, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": -0.006624996662139893, |
| "step": 238 |
| }, |
| { |
| "completion_length": 206.5, |
| "epoch": 0.7134328358208956, |
| "grad_norm": 0.33303573727607727, |
| "kl": 0.020331714302301407, |
| "learning_rate": 1.1533905717012425e-06, |
| "loss": 0.0008, |
| "reward": 0.07275000214576721, |
| "reward_std": 0.3491785228252411, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.07275000214576721, |
| "step": 239 |
| }, |
| { |
| "completion_length": 528.25, |
| "epoch": 0.7164179104477612, |
| "grad_norm": 0.380811870098114, |
| "kl": 0.012931251898407936, |
| "learning_rate": 1.1314801187803687e-06, |
| "loss": 0.0005, |
| "reward": 0.13875000178813934, |
| "reward_std": 0.23173367977142334, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.13874998688697815, |
| "step": 240 |
| }, |
| { |
| "completion_length": 333.125, |
| "epoch": 0.7194029850746269, |
| "grad_norm": 0.4135821759700775, |
| "kl": 0.01626397855579853, |
| "learning_rate": 1.109718743984794e-06, |
| "loss": 0.0007, |
| "reward": 0.4181250035762787, |
| "reward_std": 0.7555687427520752, |
| "rewards/correctness_reward_func": 0.25, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.1681250035762787, |
| "step": 241 |
| }, |
| { |
| "completion_length": 135.125, |
| "epoch": 0.7223880597014926, |
| "grad_norm": 0.41655805706977844, |
| "kl": 0.05684615299105644, |
| "learning_rate": 1.0881088178646291e-06, |
| "loss": 0.0023, |
| "reward": 0.4477500021457672, |
| "reward_std": 0.7376521229743958, |
| "rewards/correctness_reward_func": 0.25, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.1977500021457672, |
| "step": 242 |
| }, |
| { |
| "completion_length": 531.875, |
| "epoch": 0.7253731343283583, |
| "grad_norm": 0.47717010974884033, |
| "kl": 0.019140595570206642, |
| "learning_rate": 1.0666526944721017e-06, |
| "loss": 0.0008, |
| "reward": 0.1667499989271164, |
| "reward_std": 0.20734770596027374, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.1667499989271164, |
| "step": 243 |
| }, |
| { |
| "completion_length": 257.875, |
| "epoch": 0.7283582089552239, |
| "grad_norm": 0.2818719446659088, |
| "kl": 0.017366956919431686, |
| "learning_rate": 1.0453527111051183e-06, |
| "loss": 0.0007, |
| "reward": 0.700249969959259, |
| "reward_std": 1.00717031955719, |
| "rewards/correctness_reward_func": 0.5, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.20024999976158142, |
| "step": 244 |
| }, |
| { |
| "completion_length": 336.625, |
| "epoch": 0.7313432835820896, |
| "grad_norm": 0.2551906108856201, |
| "kl": 0.017408214509487152, |
| "learning_rate": 1.0242111880526495e-06, |
| "loss": 0.0007, |
| "reward": 0.5737500190734863, |
| "reward_std": 0.954799234867096, |
| "rewards/correctness_reward_func": 0.5, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.07374998927116394, |
| "step": 245 |
| }, |
| { |
| "completion_length": 246.75, |
| "epoch": 0.7343283582089553, |
| "grad_norm": 0.2634367048740387, |
| "kl": 0.020746061578392982, |
| "learning_rate": 1.0032304283419792e-06, |
| "loss": 0.0008, |
| "reward": 0.6588749885559082, |
| "reward_std": 0.8327701687812805, |
| "rewards/correctness_reward_func": 0.75, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": -0.0911249965429306, |
| "step": 246 |
| }, |
| { |
| "completion_length": 262.25, |
| "epoch": 0.7373134328358208, |
| "grad_norm": 0.2753911316394806, |
| "kl": 0.025336353108286858, |
| "learning_rate": 9.824127174878196e-07, |
| "loss": 0.001, |
| "reward": 0.10499998927116394, |
| "reward_std": 0.3267969787120819, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.10499998927116394, |
| "step": 247 |
| }, |
| { |
| "completion_length": 434.625, |
| "epoch": 0.7402985074626866, |
| "grad_norm": 0.2999410927295685, |
| "kl": 0.013439298607409, |
| "learning_rate": 9.617603232433475e-07, |
| "loss": 0.0005, |
| "reward": 0.890625, |
| "reward_std": 1.1268585920333862, |
| "rewards/correctness_reward_func": 0.75, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.140625, |
| "step": 248 |
| }, |
| { |
| "completion_length": 507.375, |
| "epoch": 0.7432835820895523, |
| "grad_norm": 0.2620807886123657, |
| "kl": 0.012813393957912922, |
| "learning_rate": 9.412754953531664e-07, |
| "loss": 0.0005, |
| "reward": 0.034749992191791534, |
| "reward_std": 0.5604296326637268, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.03474998474121094, |
| "step": 249 |
| }, |
| { |
| "completion_length": 249.125, |
| "epoch": 0.746268656716418, |
| "grad_norm": 0.5003867149353027, |
| "kl": 0.024488642811775208, |
| "learning_rate": 9.209604653082326e-07, |
| "loss": 0.001, |
| "reward": 0.7232499718666077, |
| "reward_std": 1.039300799369812, |
| "rewards/correctness_reward_func": 0.75, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": -0.026749998331069946, |
| "step": 250 |
| }, |
| { |
| "completion_length": 453.375, |
| "epoch": 0.7492537313432835, |
| "grad_norm": 0.2221699208021164, |
| "kl": 0.01704486459493637, |
| "learning_rate": 9.008174461027724e-07, |
| "loss": 0.0007, |
| "reward": -0.044124990701675415, |
| "reward_std": 0.879956841468811, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": -0.044124990701675415, |
| "step": 251 |
| }, |
| { |
| "completion_length": 199.25, |
| "epoch": 0.7522388059701492, |
| "grad_norm": 0.3644316792488098, |
| "kl": 0.036831244826316833, |
| "learning_rate": 8.808486319932083e-07, |
| "loss": 0.0015, |
| "reward": 0.07199999690055847, |
| "reward_std": 0.20690439641475677, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.07199999690055847, |
| "step": 252 |
| }, |
| { |
| "completion_length": 140.875, |
| "epoch": 0.755223880597015, |
| "grad_norm": 0.45422235131263733, |
| "kl": 0.040265657007694244, |
| "learning_rate": 8.610561982591356e-07, |
| "loss": 0.0016, |
| "reward": 0.703125, |
| "reward_std": 0.8811962008476257, |
| "rewards/correctness_reward_func": 0.5, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.203125, |
| "step": 253 |
| }, |
| { |
| "completion_length": 460.625, |
| "epoch": 0.7582089552238805, |
| "grad_norm": 0.22300752997398376, |
| "kl": 0.011045263148844242, |
| "learning_rate": 8.414423009663564e-07, |
| "loss": 0.0004, |
| "reward": -0.3152500092983246, |
| "reward_std": 0.8068054914474487, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": -0.3152500092983246, |
| "step": 254 |
| }, |
| { |
| "completion_length": 443.375, |
| "epoch": 0.7611940298507462, |
| "grad_norm": 0.4048740863800049, |
| "kl": 0.020228559151291847, |
| "learning_rate": 8.220090767320138e-07, |
| "loss": 0.0008, |
| "reward": 0.234624981880188, |
| "reward_std": 0.08016578108072281, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.23462499678134918, |
| "step": 255 |
| }, |
| { |
| "completion_length": 372.75, |
| "epoch": 0.764179104477612, |
| "grad_norm": 0.2398412674665451, |
| "kl": 0.014233190566301346, |
| "learning_rate": 8.027586424918413e-07, |
| "loss": 0.0006, |
| "reward": 0.03700000047683716, |
| "reward_std": 0.41604670882225037, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.03700000047683716, |
| "step": 256 |
| }, |
| { |
| "completion_length": 321.125, |
| "epoch": 0.7671641791044777, |
| "grad_norm": 0.2411520928144455, |
| "kl": 0.020644700154662132, |
| "learning_rate": 7.836930952695535e-07, |
| "loss": 0.0008, |
| "reward": 0.12312500178813934, |
| "reward_std": 0.2639526128768921, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.12312500178813934, |
| "step": 257 |
| }, |
| { |
| "completion_length": 130.5, |
| "epoch": 0.7701492537313432, |
| "grad_norm": 0.49876320362091064, |
| "kl": 0.035634126514196396, |
| "learning_rate": 7.648145119484152e-07, |
| "loss": 0.0014, |
| "reward": 0.9228750467300415, |
| "reward_std": 1.0723521709442139, |
| "rewards/correctness_reward_func": 0.75, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.17287498712539673, |
| "step": 258 |
| }, |
| { |
| "completion_length": 359.375, |
| "epoch": 0.7731343283582089, |
| "grad_norm": 0.22742308676242828, |
| "kl": 0.018567712977528572, |
| "learning_rate": 7.461249490449954e-07, |
| "loss": 0.0007, |
| "reward": 0.33787500858306885, |
| "reward_std": 0.8415562510490417, |
| "rewards/correctness_reward_func": 0.25, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.08787500113248825, |
| "step": 259 |
| }, |
| { |
| "completion_length": 508.75, |
| "epoch": 0.7761194029850746, |
| "grad_norm": 0.19451585412025452, |
| "kl": 0.016084793955087662, |
| "learning_rate": 7.276264424851426e-07, |
| "loss": 0.0006, |
| "reward": 0.1028750017285347, |
| "reward_std": 0.32013678550720215, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.1028750017285347, |
| "step": 260 |
| }, |
| { |
| "completion_length": 326.75, |
| "epoch": 0.7791044776119403, |
| "grad_norm": 0.27631670236587524, |
| "kl": 0.018710516393184662, |
| "learning_rate": 7.093210073822027e-07, |
| "loss": 0.0007, |
| "reward": -0.4192500114440918, |
| "reward_std": 0.6906015872955322, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": -0.4192500114440918, |
| "step": 261 |
| }, |
| { |
| "completion_length": 413.625, |
| "epoch": 0.7820895522388059, |
| "grad_norm": 0.2365199774503708, |
| "kl": 0.01709006167948246, |
| "learning_rate": 6.912106378175098e-07, |
| "loss": 0.0007, |
| "reward": 0.234375, |
| "reward_std": 0.04419417306780815, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.234375, |
| "step": 262 |
| }, |
| { |
| "completion_length": 264.5, |
| "epoch": 0.7850746268656716, |
| "grad_norm": 0.3793802261352539, |
| "kl": 0.02793733775615692, |
| "learning_rate": 6.732973066231563e-07, |
| "loss": 0.0011, |
| "reward": 0.6676249504089355, |
| "reward_std": 0.9522945284843445, |
| "rewards/correctness_reward_func": 0.5, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.16762499511241913, |
| "step": 263 |
| }, |
| { |
| "completion_length": 505.625, |
| "epoch": 0.7880597014925373, |
| "grad_norm": 0.2976485788822174, |
| "kl": 0.010713014751672745, |
| "learning_rate": 6.555829651670912e-07, |
| "loss": 0.0004, |
| "reward": 0.08262500166893005, |
| "reward_std": 0.2939401865005493, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.08262500166893005, |
| "step": 264 |
| }, |
| { |
| "completion_length": 249.25, |
| "epoch": 0.7910447761194029, |
| "grad_norm": 0.27218976616859436, |
| "kl": 0.029523877426981926, |
| "learning_rate": 6.380695431405453e-07, |
| "loss": 0.0012, |
| "reward": 0.05262499302625656, |
| "reward_std": 0.3924490809440613, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.05262500047683716, |
| "step": 265 |
| }, |
| { |
| "completion_length": 333.25, |
| "epoch": 0.7940298507462686, |
| "grad_norm": 0.25616171956062317, |
| "kl": 0.022499781101942062, |
| "learning_rate": 6.207589483478266e-07, |
| "loss": 0.0009, |
| "reward": 0.5889999866485596, |
| "reward_std": 0.9673720002174377, |
| "rewards/correctness_reward_func": 0.5, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.08900000154972076, |
| "step": 266 |
| }, |
| { |
| "completion_length": 257.625, |
| "epoch": 0.7970149253731343, |
| "grad_norm": 0.32119879126548767, |
| "kl": 0.028798656538128853, |
| "learning_rate": 6.036530664984922e-07, |
| "loss": 0.0012, |
| "reward": 0.7595000267028809, |
| "reward_std": 1.2280067205429077, |
| "rewards/correctness_reward_func": 0.75, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.00950000062584877, |
| "step": 267 |
| }, |
| { |
| "completion_length": 330.125, |
| "epoch": 0.8, |
| "grad_norm": 0.4567493498325348, |
| "kl": 0.03090793453156948, |
| "learning_rate": 5.867537610019317e-07, |
| "loss": 0.0012, |
| "reward": 0.06974999606609344, |
| "reward_std": 0.1446550041437149, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.06975000351667404, |
| "step": 268 |
| }, |
| { |
| "completion_length": 483.25, |
| "epoch": 0.8029850746268656, |
| "grad_norm": 0.1947696954011917, |
| "kl": 0.014657390303909779, |
| "learning_rate": 5.700628727643806e-07, |
| "loss": 0.0006, |
| "reward": 0.2797499895095825, |
| "reward_std": 0.4771743416786194, |
| "rewards/correctness_reward_func": 0.25, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.029750000685453415, |
| "step": 269 |
| }, |
| { |
| "completion_length": 390.5, |
| "epoch": 0.8059701492537313, |
| "grad_norm": 0.28339794278144836, |
| "kl": 0.01822579652070999, |
| "learning_rate": 5.53582219988382e-07, |
| "loss": 0.0007, |
| "reward": 0.0495000034570694, |
| "reward_std": 0.2241402566432953, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0494999997317791, |
| "step": 270 |
| }, |
| { |
| "completion_length": 612.375, |
| "epoch": 0.808955223880597, |
| "grad_norm": 0.19297142326831818, |
| "kl": 0.008483514189720154, |
| "learning_rate": 5.373135979747226e-07, |
| "loss": 0.0003, |
| "reward": 0.234375, |
| "reward_std": 0.04419417306780815, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.234375, |
| "step": 271 |
| }, |
| { |
| "completion_length": 442.25, |
| "epoch": 0.8119402985074626, |
| "grad_norm": 0.18323534727096558, |
| "kl": 0.015557816252112389, |
| "learning_rate": 5.21258778926865e-07, |
| "loss": 0.0006, |
| "reward": 0.4873749911785126, |
| "reward_std": 0.7220054864883423, |
| "rewards/correctness_reward_func": 0.25, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.23737499117851257, |
| "step": 272 |
| }, |
| { |
| "completion_length": 420.375, |
| "epoch": 0.8149253731343283, |
| "grad_norm": 0.29259246587753296, |
| "kl": 0.014818736352026463, |
| "learning_rate": 5.054195117578914e-07, |
| "loss": 0.0006, |
| "reward": 0.10899998992681503, |
| "reward_std": 0.30351370573043823, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.10899999737739563, |
| "step": 273 |
| }, |
| { |
| "completion_length": 144.875, |
| "epoch": 0.817910447761194, |
| "grad_norm": 0.4706750810146332, |
| "kl": 0.04596470296382904, |
| "learning_rate": 4.897975218999926e-07, |
| "loss": 0.0018, |
| "reward": 0.7043750286102295, |
| "reward_std": 0.9201574325561523, |
| "rewards/correctness_reward_func": 0.5, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.2043750137090683, |
| "step": 274 |
| }, |
| { |
| "completion_length": 296.75, |
| "epoch": 0.8208955223880597, |
| "grad_norm": 0.24620626866817474, |
| "kl": 0.02245059236884117, |
| "learning_rate": 4.7439451111650685e-07, |
| "loss": 0.0009, |
| "reward": 0.40625, |
| "reward_std": 0.752228856086731, |
| "rewards/correctness_reward_func": 0.25, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.15625, |
| "step": 275 |
| }, |
| { |
| "completion_length": 202.875, |
| "epoch": 0.8238805970149253, |
| "grad_norm": 0.4145122766494751, |
| "kl": 0.024612464010715485, |
| "learning_rate": 4.5921215731654144e-07, |
| "loss": 0.001, |
| "reward": 0.12150000035762787, |
| "reward_std": 0.18457983434200287, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.12150000035762787, |
| "step": 276 |
| }, |
| { |
| "completion_length": 325.25, |
| "epoch": 0.826865671641791, |
| "grad_norm": 0.4407789409160614, |
| "kl": 0.03133422136306763, |
| "learning_rate": 4.4425211437218926e-07, |
| "loss": 0.0013, |
| "reward": 0.20337499678134918, |
| "reward_std": 0.09237955510616302, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.20337499678134918, |
| "step": 277 |
| }, |
| { |
| "completion_length": 278.375, |
| "epoch": 0.8298507462686567, |
| "grad_norm": 0.39592450857162476, |
| "kl": 0.024857958778738976, |
| "learning_rate": 4.2951601193837124e-07, |
| "loss": 0.001, |
| "reward": 0.43712499737739563, |
| "reward_std": 1.1832053661346436, |
| "rewards/correctness_reward_func": 0.5, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": -0.06287499517202377, |
| "step": 278 |
| }, |
| { |
| "completion_length": 557.625, |
| "epoch": 0.8328358208955224, |
| "grad_norm": 0.21943753957748413, |
| "kl": 0.008740151301026344, |
| "learning_rate": 4.150054552753055e-07, |
| "loss": 0.0003, |
| "reward": -0.22687500715255737, |
| "reward_std": 0.8986358642578125, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": -0.22687500715255737, |
| "step": 279 |
| }, |
| { |
| "completion_length": 147.125, |
| "epoch": 0.835820895522388, |
| "grad_norm": 0.452767550945282, |
| "kl": 0.03773313760757446, |
| "learning_rate": 4.0072202507364543e-07, |
| "loss": 0.0015, |
| "reward": 1.2141249179840088, |
| "reward_std": 1.0455961227416992, |
| "rewards/correctness_reward_func": 1.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.21412500739097595, |
| "step": 280 |
| }, |
| { |
| "completion_length": 446.25, |
| "epoch": 0.8388059701492537, |
| "grad_norm": 0.19403225183486938, |
| "kl": 0.01679888553917408, |
| "learning_rate": 3.866672772822863e-07, |
| "loss": 0.0007, |
| "reward": 0.056999996304512024, |
| "reward_std": 0.3133218586444855, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.056999996304512024, |
| "step": 281 |
| }, |
| { |
| "completion_length": 200.25, |
| "epoch": 0.8417910447761194, |
| "grad_norm": 0.60846346616745, |
| "kl": 0.029500508680939674, |
| "learning_rate": 3.728427429388709e-07, |
| "loss": 0.0012, |
| "reward": 0.7407500147819519, |
| "reward_std": 0.9356353878974915, |
| "rewards/correctness_reward_func": 0.5, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.2407499998807907, |
| "step": 282 |
| }, |
| { |
| "completion_length": 339.125, |
| "epoch": 0.844776119402985, |
| "grad_norm": 0.25175222754478455, |
| "kl": 0.02122483216226101, |
| "learning_rate": 3.592499280030057e-07, |
| "loss": 0.0008, |
| "reward": 0.07399998605251312, |
| "reward_std": 0.5027032494544983, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.07399999350309372, |
| "step": 283 |
| }, |
| { |
| "completion_length": 275.625, |
| "epoch": 0.8477611940298507, |
| "grad_norm": 0.8914472460746765, |
| "kl": 0.029980091378092766, |
| "learning_rate": 3.458903131922134e-07, |
| "loss": 0.0012, |
| "reward": 0.2407499998807907, |
| "reward_std": 0.11513439565896988, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.2407499998807907, |
| "step": 284 |
| }, |
| { |
| "completion_length": 620.875, |
| "epoch": 0.8507462686567164, |
| "grad_norm": 0.16470280289649963, |
| "kl": 0.006700257305055857, |
| "learning_rate": 3.3276535382063184e-07, |
| "loss": 0.0003, |
| "reward": 0.06174999475479126, |
| "reward_std": 0.33658909797668457, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.06174999475479126, |
| "step": 285 |
| }, |
| { |
| "completion_length": 506.25, |
| "epoch": 0.8537313432835821, |
| "grad_norm": 0.26415377855300903, |
| "kl": 0.01859319768846035, |
| "learning_rate": 3.1987647964048075e-07, |
| "loss": 0.0007, |
| "reward": 0.06987500190734863, |
| "reward_std": 0.2795202136039734, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.06987500190734863, |
| "step": 286 |
| }, |
| { |
| "completion_length": 318.625, |
| "epoch": 0.8567164179104477, |
| "grad_norm": 0.41256532073020935, |
| "kl": 0.016763173043727875, |
| "learning_rate": 3.07225094686314e-07, |
| "loss": 0.0007, |
| "reward": 0.5, |
| "reward_std": 0.7071067690849304, |
| "rewards/correctness_reward_func": 0.25, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.25, |
| "step": 287 |
| }, |
| { |
| "completion_length": 248.25, |
| "epoch": 0.8597014925373134, |
| "grad_norm": 0.3886096775531769, |
| "kl": 0.0391070619225502, |
| "learning_rate": 2.9481257712206974e-07, |
| "loss": 0.0016, |
| "reward": 1.424375057220459, |
| "reward_std": 1.079121470451355, |
| "rewards/correctness_reward_func": 1.25, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.1743749976158142, |
| "step": 288 |
| }, |
| { |
| "completion_length": 198.875, |
| "epoch": 0.8626865671641791, |
| "grad_norm": 0.3038368821144104, |
| "kl": 0.028413468971848488, |
| "learning_rate": 2.8264027909094715e-07, |
| "loss": 0.0011, |
| "reward": 0.04787500202655792, |
| "reward_std": 0.277681440114975, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.04787500202655792, |
| "step": 289 |
| }, |
| { |
| "completion_length": 241.375, |
| "epoch": 0.8656716417910447, |
| "grad_norm": 0.6994856595993042, |
| "kl": 0.025603370741009712, |
| "learning_rate": 2.707095265681081e-07, |
| "loss": 0.001, |
| "reward": 0.36537498235702515, |
| "reward_std": 1.275699257850647, |
| "rewards/correctness_reward_func": 0.5, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": -0.13462501764297485, |
| "step": 290 |
| }, |
| { |
| "completion_length": 269.75, |
| "epoch": 0.8686567164179104, |
| "grad_norm": 0.5225069522857666, |
| "kl": 0.021305205300450325, |
| "learning_rate": 2.5902161921623454e-07, |
| "loss": 0.0009, |
| "reward": 0.24774998426437378, |
| "reward_std": 0.13129328191280365, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.24774998426437378, |
| "step": 291 |
| }, |
| { |
| "completion_length": 301.875, |
| "epoch": 0.8716417910447761, |
| "grad_norm": 0.3499370813369751, |
| "kl": 0.019313901662826538, |
| "learning_rate": 2.4757783024395244e-07, |
| "loss": 0.0008, |
| "reward": 0.5285000205039978, |
| "reward_std": 0.8365910053253174, |
| "rewards/correctness_reward_func": 0.5, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.02850000374019146, |
| "step": 292 |
| }, |
| { |
| "completion_length": 382.125, |
| "epoch": 0.8746268656716418, |
| "grad_norm": 0.3204389214515686, |
| "kl": 0.01630595698952675, |
| "learning_rate": 2.3637940626713346e-07, |
| "loss": 0.0007, |
| "reward": 0.8981249928474426, |
| "reward_std": 1.1263054609298706, |
| "rewards/correctness_reward_func": 0.75, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.14812499284744263, |
| "step": 293 |
| }, |
| { |
| "completion_length": 364.625, |
| "epoch": 0.8776119402985074, |
| "grad_norm": 0.43380382657051086, |
| "kl": 0.014025033451616764, |
| "learning_rate": 2.254275671731007e-07, |
| "loss": 0.0006, |
| "reward": 0.13312500715255737, |
| "reward_std": 0.20087197422981262, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.13312500715255737, |
| "step": 294 |
| }, |
| { |
| "completion_length": 362.125, |
| "epoch": 0.8805970149253731, |
| "grad_norm": 0.25492313504219055, |
| "kl": 0.02461504563689232, |
| "learning_rate": 2.14723505987737e-07, |
| "loss": 0.001, |
| "reward": 1.11899995803833, |
| "reward_std": 1.0727145671844482, |
| "rewards/correctness_reward_func": 1.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.11900000274181366, |
| "step": 295 |
| }, |
| { |
| "completion_length": 239.625, |
| "epoch": 0.8835820895522388, |
| "grad_norm": 0.2532467842102051, |
| "kl": 0.019054116681218147, |
| "learning_rate": 2.0426838874552713e-07, |
| "loss": 0.0008, |
| "reward": -0.02237500250339508, |
| "reward_std": 0.34399375319480896, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": -0.022374995052814484, |
| "step": 296 |
| }, |
| { |
| "completion_length": 406.125, |
| "epoch": 0.8865671641791045, |
| "grad_norm": 0.3017323613166809, |
| "kl": 0.014569297432899475, |
| "learning_rate": 1.9406335436253727e-07, |
| "loss": 0.0006, |
| "reward": 0.1875, |
| "reward_std": 0.09449111670255661, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.1875, |
| "step": 297 |
| }, |
| { |
| "completion_length": 419.625, |
| "epoch": 0.8895522388059701, |
| "grad_norm": 0.27221450209617615, |
| "kl": 0.017288224771618843, |
| "learning_rate": 1.8410951451234533e-07, |
| "loss": 0.0007, |
| "reward": 0.06112499535083771, |
| "reward_std": 0.48568522930145264, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.06112499535083771, |
| "step": 298 |
| }, |
| { |
| "completion_length": 499.625, |
| "epoch": 0.8925373134328358, |
| "grad_norm": 0.2659822404384613, |
| "kl": 0.013604966923594475, |
| "learning_rate": 1.7440795350494588e-07, |
| "loss": 0.0005, |
| "reward": 0.45787498354911804, |
| "reward_std": 0.7346572875976562, |
| "rewards/correctness_reward_func": 0.25, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.20787498354911804, |
| "step": 299 |
| }, |
| { |
| "completion_length": 575.25, |
| "epoch": 0.8955223880597015, |
| "grad_norm": 0.40231838822364807, |
| "kl": 0.014159854501485825, |
| "learning_rate": 1.649597281686302e-07, |
| "loss": 0.0006, |
| "reward": 0.875, |
| "reward_std": 1.1417405605316162, |
| "rewards/correctness_reward_func": 0.75, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.125, |
| "step": 300 |
| }, |
| { |
| "completion_length": 499.0, |
| "epoch": 0.8985074626865671, |
| "grad_norm": 0.21424373984336853, |
| "kl": 0.032591626048088074, |
| "learning_rate": 1.5576586773486198e-07, |
| "loss": 0.0013, |
| "reward": -0.4646250009536743, |
| "reward_std": 0.961997926235199, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": -0.4646250009536743, |
| "step": 301 |
| }, |
| { |
| "completion_length": 374.375, |
| "epoch": 0.9014925373134328, |
| "grad_norm": 0.24450530111789703, |
| "kl": 0.012314786203205585, |
| "learning_rate": 1.4682737372615968e-07, |
| "loss": 0.0005, |
| "reward": -0.10649999976158142, |
| "reward_std": 0.7154986262321472, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": -0.10650000721216202, |
| "step": 302 |
| }, |
| { |
| "completion_length": 450.25, |
| "epoch": 0.9044776119402985, |
| "grad_norm": 0.22723200917243958, |
| "kl": 0.015731429681181908, |
| "learning_rate": 1.3814521984699597e-07, |
| "loss": 0.0006, |
| "reward": 0.28125, |
| "reward_std": 0.0883931964635849, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.28125, |
| "step": 303 |
| }, |
| { |
| "completion_length": 432.0, |
| "epoch": 0.9074626865671642, |
| "grad_norm": 0.22048191726207733, |
| "kl": 0.01865359954535961, |
| "learning_rate": 1.297203518777293e-07, |
| "loss": 0.0007, |
| "reward": 0.924375057220459, |
| "reward_std": 1.1127361059188843, |
| "rewards/correctness_reward_func": 0.75, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.1743749976158142, |
| "step": 304 |
| }, |
| { |
| "completion_length": 216.75, |
| "epoch": 0.9104477611940298, |
| "grad_norm": 0.37429243326187134, |
| "kl": 0.023615093901753426, |
| "learning_rate": 1.2155368757157644e-07, |
| "loss": 0.0009, |
| "reward": 0.21875, |
| "reward_std": 0.0578637570142746, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.21875, |
| "step": 305 |
| }, |
| { |
| "completion_length": 363.0, |
| "epoch": 0.9134328358208955, |
| "grad_norm": 0.25917860865592957, |
| "kl": 0.016996635124087334, |
| "learning_rate": 1.1364611655463737e-07, |
| "loss": 0.0007, |
| "reward": 0.2605000138282776, |
| "reward_std": 0.029698481783270836, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.2605000138282776, |
| "step": 306 |
| }, |
| { |
| "completion_length": 429.875, |
| "epoch": 0.9164179104477612, |
| "grad_norm": 0.32845067977905273, |
| "kl": 0.006652818527072668, |
| "learning_rate": 1.0599850022898539e-07, |
| "loss": 0.0003, |
| "reward": 0.13762500882148743, |
| "reward_std": 0.2873464524745941, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.13762499392032623, |
| "step": 307 |
| }, |
| { |
| "completion_length": 425.875, |
| "epoch": 0.9194029850746268, |
| "grad_norm": 0.45313775539398193, |
| "kl": 0.022167667746543884, |
| "learning_rate": 9.861167167883046e-08, |
| "loss": 0.0009, |
| "reward": 0.26487499475479126, |
| "reward_std": 0.042072843760252, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.26487499475479126, |
| "step": 308 |
| }, |
| { |
| "completion_length": 801.125, |
| "epoch": 0.9223880597014925, |
| "grad_norm": 0.16913585364818573, |
| "kl": 0.004942856729030609, |
| "learning_rate": 9.148643557976955e-08, |
| "loss": 0.0002, |
| "reward": -0.06262499839067459, |
| "reward_std": 0.6919310092926025, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": -0.062624990940094, |
| "step": 309 |
| }, |
| { |
| "completion_length": 368.875, |
| "epoch": 0.9253731343283582, |
| "grad_norm": 0.5053035020828247, |
| "kl": 0.026447134092450142, |
| "learning_rate": 8.462356811112987e-08, |
| "loss": 0.0011, |
| "reward": 0.9963749647140503, |
| "reward_std": 1.0054038763046265, |
| "rewards/correctness_reward_func": 0.75, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.2463749796152115, |
| "step": 310 |
| }, |
| { |
| "completion_length": 188.875, |
| "epoch": 0.9283582089552239, |
| "grad_norm": 0.3338444232940674, |
| "kl": 0.027413196861743927, |
| "learning_rate": 7.802381687141537e-08, |
| "loss": 0.0011, |
| "reward": 0.19200000166893005, |
| "reward_std": 0.18996766209602356, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.19200000166893005, |
| "step": 311 |
| }, |
| { |
| "completion_length": 450.875, |
| "epoch": 0.9313432835820895, |
| "grad_norm": 0.27691736817359924, |
| "kl": 0.016083208844065666, |
| "learning_rate": 7.168790079686932e-08, |
| "loss": 0.0006, |
| "reward": 0.234375, |
| "reward_std": 0.04419417306780815, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.234375, |
| "step": 312 |
| }, |
| { |
| "completion_length": 151.375, |
| "epoch": 0.9343283582089552, |
| "grad_norm": 0.48099038004875183, |
| "kl": 0.039979659020900726, |
| "learning_rate": 6.561651008315739e-08, |
| "loss": 0.0016, |
| "reward": 1.2289999723434448, |
| "reward_std": 1.0798243284225464, |
| "rewards/correctness_reward_func": 1.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.2290000021457672, |
| "step": 313 |
| }, |
| { |
| "completion_length": 550.25, |
| "epoch": 0.9373134328358209, |
| "grad_norm": 0.23031727969646454, |
| "kl": 0.008523189462721348, |
| "learning_rate": 5.981030611018235e-08, |
| "loss": 0.0003, |
| "reward": -0.05887500196695328, |
| "reward_std": 0.525315523147583, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": -0.05887499451637268, |
| "step": 314 |
| }, |
| { |
| "completion_length": 265.125, |
| "epoch": 0.9402985074626866, |
| "grad_norm": 0.395933598279953, |
| "kl": 0.024086318910121918, |
| "learning_rate": 5.426992137003623e-08, |
| "loss": 0.001, |
| "reward": 0.44862499833106995, |
| "reward_std": 0.7450435757637024, |
| "rewards/correctness_reward_func": 0.25, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.19862499833106995, |
| "step": 315 |
| }, |
| { |
| "completion_length": 305.75, |
| "epoch": 0.9432835820895522, |
| "grad_norm": 0.2765781283378601, |
| "kl": 0.02224023826420307, |
| "learning_rate": 4.899595939810237e-08, |
| "loss": 0.0009, |
| "reward": 0.22337499260902405, |
| "reward_std": 0.06204591691493988, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.22337499260902405, |
| "step": 316 |
| }, |
| { |
| "completion_length": 154.875, |
| "epoch": 0.9462686567164179, |
| "grad_norm": 0.6462637782096863, |
| "kl": 0.04086542874574661, |
| "learning_rate": 4.3988994707308274e-08, |
| "loss": 0.0016, |
| "reward": 1.187749981880188, |
| "reward_std": 1.1077649593353271, |
| "rewards/correctness_reward_func": 1.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.18774999678134918, |
| "step": 317 |
| }, |
| { |
| "completion_length": 366.25, |
| "epoch": 0.9492537313432836, |
| "grad_norm": 0.315337598323822, |
| "kl": 0.01272547710686922, |
| "learning_rate": 3.92495727255432e-08, |
| "loss": 0.0005, |
| "reward": 0.171875, |
| "reward_std": 0.0646936446428299, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.171875, |
| "step": 318 |
| }, |
| { |
| "completion_length": 252.375, |
| "epoch": 0.9522388059701492, |
| "grad_norm": 0.3268197774887085, |
| "kl": 0.012484841048717499, |
| "learning_rate": 3.4778209736240633e-08, |
| "loss": 0.0005, |
| "reward": 0.1706250011920929, |
| "reward_std": 0.17941246926784515, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.1706250011920929, |
| "step": 319 |
| }, |
| { |
| "completion_length": 782.25, |
| "epoch": 0.9552238805970149, |
| "grad_norm": 0.17558734118938446, |
| "kl": 0.004195680376142263, |
| "learning_rate": 3.057539282213973e-08, |
| "loss": 0.0002, |
| "reward": 0.203249990940094, |
| "reward_std": 0.0932672768831253, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.203249990940094, |
| "step": 320 |
| }, |
| { |
| "completion_length": 441.75, |
| "epoch": 0.9582089552238806, |
| "grad_norm": 0.21393883228302002, |
| "kl": 0.01248833630234003, |
| "learning_rate": 2.6641579812224373e-08, |
| "loss": 0.0005, |
| "reward": 0.07662499696016312, |
| "reward_std": 0.29490867257118225, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.07662499696016312, |
| "step": 321 |
| }, |
| { |
| "completion_length": 269.625, |
| "epoch": 0.9611940298507463, |
| "grad_norm": 0.3684273064136505, |
| "kl": 0.034730613231658936, |
| "learning_rate": 2.2977199231850323e-08, |
| "loss": 0.0014, |
| "reward": 0.43137499690055847, |
| "reward_std": 0.7381420731544495, |
| "rewards/correctness_reward_func": 0.25, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.18137499690055847, |
| "step": 322 |
| }, |
| { |
| "completion_length": 272.75, |
| "epoch": 0.9641791044776119, |
| "grad_norm": 0.4992087185382843, |
| "kl": 0.021383756771683693, |
| "learning_rate": 1.9582650256064206e-08, |
| "loss": 0.0009, |
| "reward": 0.20925000309944153, |
| "reward_std": 0.13599659502506256, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.20924998819828033, |
| "step": 323 |
| }, |
| { |
| "completion_length": 278.0, |
| "epoch": 0.9671641791044776, |
| "grad_norm": 0.2685709595680237, |
| "kl": 0.021522654220461845, |
| "learning_rate": 1.6458302666119142e-08, |
| "loss": 0.0009, |
| "reward": 0.4894999861717224, |
| "reward_std": 0.7203638553619385, |
| "rewards/correctness_reward_func": 0.25, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.2395000010728836, |
| "step": 324 |
| }, |
| { |
| "completion_length": 325.375, |
| "epoch": 0.9701492537313433, |
| "grad_norm": 0.44149407744407654, |
| "kl": 0.012358118779957294, |
| "learning_rate": 1.3604496809195289e-08, |
| "loss": 0.0005, |
| "reward": 0.13312500715255737, |
| "reward_std": 0.23681482672691345, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.13312500715255737, |
| "step": 325 |
| }, |
| { |
| "completion_length": 322.25, |
| "epoch": 0.9731343283582089, |
| "grad_norm": 0.2590884268283844, |
| "kl": 0.010153868235647678, |
| "learning_rate": 1.1021543561322012e-08, |
| "loss": 0.0004, |
| "reward": 0.06912499666213989, |
| "reward_std": 0.4021095931529999, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.06912501156330109, |
| "step": 326 |
| }, |
| { |
| "completion_length": 300.875, |
| "epoch": 0.9761194029850746, |
| "grad_norm": 0.259194016456604, |
| "kl": 0.035336654633283615, |
| "learning_rate": 8.709724293513855e-09, |
| "loss": 0.0014, |
| "reward": 0.6192499995231628, |
| "reward_std": 0.7979370951652527, |
| "rewards/correctness_reward_func": 0.5, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.11924999952316284, |
| "step": 327 |
| }, |
| { |
| "completion_length": 224.5, |
| "epoch": 0.9791044776119403, |
| "grad_norm": 0.3342638909816742, |
| "kl": 0.046725135296583176, |
| "learning_rate": 6.66929084112089e-09, |
| "loss": 0.0019, |
| "reward": 0.41975000500679016, |
| "reward_std": 0.7516071796417236, |
| "rewards/correctness_reward_func": 0.25, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.16975000500679016, |
| "step": 328 |
| }, |
| { |
| "completion_length": 346.0, |
| "epoch": 0.982089552238806, |
| "grad_norm": 0.31775063276290894, |
| "kl": 0.03616100549697876, |
| "learning_rate": 4.900465476393168e-09, |
| "loss": 0.0014, |
| "reward": 0.6196249723434448, |
| "reward_std": 0.8508279323577881, |
| "rewards/correctness_reward_func": 0.5, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.11962500214576721, |
| "step": 329 |
| }, |
| { |
| "completion_length": 183.25, |
| "epoch": 0.9850746268656716, |
| "grad_norm": 0.5430064797401428, |
| "kl": 0.029290281236171722, |
| "learning_rate": 3.4034408842695264e-09, |
| "loss": 0.0012, |
| "reward": 1.1321250200271606, |
| "reward_std": 0.9209251403808594, |
| "rewards/correctness_reward_func": 1.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.13212500512599945, |
| "step": 330 |
| }, |
| { |
| "completion_length": 308.625, |
| "epoch": 0.9880597014925373, |
| "grad_norm": 0.36556127667427063, |
| "kl": 0.021233482286334038, |
| "learning_rate": 2.1783801413866044e-09, |
| "loss": 0.0008, |
| "reward": 0.12475001811981201, |
| "reward_std": 0.9743397235870361, |
| "rewards/correctness_reward_func": 0.25, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": -0.12524999678134918, |
| "step": 331 |
| }, |
| { |
| "completion_length": 334.125, |
| "epoch": 0.991044776119403, |
| "grad_norm": 0.44806885719299316, |
| "kl": 0.03072451800107956, |
| "learning_rate": 1.2254166983152737e-09, |
| "loss": 0.0012, |
| "reward": 0.16687500476837158, |
| "reward_std": 0.08920031785964966, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.16687500476837158, |
| "step": 332 |
| }, |
| { |
| "completion_length": 501.25, |
| "epoch": 0.9940298507462687, |
| "grad_norm": 0.18010862171649933, |
| "kl": 0.009232274256646633, |
| "learning_rate": 5.446543650219905e-10, |
| "loss": 0.0004, |
| "reward": -0.12587499618530273, |
| "reward_std": 0.5240494608879089, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": -0.12587499618530273, |
| "step": 333 |
| }, |
| { |
| "completion_length": 265.25, |
| "epoch": 0.9970149253731343, |
| "grad_norm": 0.42972275614738464, |
| "kl": 0.01687278039753437, |
| "learning_rate": 1.3616729956228425e-10, |
| "loss": 0.0007, |
| "reward": 0.21875, |
| "reward_std": 0.0578637570142746, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.21875, |
| "step": 334 |
| }, |
| { |
| "completion_length": 280.625, |
| "epoch": 1.0, |
| "grad_norm": 0.3359426259994507, |
| "kl": 0.017340093851089478, |
| "learning_rate": 0.0, |
| "loss": 0.0007, |
| "reward": 0.43549999594688416, |
| "reward_std": 0.7575456500053406, |
| "rewards/correctness_reward_func": 0.25, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.18549999594688416, |
| "step": 335 |
| } |
| ], |
| "logging_steps": 1, |
| "max_steps": 335, |
| "num_input_tokens_seen": 0, |
| "num_train_epochs": 1, |
| "save_steps": 100, |
| "stateful_callbacks": { |
| "TrainerControl": { |
| "args": { |
| "should_epoch_stop": false, |
| "should_evaluate": false, |
| "should_log": false, |
| "should_save": true, |
| "should_training_stop": true |
| }, |
| "attributes": {} |
| } |
| }, |
| "total_flos": 0.0, |
| "train_batch_size": 8, |
| "trial_name": null, |
| "trial_params": null |
| } |
|
|
