| { |
| "best_global_step": null, |
| "best_metric": null, |
| "best_model_checkpoint": null, |
| "epoch": 0.0002, |
| "eval_steps": 500, |
| "global_step": 10, |
| "is_hyper_param_search": false, |
| "is_local_process_zero": true, |
| "is_world_process_zero": true, |
| "log_history": [ |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.90625, |
| "completions/max_length": 16.0, |
| "completions/max_terminated_length": 16.0, |
| "completions/mean_length": 15.59375, |
| "completions/mean_terminated_length": 11.666666984558105, |
| "completions/min_length": 9.0, |
| "completions/min_terminated_length": 9.0, |
| "entropy": 0.45986470580101013, |
| "epoch": 2e-05, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.5555855631828308, |
| "kl": 0.0, |
| "learning_rate": 0.0, |
| "loss": -0.0036, |
| "num_tokens": 53302.0, |
| "reward": -20.565004348754883, |
| "reward_std": 12.108231544494629, |
| "rewards/rollout_reward_func/mean": -20.565004348754883, |
| "rewards/rollout_reward_func/std": 12.108230590820312, |
| "sampling/importance_sampling_ratio/max": 1.2069631814956665, |
| "sampling/importance_sampling_ratio/mean": 1.0164316892623901, |
| "sampling/importance_sampling_ratio/min": 0.691445529460907, |
| "sampling/sampling_logp_difference/max": 0.14298677444458008, |
| "sampling/sampling_logp_difference/mean": 0.01139700785279274, |
| "step": 1, |
| "step_time": 13.593145795000055 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "entropy": 0.45986470580101013, |
| "epoch": 4e-05, |
| "grad_norm": 0.5478876829147339, |
| "kl": 0.0, |
| "learning_rate": 8.571428571428574e-08, |
| "loss": -0.0036, |
| "step": 2, |
| "step_time": 3.5665654679996805 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0031250000465661287, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0031250000465661287, |
| "completions/clipped_ratio": 0.75, |
| "completions/max_length": 16.0, |
| "completions/max_terminated_length": 16.0, |
| "completions/mean_length": 14.4375, |
| "completions/mean_terminated_length": 9.75, |
| "completions/min_length": 4.0, |
| "completions/min_terminated_length": 4.0, |
| "entropy": 0.4573390781879425, |
| "epoch": 6e-05, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.4110209047794342, |
| "kl": 0.00046882770402589813, |
| "learning_rate": 1.7142857142857149e-07, |
| "loss": 0.1188, |
| "num_tokens": 106349.0, |
| "reward": -22.559879302978516, |
| "reward_std": 14.563770294189453, |
| "rewards/rollout_reward_func/mean": -22.559879302978516, |
| "rewards/rollout_reward_func/std": 14.563770294189453, |
| "sampling/importance_sampling_ratio/max": 1.2753626108169556, |
| "sampling/importance_sampling_ratio/mean": 0.9817989468574524, |
| "sampling/importance_sampling_ratio/min": 0.8001569509506226, |
| "sampling/sampling_logp_difference/max": 0.2265012264251709, |
| "sampling/sampling_logp_difference/mean": 0.012659025378525257, |
| "step": 3, |
| "step_time": 13.572802005000085 |
| }, |
| { |
| "clip_ratio/high_max": 0.0078125, |
| "clip_ratio/high_mean": 0.001953125, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.001953125, |
| "entropy": 0.45671091973781586, |
| "epoch": 8e-05, |
| "grad_norm": 0.3884861469268799, |
| "kl": 0.00040267666918225586, |
| "learning_rate": 2.571428571428572e-07, |
| "loss": 0.1171, |
| "step": 4, |
| "step_time": 3.672753089998878 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.78125, |
| "completions/max_length": 16.0, |
| "completions/max_terminated_length": 13.0, |
| "completions/mean_length": 14.09375, |
| "completions/mean_terminated_length": 7.285714626312256, |
| "completions/min_length": 4.0, |
| "completions/min_terminated_length": 4.0, |
| "entropy": 0.4637308567762375, |
| "epoch": 0.0001, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.5923936367034912, |
| "kl": 0.00045897291784058325, |
| "learning_rate": 3.4285714285714297e-07, |
| "loss": 0.1461, |
| "num_tokens": 158201.0, |
| "reward": -23.239702224731445, |
| "reward_std": 11.33220386505127, |
| "rewards/rollout_reward_func/mean": -23.239702224731445, |
| "rewards/rollout_reward_func/std": 11.332201957702637, |
| "sampling/importance_sampling_ratio/max": 1.179511308670044, |
| "sampling/importance_sampling_ratio/mean": 0.9810092449188232, |
| "sampling/importance_sampling_ratio/min": 0.7553079128265381, |
| "sampling/sampling_logp_difference/max": 0.19478893280029297, |
| "sampling/sampling_logp_difference/mean": 0.012187635526061058, |
| "step": 5, |
| "step_time": 12.394922012998904 |
| }, |
| { |
| "epoch": 0.0001, |
| "eval_clip_ratio/high_max": 0.0, |
| "eval_clip_ratio/high_mean": 0.0, |
| "eval_clip_ratio/low_mean": 0.0, |
| "eval_clip_ratio/low_min": 0.0, |
| "eval_clip_ratio/region_mean": 0.0, |
| "eval_completions/clipped_ratio": 0.85, |
| "eval_completions/max_length": 16.0, |
| "eval_completions/max_terminated_length": 5.4, |
| "eval_completions/mean_length": 15.225, |
| "eval_completions/mean_terminated_length": 5.15, |
| "eval_completions/min_length": 12.9, |
| "eval_completions/min_terminated_length": 4.9, |
| "eval_entropy": 0.45741056799888613, |
| "eval_frac_reward_zero_std": 0.0, |
| "eval_kl": 0.00042195062269456685, |
| "eval_loss": 0.004154330585151911, |
| "eval_num_tokens": 158201.0, |
| "eval_reward": -25.101631736755373, |
| "eval_reward_std": 13.229044437408447, |
| "eval_rewards/rollout_reward_func/mean": -25.101631736755373, |
| "eval_rewards/rollout_reward_func/std": 13.2290442943573, |
| "eval_runtime": 13.4386, |
| "eval_samples_per_second": 0.744, |
| "eval_sampling/importance_sampling_ratio/max": 1.112263822555542, |
| "eval_sampling/importance_sampling_ratio/mean": 0.9942494869232178, |
| "eval_sampling/importance_sampling_ratio/min": 0.8756316006183624, |
| "eval_sampling/sampling_logp_difference/max": 0.1471207022666931, |
| "eval_sampling/sampling_logp_difference/mean": 0.013286880496889353, |
| "eval_steps_per_second": 0.223, |
| "step": 5 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "entropy": 0.46300478279590607, |
| "epoch": 0.00012, |
| "grad_norm": 0.5754485726356506, |
| "kl": 0.0004794619817403145, |
| "learning_rate": 4.2857142857142867e-07, |
| "loss": 0.1461, |
| "step": 6, |
| "step_time": 3.5247847910004566 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.78125, |
| "completions/max_length": 16.0, |
| "completions/max_terminated_length": 14.0, |
| "completions/mean_length": 14.6875, |
| "completions/mean_terminated_length": 10.0, |
| "completions/min_length": 3.0, |
| "completions/min_terminated_length": 3.0, |
| "entropy": 0.4924817681312561, |
| "epoch": 0.00014, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.5342097282409668, |
| "kl": 0.00047566716602887027, |
| "learning_rate": 5.142857142857144e-07, |
| "loss": 0.0788, |
| "num_tokens": 211022.0, |
| "reward": -21.68277359008789, |
| "reward_std": 14.98656940460205, |
| "rewards/rollout_reward_func/mean": -21.68277359008789, |
| "rewards/rollout_reward_func/std": 14.986568450927734, |
| "sampling/importance_sampling_ratio/max": 1.2211930751800537, |
| "sampling/importance_sampling_ratio/mean": 0.9879010319709778, |
| "sampling/importance_sampling_ratio/min": 0.7968342304229736, |
| "sampling/sampling_logp_difference/max": 0.2077500820159912, |
| "sampling/sampling_logp_difference/mean": 0.015078969299793243, |
| "step": 7, |
| "step_time": 13.125986247000583 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.001953125, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.001953125, |
| "entropy": 0.4939192831516266, |
| "epoch": 0.00016, |
| "grad_norm": 0.5269240140914917, |
| "kl": 0.0005614961846731603, |
| "learning_rate": 6.000000000000002e-07, |
| "loss": 0.078, |
| "step": 8, |
| "step_time": 4.57110312200075 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.84375, |
| "completions/max_length": 16.0, |
| "completions/max_terminated_length": 16.0, |
| "completions/mean_length": 15.53125, |
| "completions/mean_terminated_length": 13.0, |
| "completions/min_length": 3.0, |
| "completions/min_terminated_length": 3.0, |
| "entropy": 0.4650999382138252, |
| "epoch": 0.00018, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.4413743019104004, |
| "kl": 0.0005437940853880718, |
| "learning_rate": 6.857142857142859e-07, |
| "loss": 0.0419, |
| "num_tokens": 262651.0, |
| "reward": -24.077335357666016, |
| "reward_std": 11.67958927154541, |
| "rewards/rollout_reward_func/mean": -24.077335357666016, |
| "rewards/rollout_reward_func/std": 11.67958927154541, |
| "sampling/importance_sampling_ratio/max": 1.2573319673538208, |
| "sampling/importance_sampling_ratio/mean": 1.0008736848831177, |
| "sampling/importance_sampling_ratio/min": 0.8466115593910217, |
| "sampling/sampling_logp_difference/max": 0.15867352485656738, |
| "sampling/sampling_logp_difference/mean": 0.011756965890526772, |
| "step": 9, |
| "step_time": 12.814324255000429 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "entropy": 0.46368078887462616, |
| "epoch": 0.0002, |
| "grad_norm": 0.4458825886249542, |
| "kl": 0.0005188173963688314, |
| "learning_rate": 7.714285714285716e-07, |
| "loss": 0.0428, |
| "step": 10, |
| "step_time": 3.553372275999209 |
| }, |
| { |
| "epoch": 0.0002, |
| "eval_clip_ratio/high_max": 0.0, |
| "eval_clip_ratio/high_mean": 0.0, |
| "eval_clip_ratio/low_mean": 0.0, |
| "eval_clip_ratio/low_min": 0.0, |
| "eval_clip_ratio/region_mean": 0.0, |
| "eval_completions/clipped_ratio": 0.7, |
| "eval_completions/max_length": 16.0, |
| "eval_completions/max_terminated_length": 9.8, |
| "eval_completions/mean_length": 14.7, |
| "eval_completions/mean_terminated_length": 9.16666669845581, |
| "eval_completions/min_length": 11.8, |
| "eval_completions/min_terminated_length": 8.6, |
| "eval_entropy": 0.40578661262989046, |
| "eval_frac_reward_zero_std": 0.0, |
| "eval_kl": 0.00042531737562967464, |
| "eval_loss": 0.02340291067957878, |
| "eval_num_tokens": 262651.0, |
| "eval_reward": -18.063774490356444, |
| "eval_reward_std": 14.382386589050293, |
| "eval_rewards/rollout_reward_func/mean": -18.063774490356444, |
| "eval_rewards/rollout_reward_func/std": 14.382386589050293, |
| "eval_runtime": 13.98, |
| "eval_samples_per_second": 0.715, |
| "eval_sampling/importance_sampling_ratio/max": 1.1354158759117126, |
| "eval_sampling/importance_sampling_ratio/mean": 1.0028302788734436, |
| "eval_sampling/importance_sampling_ratio/min": 0.9048627257347107, |
| "eval_sampling/sampling_logp_difference/max": 0.13887375593185425, |
| "eval_sampling/sampling_logp_difference/mean": 0.011902038706466556, |
| "eval_steps_per_second": 0.215, |
| "step": 10 |
| } |
| ], |
| "logging_steps": 1.0, |
| "max_steps": 100000, |
| "num_input_tokens_seen": 262651, |
| "num_train_epochs": 2, |
| "save_steps": 500, |
| "stateful_callbacks": { |
| "TrainerControl": { |
| "args": { |
| "should_epoch_stop": false, |
| "should_evaluate": false, |
| "should_log": false, |
| "should_save": true, |
| "should_training_stop": false |
| }, |
| "attributes": {} |
| } |
| }, |
| "total_flos": 0.0, |
| "train_batch_size": 2, |
| "trial_name": null, |
| "trial_params": null |
| } |
|
|
