a2_rl_crosscodeeval_python_v2 / training_logs /20260428_203121_metrics_table.csv
atutej's picture
Add parsed training metrics and plots
d7fbe0a verified
log_file,async/discard_rate,async/discarded_count,async/effective_batch_groups,async/effective_batch_samples,async/staleness_max,async/staleness_mean,async/staleness_min,async/staleness_ratio,generate/avg_num_tokens,generate/avg_tokens_non_zero_rewards,generate/avg_tokens_zero_rewards,generate/max_num_tokens,generate/min_num_tokens,generate/std_num_tokens,loss/avg_final_rewards,loss/avg_raw_advantages,loss/avg_raw_advantages_abs,policy/final_loss,policy/policy_entropy,policy/policy_loss,policy/policy_lr,policy/policy_update_steps,policy/ppo_clip_ratio,policy/raw_grad_norm,reward/avg_pass_at_8,reward/avg_raw_reward,system/process_rss_gb,system/process_vms_gb,system/ram_available_gb,system/ram_percent,system/ram_total_gb,system/ram_used_gb,timing/cleanup_old_checkpoints,timing/compute_advantages_and_returns,timing/convert_to_training_input,timing/fwd_logprobs_values_reward,timing/policy_train,timing/run_training,timing/save_checkpoints,timing/step,timing/sync_weights,timing/train_critic_and_policy,timing/wait_for_generation_buffer,trainer/epoch,trainer/global_step,batch_errors/total_batches,batch_errors/total_instances,batch_errors/total_successful,batch_errors/total_failed,batch_errors/total_masked,timing/save_hf_model,global_step
job_389754,0.0,0,64,512,0,0.0,0,0.0,3837.043,1844.4,3856.6943,11791,1000,1728.238,0.0098,-0.003,0.0108,-0.0,0.2114,-0.0,0.0,1.0,0.0,0.0146,0.0469,0.0098,11.0404,73.0374,334.9649,61.0,857.9687,523.0038,10.3772,0.0525,2.3302,54.8335,599.7188,654.8685,44.3151,2252.5445,55.9998,599.982,1539.3458,0,1,128.0,1024.0,1024.0,0.0,0.0,,1
job_389754,0.0,0,64,512,1,1.0,1,1.0,3734.7441,4189.0,3722.9098,14143,1008,1771.7086,0.0254,-0.003,0.0135,0.0,0.2119,0.0,0.0,1.0,0.0,0.0147,0.0781,0.0254,12.3731,73.238,331.4794,61.4,857.9687,526.4893,0.0156,0.0625,2.608,51.6906,593.7021,645.7138,33.1731,927.602,58.3788,593.9603,220.8803,0,2,64.0,512.0,512.0,0.0,0.0,,2
job_389754,0.0,0,64,512,2,2.0,2,1.0,3680.1211,3429.5,3685.1135,17798,960,1903.0569,0.0195,-0.0057,0.0404,0.0,0.2131,0.0,0.0,1.0,0.0,0.017,0.125,0.0195,15.006,73.3821,331.1569,61.4,857.9687,526.8118,0.0079,0.0712,3.338,54.9564,607.7339,663.0308,33.7102,1161.2,54.4484,608.0028,440.3575,0,3,64.0,512.0,512.0,0.0,0.0,,3
job_389754,0.0,0,64,512,3,3.0,3,1.0,4038.8242,3594.9412,4054.0687,19942,1070,2204.7849,0.0332,-0.0083,0.0398,0.0,0.1988,0.0,0.0,1.0,0.0,0.0159,0.1094,0.0332,18.2571,73.6598,332.0689,61.3,857.9687,525.8998,0.0123,0.0638,3.988,64.8817,648.7096,713.9267,33.5855,774.7256,56.7814,648.9808,0.0053,0,4,64.0,512.0,512.0,0.0,0.0,,4
job_389754,0.0,0,64,512,4,4.0,4,1.0,3734.0176,3721.08,3734.6817,13083,1049,1658.2439,0.0488,-0.0016,0.0331,0.0,0.2072,0.0,0.0,1.0,0.0,0.0298,0.125,0.0488,18.5352,73.6546,329.1295,61.6,857.9687,528.8392,0.0099,0.0428,2.9611,54.8169,588.7437,643.8552,33.7826,946.1626,58.0555,588.9951,241.2625,0,5,64.0,512.0,512.0,0.0,0.0,38.1036,5
job_389754,0.0,0,64,512,5,5.0,5,1.0,3678.4492,2779.8125,3707.4375,16575,982,1778.7688,0.0312,-0.0069,0.0393,0.0,0.2064,0.0,0.0,1.0,0.0,0.0194,0.1406,0.0312,18.5881,73.6669,327.9209,61.8,857.9687,530.0478,0.0087,0.0617,3.0487,49.8621,590.8918,641.066,34.0988,888.5801,61.7658,591.1417,182.6699,0,6,64.0,512.0,512.0,0.0,0.0,,6
job_389754,0.0,0,64,512,6,6.0,6,1.0,4026.377,2695.15,4080.4919,30105,860,2435.5002,0.0391,-0.0117,0.0579,0.0,0.2004,0.0,0.0,1.0,0.0,0.0242,0.2031,0.0391,24.2751,73.9734,398.3101,53.6,857.9687,459.6586,0.0095,0.0949,4.4248,61.0183,660.677,722.0682,35.0715,2094.5908,55.4597,660.9545,1312.6129,0,7,64.0,512.0,512.0,0.0,0.0,,7
job_389754,0.0,0,64,512,0,0.0,0,0.0,3169.8066,2304.1277,3257.3054,9841,950,1527.965,0.0918,-0.023,0.1174,0.0,0.2084,0.0,0.0,1.0,0.0,0.0364,0.3281,0.0918,24.1753,73.8303,401.2653,53.2,857.9687,456.7034,0.0112,0.0425,2.0132,44.7602,566.5425,611.7071,33.0222,1954.7684,56.9235,566.9038,1284.0936,1,8,64.0,512.0,512.0,0.0,0.0,,8
job_389754,0.0,0,64,512,1,1.0,1,1.0,3208.3555,2564.1489,3273.4688,19175,927,1685.1496,0.0918,-0.0201,0.1149,0.0,0.2081,0.0,0.0,1.0,0.0,0.0337,0.3281,0.0918,24.6965,74.2126,396.9511,53.7,857.9687,461.0176,0.0079,0.0446,3.5998,47.727,576.6595,624.6952,33.46,810.2478,57.0584,576.9233,124.8704,1,9,64.0,512.0,512.0,0.0,0.0,,9
job_389754,0.0,0,64,512,2,2.0,2,1.0,3237.8379,2406.9649,3341.9253,12501,890,1700.2375,0.1113,-0.0333,0.1439,0.0,0.2076,0.0,0.0,1.0,0.0,0.0374,0.3906,0.1113,24.8782,74.2612,391.3597,54.4,857.9687,466.609,0.0092,0.0533,2.4828,50.2219,593.8531,644.3998,33.4188,912.5225,56.7812,594.124,208.8362,1,10,64.0,512.0,512.0,0.0,0.0,37.5099,10
job_389754,0.0,0,64,512,3,3.0,3,1.0,3001.6465,2650.619,3050.8998,13406,941,1483.113,0.123,-0.0266,0.1658,0.0,0.199,0.0,0.0,1.0,0.0,0.0435,0.4219,0.123,25.0717,74.3157,387.2821,54.9,857.9687,470.6866,0.0168,0.0371,2.6113,41.7835,561.0683,603.1351,33.5831,731.0664,56.667,561.3141,68.6264,1,11,64.0,512.0,512.0,0.0,0.0,,11
job_389754,0.0,0,64,512,4,4.0,4,1.0,2883.377,2714.9545,2918.3325,14702,798,1601.3015,0.1719,-0.0285,0.1735,-0.0,0.1995,-0.0,0.0,1.0,0.0,0.0405,0.4219,0.1719,25.1736,74.6695,384.2253,55.2,857.9687,473.7434,0.011,0.0379,2.9524,47.2009,561.4127,608.9848,33.1065,931.2563,58.3024,561.7455,260.9926,1,12,64.0,512.0,512.0,0.0,0.0,,12
job_389754,0.0,0,64,512,5,5.0,5,1.0,2857.6934,2411.3333,2960.6995,13216,868,1685.3561,0.1875,-0.0066,0.1342,0.0,0.1896,0.0,0.0,1.0,0.0,0.0396,0.3906,0.1875,25.3254,74.5136,380.6907,55.6,857.9687,477.278,0.0093,0.0471,3.5522,43.8172,560.515,604.6342,33.2148,682.4333,57.7062,560.7695,16.5161,1,13,64.0,512.0,512.0,0.0,0.0,,13
job_389754,0.0,0,64,512,6,6.0,6,1.0,3206.502,2278.8974,3373.2143,15130,906,2031.0791,0.1523,-0.0278,0.1606,-0.0,0.1855,-0.0,0.0,1.0,0.0,0.0446,0.4688,0.1523,25.6977,74.7123,378.3962,55.9,857.9687,479.5725,0.0096,0.0577,2.5454,52.8053,611.3389,664.4901,33.0486,4631.2906,13.904,611.6265,3950.3283,1,14,,,,,,,14