sam1120's picture
Training in progress, step 80
8bf97e1
raw
history blame
14.8 kB
{
"best_metric": 0.8647122979164124,
"best_model_checkpoint": "/robodata/smodak/Projects/nspl/scripts/terrainseg/training/models/safety-utcustom-train-SF-RGB-b5/checkpoint-80",
"epoch": 7.2727272727272725,
"global_step": 80,
"is_hyper_param_search": false,
"is_local_process_zero": true,
"is_world_process_zero": true,
"log_history": [
{
"epoch": 0.09,
"learning_rate": 1.764705882352941e-07,
"loss": 1.2293,
"step": 1
},
{
"epoch": 0.18,
"learning_rate": 3.529411764705882e-07,
"loss": 1.2349,
"step": 2
},
{
"epoch": 0.27,
"learning_rate": 5.294117647058824e-07,
"loss": 1.2357,
"step": 3
},
{
"epoch": 0.36,
"learning_rate": 7.058823529411765e-07,
"loss": 1.2242,
"step": 4
},
{
"epoch": 0.45,
"learning_rate": 8.823529411764706e-07,
"loss": 1.2247,
"step": 5
},
{
"epoch": 0.55,
"learning_rate": 1.0588235294117648e-06,
"loss": 1.227,
"step": 6
},
{
"epoch": 0.64,
"learning_rate": 1.2352941176470588e-06,
"loss": 1.2292,
"step": 7
},
{
"epoch": 0.73,
"learning_rate": 1.411764705882353e-06,
"loss": 1.2257,
"step": 8
},
{
"epoch": 0.82,
"learning_rate": 1.5882352941176472e-06,
"loss": 1.2173,
"step": 9
},
{
"epoch": 0.91,
"learning_rate": 1.7647058823529412e-06,
"loss": 1.2239,
"step": 10
},
{
"epoch": 0.91,
"eval_accuracy_safe": 0.39920432072351303,
"eval_accuracy_unlabeled": NaN,
"eval_accuracy_unsafe": 0.29510386893492613,
"eval_iou_safe": 0.031418797454432365,
"eval_iou_unlabeled": 0.0,
"eval_iou_unsafe": 0.29392505413655395,
"eval_loss": 1.1102662086486816,
"eval_mean_accuracy": 0.34715409482921955,
"eval_mean_iou": 0.10844795053032878,
"eval_overall_accuracy": 0.2981788293639226,
"eval_runtime": 9.3061,
"eval_samples_per_second": 7.2,
"eval_steps_per_second": 0.537,
"step": 10
},
{
"epoch": 1.0,
"learning_rate": 1.9411764705882353e-06,
"loss": 1.221,
"step": 11
},
{
"epoch": 1.09,
"learning_rate": 2.1176470588235296e-06,
"loss": 1.2149,
"step": 12
},
{
"epoch": 1.18,
"learning_rate": 2.2941176470588234e-06,
"loss": 1.2197,
"step": 13
},
{
"epoch": 1.27,
"learning_rate": 2.4705882352941177e-06,
"loss": 1.2064,
"step": 14
},
{
"epoch": 1.36,
"learning_rate": 2.647058823529412e-06,
"loss": 1.2042,
"step": 15
},
{
"epoch": 1.45,
"learning_rate": 2.823529411764706e-06,
"loss": 1.2046,
"step": 16
},
{
"epoch": 1.55,
"learning_rate": 3e-06,
"loss": 1.2054,
"step": 17
},
{
"epoch": 1.64,
"learning_rate": 2.990415335463259e-06,
"loss": 1.2002,
"step": 18
},
{
"epoch": 1.73,
"learning_rate": 2.9808306709265177e-06,
"loss": 1.1982,
"step": 19
},
{
"epoch": 1.82,
"learning_rate": 2.9712460063897764e-06,
"loss": 1.1948,
"step": 20
},
{
"epoch": 1.82,
"eval_accuracy_safe": 0.5218599774094934,
"eval_accuracy_unlabeled": NaN,
"eval_accuracy_unsafe": 0.3705393407485172,
"eval_iou_safe": 0.0439841259320199,
"eval_iou_unlabeled": 0.0,
"eval_iou_unsafe": 0.36893471456827454,
"eval_loss": 1.0963102579116821,
"eval_mean_accuracy": 0.4461996590790053,
"eval_mean_iou": 0.13763961350009815,
"eval_overall_accuracy": 0.37500910972481344,
"eval_runtime": 8.3942,
"eval_samples_per_second": 7.982,
"eval_steps_per_second": 0.596,
"step": 20
},
{
"epoch": 1.91,
"learning_rate": 2.9616613418530352e-06,
"loss": 1.1776,
"step": 21
},
{
"epoch": 2.0,
"learning_rate": 2.952076677316294e-06,
"loss": 1.1936,
"step": 22
},
{
"epoch": 2.09,
"learning_rate": 2.942492012779553e-06,
"loss": 1.1664,
"step": 23
},
{
"epoch": 2.18,
"learning_rate": 2.9329073482428116e-06,
"loss": 1.1809,
"step": 24
},
{
"epoch": 2.27,
"learning_rate": 2.9233226837060704e-06,
"loss": 1.1731,
"step": 25
},
{
"epoch": 2.36,
"learning_rate": 2.913738019169329e-06,
"loss": 1.1794,
"step": 26
},
{
"epoch": 2.45,
"learning_rate": 2.904153354632588e-06,
"loss": 1.1684,
"step": 27
},
{
"epoch": 2.55,
"learning_rate": 2.8945686900958468e-06,
"loss": 1.1594,
"step": 28
},
{
"epoch": 2.64,
"learning_rate": 2.884984025559105e-06,
"loss": 1.146,
"step": 29
},
{
"epoch": 2.73,
"learning_rate": 2.8753993610223644e-06,
"loss": 1.1661,
"step": 30
},
{
"epoch": 2.73,
"eval_accuracy_safe": 0.5863200218966003,
"eval_accuracy_unlabeled": NaN,
"eval_accuracy_unsafe": 0.49882181393718666,
"eval_iou_safe": 0.06472096392925002,
"eval_iou_unlabeled": 0.0,
"eval_iou_unsafe": 0.49613792944888824,
"eval_loss": 1.0516184568405151,
"eval_mean_accuracy": 0.5425709179168935,
"eval_mean_iou": 0.1869529644593794,
"eval_overall_accuracy": 0.5014063707038537,
"eval_runtime": 9.3615,
"eval_samples_per_second": 7.157,
"eval_steps_per_second": 0.534,
"step": 30
},
{
"epoch": 2.82,
"learning_rate": 2.865814696485623e-06,
"loss": 1.153,
"step": 31
},
{
"epoch": 2.91,
"learning_rate": 2.856230031948882e-06,
"loss": 1.149,
"step": 32
},
{
"epoch": 3.0,
"learning_rate": 2.8466453674121407e-06,
"loss": 1.164,
"step": 33
},
{
"epoch": 3.09,
"learning_rate": 2.8370607028753995e-06,
"loss": 1.1479,
"step": 34
},
{
"epoch": 3.18,
"learning_rate": 2.8274760383386583e-06,
"loss": 1.1428,
"step": 35
},
{
"epoch": 3.27,
"learning_rate": 2.817891373801917e-06,
"loss": 1.1398,
"step": 36
},
{
"epoch": 3.36,
"learning_rate": 2.8083067092651755e-06,
"loss": 1.1415,
"step": 37
},
{
"epoch": 3.45,
"learning_rate": 2.7987220447284347e-06,
"loss": 1.1146,
"step": 38
},
{
"epoch": 3.55,
"learning_rate": 2.7891373801916935e-06,
"loss": 1.1234,
"step": 39
},
{
"epoch": 3.64,
"learning_rate": 2.7795527156549523e-06,
"loss": 1.1112,
"step": 40
},
{
"epoch": 3.64,
"eval_accuracy_safe": 0.5458517893146133,
"eval_accuracy_unlabeled": NaN,
"eval_accuracy_unsafe": 0.5794390867479824,
"eval_iou_safe": 0.0899865428032596,
"eval_iou_unlabeled": 0.0,
"eval_iou_unsafe": 0.5753581085678546,
"eval_loss": 1.0047743320465088,
"eval_mean_accuracy": 0.5626454380312979,
"eval_mean_iou": 0.22178155045703807,
"eval_overall_accuracy": 0.5784469718363747,
"eval_runtime": 9.38,
"eval_samples_per_second": 7.143,
"eval_steps_per_second": 0.533,
"step": 40
},
{
"epoch": 3.73,
"learning_rate": 2.769968051118211e-06,
"loss": 1.1212,
"step": 41
},
{
"epoch": 3.82,
"learning_rate": 2.7603833865814694e-06,
"loss": 1.1117,
"step": 42
},
{
"epoch": 3.91,
"learning_rate": 2.7507987220447286e-06,
"loss": 1.1051,
"step": 43
},
{
"epoch": 4.0,
"learning_rate": 2.7412140575079874e-06,
"loss": 1.1382,
"step": 44
},
{
"epoch": 4.09,
"learning_rate": 2.731629392971246e-06,
"loss": 1.1075,
"step": 45
},
{
"epoch": 4.18,
"learning_rate": 2.722044728434505e-06,
"loss": 1.1096,
"step": 46
},
{
"epoch": 4.27,
"learning_rate": 2.7124600638977634e-06,
"loss": 1.0974,
"step": 47
},
{
"epoch": 4.36,
"learning_rate": 2.7028753993610226e-06,
"loss": 1.0851,
"step": 48
},
{
"epoch": 4.45,
"learning_rate": 2.6932907348242814e-06,
"loss": 1.0851,
"step": 49
},
{
"epoch": 4.55,
"learning_rate": 2.6837060702875398e-06,
"loss": 1.0907,
"step": 50
},
{
"epoch": 4.55,
"eval_accuracy_safe": 0.5993018531154467,
"eval_accuracy_unlabeled": NaN,
"eval_accuracy_unsafe": 0.6366664738420048,
"eval_iou_safe": 0.10939424133801141,
"eval_iou_unlabeled": 0.0,
"eval_iou_unsafe": 0.6320960572783577,
"eval_loss": 0.9689871668815613,
"eval_mean_accuracy": 0.6179841634787258,
"eval_mean_iou": 0.24716343287212306,
"eval_overall_accuracy": 0.6355627828569554,
"eval_runtime": 9.8119,
"eval_samples_per_second": 6.828,
"eval_steps_per_second": 0.51,
"step": 50
},
{
"epoch": 4.64,
"learning_rate": 2.674121405750799e-06,
"loss": 1.0789,
"step": 51
},
{
"epoch": 4.73,
"learning_rate": 2.6645367412140573e-06,
"loss": 1.0566,
"step": 52
},
{
"epoch": 4.82,
"learning_rate": 2.654952076677316e-06,
"loss": 1.0718,
"step": 53
},
{
"epoch": 4.91,
"learning_rate": 2.6453674121405753e-06,
"loss": 1.0808,
"step": 54
},
{
"epoch": 5.0,
"learning_rate": 2.6357827476038337e-06,
"loss": 1.0808,
"step": 55
},
{
"epoch": 5.09,
"learning_rate": 2.626198083067093e-06,
"loss": 1.0625,
"step": 56
},
{
"epoch": 5.18,
"learning_rate": 2.6166134185303517e-06,
"loss": 1.0508,
"step": 57
},
{
"epoch": 5.27,
"learning_rate": 2.60702875399361e-06,
"loss": 1.0521,
"step": 58
},
{
"epoch": 5.36,
"learning_rate": 2.5974440894568693e-06,
"loss": 1.052,
"step": 59
},
{
"epoch": 5.45,
"learning_rate": 2.5878594249201277e-06,
"loss": 1.047,
"step": 60
},
{
"epoch": 5.45,
"eval_accuracy_safe": 0.6691570194409427,
"eval_accuracy_unlabeled": NaN,
"eval_accuracy_unsafe": 0.669918812994849,
"eval_iou_safe": 0.11592410370525706,
"eval_iou_unlabeled": 0.0,
"eval_iou_unsafe": 0.6655783206367077,
"eval_loss": 0.9437160491943359,
"eval_mean_accuracy": 0.6695379162178958,
"eval_mean_iou": 0.2605008081139883,
"eval_overall_accuracy": 0.6698963108347423,
"eval_runtime": 9.3506,
"eval_samples_per_second": 7.165,
"eval_steps_per_second": 0.535,
"step": 60
},
{
"epoch": 5.55,
"learning_rate": 2.5782747603833865e-06,
"loss": 1.0456,
"step": 61
},
{
"epoch": 5.64,
"learning_rate": 2.5686900958466457e-06,
"loss": 1.0296,
"step": 62
},
{
"epoch": 5.73,
"learning_rate": 2.559105431309904e-06,
"loss": 1.0334,
"step": 63
},
{
"epoch": 5.82,
"learning_rate": 2.5495207667731633e-06,
"loss": 1.0525,
"step": 64
},
{
"epoch": 5.91,
"learning_rate": 2.5399361022364216e-06,
"loss": 1.0397,
"step": 65
},
{
"epoch": 6.0,
"learning_rate": 2.5303514376996804e-06,
"loss": 1.0575,
"step": 66
},
{
"epoch": 6.09,
"learning_rate": 2.5207667731629396e-06,
"loss": 1.0245,
"step": 67
},
{
"epoch": 6.18,
"learning_rate": 2.511182108626198e-06,
"loss": 1.0284,
"step": 68
},
{
"epoch": 6.27,
"learning_rate": 2.501597444089457e-06,
"loss": 1.0127,
"step": 69
},
{
"epoch": 6.36,
"learning_rate": 2.4920127795527156e-06,
"loss": 1.0112,
"step": 70
},
{
"epoch": 6.36,
"eval_accuracy_safe": 0.6673046750012529,
"eval_accuracy_unlabeled": NaN,
"eval_accuracy_unsafe": 0.718865515124044,
"eval_iou_safe": 0.13494764234639625,
"eval_iou_unlabeled": 0.0,
"eval_iou_unsafe": 0.7136723036690169,
"eval_loss": 0.9084014892578125,
"eval_mean_accuracy": 0.6930850950626484,
"eval_mean_iou": 0.28287331533847104,
"eval_overall_accuracy": 0.7173424905805446,
"eval_runtime": 9.0845,
"eval_samples_per_second": 7.375,
"eval_steps_per_second": 0.55,
"step": 70
},
{
"epoch": 6.45,
"learning_rate": 2.4824281150159744e-06,
"loss": 1.0107,
"step": 71
},
{
"epoch": 6.55,
"learning_rate": 2.4728434504792336e-06,
"loss": 1.0122,
"step": 72
},
{
"epoch": 6.64,
"learning_rate": 2.463258785942492e-06,
"loss": 1.0087,
"step": 73
},
{
"epoch": 6.73,
"learning_rate": 2.4536741214057507e-06,
"loss": 0.991,
"step": 74
},
{
"epoch": 6.82,
"learning_rate": 2.44408945686901e-06,
"loss": 1.0042,
"step": 75
},
{
"epoch": 6.91,
"learning_rate": 2.4345047923322683e-06,
"loss": 0.9788,
"step": 76
},
{
"epoch": 7.0,
"learning_rate": 2.424920127795527e-06,
"loss": 1.0371,
"step": 77
},
{
"epoch": 7.09,
"learning_rate": 2.415335463258786e-06,
"loss": 0.9884,
"step": 78
},
{
"epoch": 7.18,
"learning_rate": 2.4057507987220447e-06,
"loss": 0.9759,
"step": 79
},
{
"epoch": 7.27,
"learning_rate": 2.396166134185304e-06,
"loss": 0.9925,
"step": 80
},
{
"epoch": 7.27,
"eval_accuracy_safe": 0.6842282797676185,
"eval_accuracy_unlabeled": NaN,
"eval_accuracy_unsafe": 0.7664939888573942,
"eval_iou_safe": 0.14521835177647324,
"eval_iou_unlabeled": 0.0,
"eval_iou_unsafe": 0.7605469001045981,
"eval_loss": 0.8647122979164124,
"eval_mean_accuracy": 0.7253611343125064,
"eval_mean_iou": 0.3019217506270238,
"eval_overall_accuracy": 0.7640639917174382,
"eval_runtime": 9.9787,
"eval_samples_per_second": 6.714,
"eval_steps_per_second": 0.501,
"step": 80
}
],
"max_steps": 330,
"num_train_epochs": 30,
"total_flos": 4.482705625747292e+17,
"trial_name": null,
"trial_params": null
}