sam1120's picture
Training in progress, step 80
1c790ff
raw
history blame
16.6 kB
{
"best_metric": 1.4904648065567017,
"best_model_checkpoint": "/robodata/smodak/Projects/nspl/scripts/terrainseg/training/models/safety-utcustom-terrain-b0-optim/checkpoint-80",
"epoch": 7.2727272727272725,
"global_step": 80,
"is_hyper_param_search": false,
"is_local_process_zero": true,
"is_world_process_zero": true,
"log_history": [
{
"epoch": 0.09,
"learning_rate": 3.6363636363636366e-07,
"loss": 2.757,
"step": 1
},
{
"epoch": 0.18,
"learning_rate": 7.272727272727273e-07,
"loss": 2.7552,
"step": 2
},
{
"epoch": 0.27,
"learning_rate": 1.090909090909091e-06,
"loss": 2.7635,
"step": 3
},
{
"epoch": 0.36,
"learning_rate": 1.4545454545454546e-06,
"loss": 2.7567,
"step": 4
},
{
"epoch": 0.45,
"learning_rate": 1.818181818181818e-06,
"loss": 2.7588,
"step": 5
},
{
"epoch": 0.55,
"learning_rate": 2.181818181818182e-06,
"loss": 2.7621,
"step": 6
},
{
"epoch": 0.64,
"learning_rate": 2.5454545454545456e-06,
"loss": 2.7553,
"step": 7
},
{
"epoch": 0.73,
"learning_rate": 2.9090909090909093e-06,
"loss": 2.7371,
"step": 8
},
{
"epoch": 0.82,
"learning_rate": 3.2727272727272733e-06,
"loss": 2.7444,
"step": 9
},
{
"epoch": 0.91,
"learning_rate": 3.636363636363636e-06,
"loss": 2.7363,
"step": 10
},
{
"epoch": 1.0,
"learning_rate": 4.000000000000001e-06,
"loss": 2.7364,
"step": 11
},
{
"epoch": 1.09,
"learning_rate": 4.363636363636364e-06,
"loss": 2.7248,
"step": 12
},
{
"epoch": 1.18,
"learning_rate": 4.727272727272727e-06,
"loss": 2.7344,
"step": 13
},
{
"epoch": 1.27,
"learning_rate": 5.090909090909091e-06,
"loss": 2.7259,
"step": 14
},
{
"epoch": 1.36,
"learning_rate": 5.4545454545454545e-06,
"loss": 2.7086,
"step": 15
},
{
"epoch": 1.45,
"learning_rate": 5.8181818181818185e-06,
"loss": 2.7087,
"step": 16
},
{
"epoch": 1.55,
"learning_rate": 6.181818181818183e-06,
"loss": 2.7055,
"step": 17
},
{
"epoch": 1.64,
"learning_rate": 6.545454545454547e-06,
"loss": 2.6959,
"step": 18
},
{
"epoch": 1.73,
"learning_rate": 6.909090909090909e-06,
"loss": 2.6983,
"step": 19
},
{
"epoch": 1.82,
"learning_rate": 7.272727272727272e-06,
"loss": 2.6746,
"step": 20
},
{
"epoch": 1.82,
"eval_accuracy_NAT": 0.019056367662015652,
"eval_accuracy_REST": 0.0,
"eval_accuracy_concrete": 0.0010381346838920125,
"eval_accuracy_concrete tiles": 0.09181801504311135,
"eval_accuracy_dark bricks": 0.09326045956660636,
"eval_accuracy_grass": 0.007958282800392846,
"eval_accuracy_red bricks": 0.00870028409090909,
"eval_accuracy_road": 0.05730198902763889,
"eval_accuracy_rough concrete": 0.03398947127489128,
"eval_accuracy_rough red sidewalk": 0.06001709260331376,
"eval_accuracy_speedway bricks": 0.03133919121631325,
"eval_accuracy_steel": 0.00842622518303542,
"eval_accuracy_tiles": 0.0,
"eval_accuracy_unlabeled": NaN,
"eval_iou_NAT": 0.018937014149192417,
"eval_iou_REST": 0.0,
"eval_iou_concrete": 0.0009231352727125269,
"eval_iou_concrete tiles": 0.002698064724802789,
"eval_iou_dark bricks": 0.0018963436490879863,
"eval_iou_grass": 0.006090841001497967,
"eval_iou_red bricks": 1.3048864535742704e-05,
"eval_iou_road": 0.0465812133841859,
"eval_iou_rough concrete": 0.004291938525567382,
"eval_iou_rough red sidewalk": 0.0013826848729701939,
"eval_iou_speedway bricks": 0.030977499755458056,
"eval_iou_steel": 0.0016645601782480097,
"eval_iou_tiles": 0.0,
"eval_iou_unlabeled": 0.0,
"eval_loss": 2.647437810897827,
"eval_mean_accuracy": 0.03176196255016307,
"eval_mean_iou": 0.008246881741304212,
"eval_overall_accuracy": 0.02327410380045573,
"eval_runtime": 3.4336,
"eval_samples_per_second": 10.485,
"eval_steps_per_second": 0.582,
"step": 20
},
{
"epoch": 1.91,
"learning_rate": 7.636363636363638e-06,
"loss": 2.6783,
"step": 21
},
{
"epoch": 2.0,
"learning_rate": 8.000000000000001e-06,
"loss": 2.6636,
"step": 22
},
{
"epoch": 2.09,
"learning_rate": 8.363636363636365e-06,
"loss": 2.6626,
"step": 23
},
{
"epoch": 2.18,
"learning_rate": 8.727272727272728e-06,
"loss": 2.6414,
"step": 24
},
{
"epoch": 2.27,
"learning_rate": 9.090909090909091e-06,
"loss": 2.6365,
"step": 25
},
{
"epoch": 2.36,
"learning_rate": 9.454545454545454e-06,
"loss": 2.6263,
"step": 26
},
{
"epoch": 2.45,
"learning_rate": 9.818181818181818e-06,
"loss": 2.6182,
"step": 27
},
{
"epoch": 2.55,
"learning_rate": 1.0181818181818182e-05,
"loss": 2.6144,
"step": 28
},
{
"epoch": 2.64,
"learning_rate": 1.0545454545454546e-05,
"loss": 2.5878,
"step": 29
},
{
"epoch": 2.73,
"learning_rate": 1.0909090909090909e-05,
"loss": 2.5814,
"step": 30
},
{
"epoch": 2.82,
"learning_rate": 1.1272727272727274e-05,
"loss": 2.5739,
"step": 31
},
{
"epoch": 2.91,
"learning_rate": 1.1636363636363637e-05,
"loss": 2.5536,
"step": 32
},
{
"epoch": 3.0,
"learning_rate": 1.2e-05,
"loss": 2.539,
"step": 33
},
{
"epoch": 3.09,
"learning_rate": 1.2363636363636365e-05,
"loss": 2.5256,
"step": 34
},
{
"epoch": 3.18,
"learning_rate": 1.2727272727272727e-05,
"loss": 2.4979,
"step": 35
},
{
"epoch": 3.27,
"learning_rate": 1.3090909090909093e-05,
"loss": 2.4936,
"step": 36
},
{
"epoch": 3.36,
"learning_rate": 1.3454545454545457e-05,
"loss": 2.4862,
"step": 37
},
{
"epoch": 3.45,
"learning_rate": 1.3818181818181818e-05,
"loss": 2.4491,
"step": 38
},
{
"epoch": 3.55,
"learning_rate": 1.4181818181818181e-05,
"loss": 2.4468,
"step": 39
},
{
"epoch": 3.64,
"learning_rate": 1.4545454545454545e-05,
"loss": 2.4385,
"step": 40
},
{
"epoch": 3.64,
"eval_accuracy_NAT": 0.447892573737157,
"eval_accuracy_REST": 0.0,
"eval_accuracy_concrete": 0.04360568050130907,
"eval_accuracy_concrete tiles": 0.10279459426404941,
"eval_accuracy_dark bricks": 0.07266848851083484,
"eval_accuracy_grass": 0.017545269942016727,
"eval_accuracy_red bricks": 0.0,
"eval_accuracy_road": 0.37485716220448445,
"eval_accuracy_rough concrete": 0.2949759670405127,
"eval_accuracy_rough red sidewalk": 0.05480660546411932,
"eval_accuracy_speedway bricks": 0.7969013802350152,
"eval_accuracy_steel": 0.0050211067871512435,
"eval_accuracy_tiles": 0.0,
"eval_accuracy_unlabeled": NaN,
"eval_iou_NAT": 0.4472536503231145,
"eval_iou_REST": 0.0,
"eval_iou_concrete": 0.039962583094048695,
"eval_iou_concrete tiles": 0.0041023463419931645,
"eval_iou_dark bricks": 0.007329448622040366,
"eval_iou_grass": 0.010573184550513284,
"eval_iou_red bricks": 0.0,
"eval_iou_road": 0.2852347789122688,
"eval_iou_rough concrete": 0.048997709321445884,
"eval_iou_rough red sidewalk": 0.003679975417468036,
"eval_iou_speedway bricks": 0.7166237181229831,
"eval_iou_steel": 0.0015635711039736272,
"eval_iou_tiles": 0.0,
"eval_iou_unlabeled": 0.0,
"eval_loss": 2.4373269081115723,
"eval_mean_accuracy": 0.17008221759128078,
"eval_mean_iou": 0.11180864041498925,
"eval_overall_accuracy": 0.4471186531914605,
"eval_runtime": 3.2402,
"eval_samples_per_second": 11.111,
"eval_steps_per_second": 0.617,
"step": 40
},
{
"epoch": 3.73,
"learning_rate": 1.4909090909090908e-05,
"loss": 2.4028,
"step": 41
},
{
"epoch": 3.82,
"learning_rate": 1.5272727272727276e-05,
"loss": 2.4083,
"step": 42
},
{
"epoch": 3.91,
"learning_rate": 1.563636363636364e-05,
"loss": 2.3613,
"step": 43
},
{
"epoch": 4.0,
"learning_rate": 1.6000000000000003e-05,
"loss": 2.3597,
"step": 44
},
{
"epoch": 4.09,
"learning_rate": 1.6363636363636366e-05,
"loss": 2.3331,
"step": 45
},
{
"epoch": 4.18,
"learning_rate": 1.672727272727273e-05,
"loss": 2.3068,
"step": 46
},
{
"epoch": 4.27,
"learning_rate": 1.7090909090909092e-05,
"loss": 2.2492,
"step": 47
},
{
"epoch": 4.36,
"learning_rate": 1.7454545454545456e-05,
"loss": 2.2469,
"step": 48
},
{
"epoch": 4.45,
"learning_rate": 1.781818181818182e-05,
"loss": 2.2194,
"step": 49
},
{
"epoch": 4.55,
"learning_rate": 1.8181818181818182e-05,
"loss": 2.2206,
"step": 50
},
{
"epoch": 4.64,
"learning_rate": 1.8545454545454545e-05,
"loss": 2.1524,
"step": 51
},
{
"epoch": 4.73,
"learning_rate": 1.890909090909091e-05,
"loss": 2.1454,
"step": 52
},
{
"epoch": 4.82,
"learning_rate": 1.9272727272727272e-05,
"loss": 2.1673,
"step": 53
},
{
"epoch": 4.91,
"learning_rate": 1.9636363636363635e-05,
"loss": 2.1274,
"step": 54
},
{
"epoch": 5.0,
"learning_rate": 2e-05,
"loss": 2.0958,
"step": 55
},
{
"epoch": 5.09,
"learning_rate": 2.0363636363636365e-05,
"loss": 2.0027,
"step": 56
},
{
"epoch": 5.18,
"learning_rate": 2.0727272727272728e-05,
"loss": 2.0081,
"step": 57
},
{
"epoch": 5.27,
"learning_rate": 2.109090909090909e-05,
"loss": 2.016,
"step": 58
},
{
"epoch": 5.36,
"learning_rate": 2.1454545454545455e-05,
"loss": 1.9957,
"step": 59
},
{
"epoch": 5.45,
"learning_rate": 2.1818181818181818e-05,
"loss": 1.9773,
"step": 60
},
{
"epoch": 5.45,
"eval_accuracy_NAT": 0.898956856290447,
"eval_accuracy_REST": 0.0,
"eval_accuracy_concrete": 0.24511379243744366,
"eval_accuracy_concrete tiles": 0.007154650522839846,
"eval_accuracy_dark bricks": 0.009314499057003194,
"eval_accuracy_grass": 0.07155464372966304,
"eval_accuracy_red bricks": 0.0,
"eval_accuracy_road": 0.7720496210944066,
"eval_accuracy_rough concrete": 0.0036621652552071412,
"eval_accuracy_rough red sidewalk": 0.010007443553055993,
"eval_accuracy_speedway bricks": 0.9787968531076106,
"eval_accuracy_steel": 0.0,
"eval_accuracy_tiles": 0.0009587727708533077,
"eval_accuracy_unlabeled": NaN,
"eval_iou_NAT": 0.89400043378223,
"eval_iou_REST": 0.0,
"eval_iou_concrete": 0.19197997304386832,
"eval_iou_concrete tiles": 0.0013976657786909726,
"eval_iou_dark bricks": 0.005338039042682255,
"eval_iou_grass": 0.07039652848458174,
"eval_iou_red bricks": 0.0,
"eval_iou_road": 0.5405843619796408,
"eval_iou_rough concrete": 0.0013066024253807521,
"eval_iou_rough red sidewalk": 0.003800927719548077,
"eval_iou_speedway bricks": 0.6998719868699171,
"eval_iou_steel": 0.0,
"eval_iou_tiles": 0.00034965034965034965,
"eval_iou_unlabeled": 0.0,
"eval_loss": 1.916090488433838,
"eval_mean_accuracy": 0.23058225367834848,
"eval_mean_iou": 0.1720732978197279,
"eval_overall_accuracy": 0.8010207282172309,
"eval_runtime": 3.5175,
"eval_samples_per_second": 10.235,
"eval_steps_per_second": 0.569,
"step": 60
},
{
"epoch": 5.55,
"learning_rate": 2.2181818181818184e-05,
"loss": 1.9281,
"step": 61
},
{
"epoch": 5.64,
"learning_rate": 2.2545454545454548e-05,
"loss": 1.8834,
"step": 62
},
{
"epoch": 5.73,
"learning_rate": 2.290909090909091e-05,
"loss": 1.8846,
"step": 63
},
{
"epoch": 5.82,
"learning_rate": 2.3272727272727274e-05,
"loss": 1.8532,
"step": 64
},
{
"epoch": 5.91,
"learning_rate": 2.3636363636363637e-05,
"loss": 1.8357,
"step": 65
},
{
"epoch": 6.0,
"learning_rate": 2.4e-05,
"loss": 1.8409,
"step": 66
},
{
"epoch": 6.09,
"learning_rate": 2.4363636363636364e-05,
"loss": 1.7691,
"step": 67
},
{
"epoch": 6.18,
"learning_rate": 2.472727272727273e-05,
"loss": 1.7549,
"step": 68
},
{
"epoch": 6.27,
"learning_rate": 2.5090909090909094e-05,
"loss": 1.8554,
"step": 69
},
{
"epoch": 6.36,
"learning_rate": 2.5454545454545454e-05,
"loss": 1.7217,
"step": 70
},
{
"epoch": 6.45,
"learning_rate": 2.581818181818182e-05,
"loss": 1.7575,
"step": 71
},
{
"epoch": 6.55,
"learning_rate": 2.6181818181818187e-05,
"loss": 1.7587,
"step": 72
},
{
"epoch": 6.64,
"learning_rate": 2.6545454545454547e-05,
"loss": 1.6981,
"step": 73
},
{
"epoch": 6.73,
"learning_rate": 2.6909090909090913e-05,
"loss": 1.6573,
"step": 74
},
{
"epoch": 6.82,
"learning_rate": 2.7272727272727273e-05,
"loss": 1.6428,
"step": 75
},
{
"epoch": 6.91,
"learning_rate": 2.7636363636363636e-05,
"loss": 1.6877,
"step": 76
},
{
"epoch": 7.0,
"learning_rate": 2.8000000000000003e-05,
"loss": 1.7226,
"step": 77
},
{
"epoch": 7.09,
"learning_rate": 2.8363636363636363e-05,
"loss": 1.6332,
"step": 78
},
{
"epoch": 7.18,
"learning_rate": 2.872727272727273e-05,
"loss": 1.6322,
"step": 79
},
{
"epoch": 7.27,
"learning_rate": 2.909090909090909e-05,
"loss": 1.6038,
"step": 80
},
{
"epoch": 7.27,
"eval_accuracy_NAT": 0.9470286690715778,
"eval_accuracy_REST": 0.0,
"eval_accuracy_concrete": 0.7309205867204601,
"eval_accuracy_concrete tiles": 0.0014064697609001407,
"eval_accuracy_dark bricks": 0.0,
"eval_accuracy_grass": 0.029533372711163616,
"eval_accuracy_red bricks": 0.015447443181818182,
"eval_accuracy_road": 0.8218001008480181,
"eval_accuracy_rough concrete": 0.0,
"eval_accuracy_rough red sidewalk": 0.0020400849116422684,
"eval_accuracy_speedway bricks": 0.9799789289994171,
"eval_accuracy_steel": 0.0,
"eval_accuracy_tiles": 0.0,
"eval_accuracy_unlabeled": NaN,
"eval_iou_NAT": 0.937594640128183,
"eval_iou_REST": 0.0,
"eval_iou_concrete": 0.4290493301504721,
"eval_iou_concrete tiles": 0.0011021395883748233,
"eval_iou_dark bricks": 0.0,
"eval_iou_grass": 0.029093488223162548,
"eval_iou_red bricks": 0.002611828279795857,
"eval_iou_road": 0.7049759862244951,
"eval_iou_rough concrete": 0.0,
"eval_iou_rough red sidewalk": 0.0009831927190593237,
"eval_iou_speedway bricks": 0.7986861621141007,
"eval_iou_steel": 0.0,
"eval_iou_tiles": 0.0,
"eval_iou_unlabeled": 0.0,
"eval_loss": 1.4904648065567017,
"eval_mean_accuracy": 0.27139658893884594,
"eval_mean_iou": 0.2074354833876888,
"eval_overall_accuracy": 0.8714875115288628,
"eval_runtime": 3.8608,
"eval_samples_per_second": 9.325,
"eval_steps_per_second": 0.518,
"step": 80
}
],
"max_steps": 5500,
"num_train_epochs": 500,
"total_flos": 4.478602205410099e+16,
"trial_name": null,
"trial_params": null
}