sam1120's picture
Training in progress, step 120
1f651ba
raw
history blame
21.9 kB
{
"best_metric": 0.25913122296333313,
"best_model_checkpoint": "/robodata/smodak/Projects/nspl/scripts/terrainseg/training/models/dropoff-utcustom-train-SF-RGB-b5_5/checkpoint-120",
"epoch": 60.0,
"global_step": 120,
"is_hyper_param_search": false,
"is_local_process_zero": true,
"is_world_process_zero": true,
"log_history": [
{
"epoch": 0.5,
"learning_rate": 7.5e-07,
"loss": 1.1306,
"step": 1
},
{
"epoch": 1.0,
"learning_rate": 1.5e-06,
"loss": 1.1189,
"step": 2
},
{
"epoch": 1.5,
"learning_rate": 2.25e-06,
"loss": 1.1168,
"step": 3
},
{
"epoch": 2.0,
"learning_rate": 3e-06,
"loss": 1.1199,
"step": 4
},
{
"epoch": 2.5,
"learning_rate": 3.75e-06,
"loss": 1.1175,
"step": 5
},
{
"epoch": 3.0,
"learning_rate": 4.5e-06,
"loss": 1.1046,
"step": 6
},
{
"epoch": 3.5,
"learning_rate": 5.2500000000000006e-06,
"loss": 1.0996,
"step": 7
},
{
"epoch": 4.0,
"learning_rate": 6e-06,
"loss": 1.0903,
"step": 8
},
{
"epoch": 4.5,
"learning_rate": 6.75e-06,
"loss": 1.0751,
"step": 9
},
{
"epoch": 5.0,
"learning_rate": 7.5e-06,
"loss": 1.0685,
"step": 10
},
{
"epoch": 5.0,
"eval_accuracy_dropoff": 0.12556710016284778,
"eval_accuracy_undropoff": 0.6194458181630027,
"eval_accuracy_unlabeled": NaN,
"eval_iou_dropoff": 0.04967425823428001,
"eval_iou_undropoff": 0.6069927004338553,
"eval_iou_unlabeled": 0.0,
"eval_loss": 1.0222245454788208,
"eval_mean_accuracy": 0.37250645916292524,
"eval_mean_iou": 0.21888898622271177,
"eval_overall_accuracy": 0.5989107131958008,
"eval_runtime": 2.0623,
"eval_samples_per_second": 9.698,
"eval_steps_per_second": 0.97,
"step": 10
},
{
"epoch": 5.5,
"learning_rate": 8.25e-06,
"loss": 1.0536,
"step": 11
},
{
"epoch": 6.0,
"learning_rate": 9e-06,
"loss": 1.0449,
"step": 12
},
{
"epoch": 6.5,
"learning_rate": 8.960526315789474e-06,
"loss": 1.0298,
"step": 13
},
{
"epoch": 7.0,
"learning_rate": 8.921052631578947e-06,
"loss": 1.0194,
"step": 14
},
{
"epoch": 7.5,
"learning_rate": 8.881578947368421e-06,
"loss": 1.0003,
"step": 15
},
{
"epoch": 8.0,
"learning_rate": 8.842105263157895e-06,
"loss": 0.9971,
"step": 16
},
{
"epoch": 8.5,
"learning_rate": 8.80263157894737e-06,
"loss": 0.9839,
"step": 17
},
{
"epoch": 9.0,
"learning_rate": 8.763157894736842e-06,
"loss": 0.9637,
"step": 18
},
{
"epoch": 9.5,
"learning_rate": 8.723684210526316e-06,
"loss": 0.9503,
"step": 19
},
{
"epoch": 10.0,
"learning_rate": 8.68421052631579e-06,
"loss": 0.9481,
"step": 20
},
{
"epoch": 10.0,
"eval_accuracy_dropoff": 0.41593614532443407,
"eval_accuracy_undropoff": 0.8637103137683748,
"eval_accuracy_unlabeled": NaN,
"eval_iou_dropoff": 0.26333490163277395,
"eval_iou_undropoff": 0.8475766394676805,
"eval_iou_unlabeled": 0.0,
"eval_loss": 0.8418876528739929,
"eval_mean_accuracy": 0.6398232295464045,
"eval_mean_iou": 0.3703038470334848,
"eval_overall_accuracy": 0.8450922012329102,
"eval_runtime": 2.1115,
"eval_samples_per_second": 9.472,
"eval_steps_per_second": 0.947,
"step": 20
},
{
"epoch": 10.5,
"learning_rate": 8.644736842105263e-06,
"loss": 0.9183,
"step": 21
},
{
"epoch": 11.0,
"learning_rate": 8.605263157894737e-06,
"loss": 0.9305,
"step": 22
},
{
"epoch": 11.5,
"learning_rate": 8.56578947368421e-06,
"loss": 0.9155,
"step": 23
},
{
"epoch": 12.0,
"learning_rate": 8.526315789473683e-06,
"loss": 0.8859,
"step": 24
},
{
"epoch": 12.5,
"learning_rate": 8.486842105263159e-06,
"loss": 0.8853,
"step": 25
},
{
"epoch": 13.0,
"learning_rate": 8.447368421052632e-06,
"loss": 0.8673,
"step": 26
},
{
"epoch": 13.5,
"learning_rate": 8.407894736842106e-06,
"loss": 0.8644,
"step": 27
},
{
"epoch": 14.0,
"learning_rate": 8.36842105263158e-06,
"loss": 0.8401,
"step": 28
},
{
"epoch": 14.5,
"learning_rate": 8.328947368421052e-06,
"loss": 0.836,
"step": 29
},
{
"epoch": 15.0,
"learning_rate": 8.289473684210526e-06,
"loss": 0.8268,
"step": 30
},
{
"epoch": 15.0,
"eval_accuracy_dropoff": 0.5023463840913782,
"eval_accuracy_undropoff": 0.8852725982783686,
"eval_accuracy_unlabeled": NaN,
"eval_iou_dropoff": 0.31364905698205614,
"eval_iou_undropoff": 0.8711414077560768,
"eval_iou_unlabeled": 0.0,
"eval_loss": 0.7165183424949646,
"eval_mean_accuracy": 0.6938094911848733,
"eval_mean_iou": 0.39493015491271094,
"eval_overall_accuracy": 0.869350814819336,
"eval_runtime": 2.2715,
"eval_samples_per_second": 8.805,
"eval_steps_per_second": 0.88,
"step": 30
},
{
"epoch": 15.5,
"learning_rate": 8.25e-06,
"loss": 0.8126,
"step": 31
},
{
"epoch": 16.0,
"learning_rate": 8.210526315789473e-06,
"loss": 0.8039,
"step": 32
},
{
"epoch": 16.5,
"learning_rate": 8.171052631578947e-06,
"loss": 0.8044,
"step": 33
},
{
"epoch": 17.0,
"learning_rate": 8.131578947368421e-06,
"loss": 0.7729,
"step": 34
},
{
"epoch": 17.5,
"learning_rate": 8.092105263157896e-06,
"loss": 0.7518,
"step": 35
},
{
"epoch": 18.0,
"learning_rate": 8.052631578947368e-06,
"loss": 0.7796,
"step": 36
},
{
"epoch": 18.5,
"learning_rate": 8.013157894736842e-06,
"loss": 0.7496,
"step": 37
},
{
"epoch": 19.0,
"learning_rate": 7.973684210526316e-06,
"loss": 0.7309,
"step": 38
},
{
"epoch": 19.5,
"learning_rate": 7.934210526315789e-06,
"loss": 0.7012,
"step": 39
},
{
"epoch": 20.0,
"learning_rate": 7.894736842105263e-06,
"loss": 0.7573,
"step": 40
},
{
"epoch": 20.0,
"eval_accuracy_dropoff": 0.5213972797541228,
"eval_accuracy_undropoff": 0.915831108572634,
"eval_accuracy_unlabeled": NaN,
"eval_iou_dropoff": 0.324261253088216,
"eval_iou_undropoff": 0.900986633054498,
"eval_iou_unlabeled": 0.0,
"eval_loss": 0.6206496953964233,
"eval_mean_accuracy": 0.7186141941633784,
"eval_mean_iou": 0.40841596204757136,
"eval_overall_accuracy": 0.8994308471679687,
"eval_runtime": 12.3072,
"eval_samples_per_second": 1.625,
"eval_steps_per_second": 0.163,
"step": 40
},
{
"epoch": 20.5,
"learning_rate": 7.855263157894737e-06,
"loss": 0.6878,
"step": 41
},
{
"epoch": 21.0,
"learning_rate": 7.815789473684211e-06,
"loss": 0.6981,
"step": 42
},
{
"epoch": 21.5,
"learning_rate": 7.776315789473685e-06,
"loss": 0.6609,
"step": 43
},
{
"epoch": 22.0,
"learning_rate": 7.736842105263158e-06,
"loss": 0.6934,
"step": 44
},
{
"epoch": 22.5,
"learning_rate": 7.697368421052632e-06,
"loss": 0.638,
"step": 45
},
{
"epoch": 23.0,
"learning_rate": 7.657894736842106e-06,
"loss": 0.6895,
"step": 46
},
{
"epoch": 23.5,
"learning_rate": 7.618421052631579e-06,
"loss": 0.6484,
"step": 47
},
{
"epoch": 24.0,
"learning_rate": 7.578947368421053e-06,
"loss": 0.6193,
"step": 48
},
{
"epoch": 24.5,
"learning_rate": 7.539473684210527e-06,
"loss": 0.6037,
"step": 49
},
{
"epoch": 25.0,
"learning_rate": 7.5e-06,
"loss": 0.636,
"step": 50
},
{
"epoch": 25.0,
"eval_accuracy_dropoff": 0.5020298630702539,
"eval_accuracy_undropoff": 0.9485253891382589,
"eval_accuracy_unlabeled": NaN,
"eval_iou_dropoff": 0.34010497757805724,
"eval_iou_undropoff": 0.9315542148635308,
"eval_iou_unlabeled": 0.0,
"eval_loss": 0.5194191932678223,
"eval_mean_accuracy": 0.7252776261042564,
"eval_mean_iou": 0.42388639748052936,
"eval_overall_accuracy": 0.9299604415893554,
"eval_runtime": 11.5699,
"eval_samples_per_second": 1.729,
"eval_steps_per_second": 0.173,
"step": 50
},
{
"epoch": 25.5,
"learning_rate": 7.460526315789474e-06,
"loss": 0.5948,
"step": 51
},
{
"epoch": 26.0,
"learning_rate": 7.421052631578948e-06,
"loss": 0.6004,
"step": 52
},
{
"epoch": 26.5,
"learning_rate": 7.381578947368421e-06,
"loss": 0.6018,
"step": 53
},
{
"epoch": 27.0,
"learning_rate": 7.342105263157895e-06,
"loss": 0.5598,
"step": 54
},
{
"epoch": 27.5,
"learning_rate": 7.302631578947368e-06,
"loss": 0.5497,
"step": 55
},
{
"epoch": 28.0,
"learning_rate": 7.263157894736842e-06,
"loss": 0.5621,
"step": 56
},
{
"epoch": 28.5,
"learning_rate": 7.223684210526317e-06,
"loss": 0.5271,
"step": 57
},
{
"epoch": 29.0,
"learning_rate": 7.18421052631579e-06,
"loss": 0.5719,
"step": 58
},
{
"epoch": 29.5,
"learning_rate": 7.144736842105263e-06,
"loss": 0.5332,
"step": 59
},
{
"epoch": 30.0,
"learning_rate": 7.1052631578947375e-06,
"loss": 0.5238,
"step": 60
},
{
"epoch": 30.0,
"eval_accuracy_dropoff": 0.5084657904997821,
"eval_accuracy_undropoff": 0.9650592998645741,
"eval_accuracy_unlabeled": NaN,
"eval_iou_dropoff": 0.3618276305566981,
"eval_iou_undropoff": 0.9476363475766674,
"eval_iou_unlabeled": 0.0,
"eval_loss": 0.4507386088371277,
"eval_mean_accuracy": 0.7367625451821781,
"eval_mean_iou": 0.4364879927111218,
"eval_overall_accuracy": 0.9460744857788086,
"eval_runtime": 2.0875,
"eval_samples_per_second": 9.581,
"eval_steps_per_second": 0.958,
"step": 60
},
{
"epoch": 30.5,
"learning_rate": 7.065789473684211e-06,
"loss": 0.5062,
"step": 61
},
{
"epoch": 31.0,
"learning_rate": 7.026315789473684e-06,
"loss": 0.5106,
"step": 62
},
{
"epoch": 31.5,
"learning_rate": 6.986842105263158e-06,
"loss": 0.504,
"step": 63
},
{
"epoch": 32.0,
"learning_rate": 6.9473684210526315e-06,
"loss": 0.4815,
"step": 64
},
{
"epoch": 32.5,
"learning_rate": 6.907894736842106e-06,
"loss": 0.4846,
"step": 65
},
{
"epoch": 33.0,
"learning_rate": 6.86842105263158e-06,
"loss": 0.4722,
"step": 66
},
{
"epoch": 33.5,
"learning_rate": 6.828947368421053e-06,
"loss": 0.4739,
"step": 67
},
{
"epoch": 34.0,
"learning_rate": 6.7894736842105264e-06,
"loss": 0.4559,
"step": 68
},
{
"epoch": 34.5,
"learning_rate": 6.75e-06,
"loss": 0.4591,
"step": 69
},
{
"epoch": 35.0,
"learning_rate": 6.710526315789474e-06,
"loss": 0.4296,
"step": 70
},
{
"epoch": 35.0,
"eval_accuracy_dropoff": 0.5123007408426798,
"eval_accuracy_undropoff": 0.972096276830216,
"eval_accuracy_unlabeled": NaN,
"eval_iou_dropoff": 0.3682749159931278,
"eval_iou_undropoff": 0.9546251911332915,
"eval_iou_unlabeled": 0.0,
"eval_loss": 0.4064384400844574,
"eval_mean_accuracy": 0.742198508836448,
"eval_mean_iou": 0.4409667023754731,
"eval_overall_accuracy": 0.9529783248901367,
"eval_runtime": 2.1231,
"eval_samples_per_second": 9.42,
"eval_steps_per_second": 0.942,
"step": 70
},
{
"epoch": 35.5,
"learning_rate": 6.671052631578947e-06,
"loss": 0.4272,
"step": 71
},
{
"epoch": 36.0,
"learning_rate": 6.6315789473684205e-06,
"loss": 0.4376,
"step": 72
},
{
"epoch": 36.5,
"learning_rate": 6.592105263157895e-06,
"loss": 0.4097,
"step": 73
},
{
"epoch": 37.0,
"learning_rate": 6.552631578947369e-06,
"loss": 0.4961,
"step": 74
},
{
"epoch": 37.5,
"learning_rate": 6.513157894736842e-06,
"loss": 0.4194,
"step": 75
},
{
"epoch": 38.0,
"learning_rate": 6.473684210526316e-06,
"loss": 0.398,
"step": 76
},
{
"epoch": 38.5,
"learning_rate": 6.4342105263157896e-06,
"loss": 0.4193,
"step": 77
},
{
"epoch": 39.0,
"learning_rate": 6.394736842105263e-06,
"loss": 0.3805,
"step": 78
},
{
"epoch": 39.5,
"learning_rate": 6.355263157894737e-06,
"loss": 0.3684,
"step": 79
},
{
"epoch": 40.0,
"learning_rate": 6.31578947368421e-06,
"loss": 0.4105,
"step": 80
},
{
"epoch": 40.0,
"eval_accuracy_dropoff": 0.5119979816050827,
"eval_accuracy_undropoff": 0.9813971862042614,
"eval_accuracy_unlabeled": NaN,
"eval_iou_dropoff": 0.3879856642785385,
"eval_iou_undropoff": 0.9627011968835347,
"eval_iou_unlabeled": 0.0,
"eval_loss": 0.3546861410140991,
"eval_mean_accuracy": 0.7466975839046721,
"eval_mean_iou": 0.4502289537206911,
"eval_overall_accuracy": 0.9618799209594726,
"eval_runtime": 2.8717,
"eval_samples_per_second": 6.965,
"eval_steps_per_second": 0.696,
"step": 80
},
{
"epoch": 40.5,
"learning_rate": 6.276315789473684e-06,
"loss": 0.3494,
"step": 81
},
{
"epoch": 41.0,
"learning_rate": 6.236842105263159e-06,
"loss": 0.3909,
"step": 82
},
{
"epoch": 41.5,
"learning_rate": 6.197368421052632e-06,
"loss": 0.3787,
"step": 83
},
{
"epoch": 42.0,
"learning_rate": 6.157894736842105e-06,
"loss": 0.3625,
"step": 84
},
{
"epoch": 42.5,
"learning_rate": 6.118421052631579e-06,
"loss": 0.3586,
"step": 85
},
{
"epoch": 43.0,
"learning_rate": 6.078947368421053e-06,
"loss": 0.3607,
"step": 86
},
{
"epoch": 43.5,
"learning_rate": 6.039473684210526e-06,
"loss": 0.3254,
"step": 87
},
{
"epoch": 44.0,
"learning_rate": 6e-06,
"loss": 0.3652,
"step": 88
},
{
"epoch": 44.5,
"learning_rate": 5.9605263157894735e-06,
"loss": 0.347,
"step": 89
},
{
"epoch": 45.0,
"learning_rate": 5.921052631578948e-06,
"loss": 0.3436,
"step": 90
},
{
"epoch": 45.0,
"eval_accuracy_dropoff": 0.5361315626505195,
"eval_accuracy_undropoff": 0.9829765258309394,
"eval_accuracy_unlabeled": NaN,
"eval_iou_dropoff": 0.40656917040046753,
"eval_iou_undropoff": 0.9647043198712515,
"eval_iou_unlabeled": 0.0,
"eval_loss": 0.33038827776908875,
"eval_mean_accuracy": 0.7595540442407295,
"eval_mean_iou": 0.4570911634239063,
"eval_overall_accuracy": 0.9643970489501953,
"eval_runtime": 2.1187,
"eval_samples_per_second": 9.44,
"eval_steps_per_second": 0.944,
"step": 90
},
{
"epoch": 45.5,
"learning_rate": 5.881578947368422e-06,
"loss": 0.326,
"step": 91
},
{
"epoch": 46.0,
"learning_rate": 5.842105263157895e-06,
"loss": 0.3193,
"step": 92
},
{
"epoch": 46.5,
"learning_rate": 5.802631578947368e-06,
"loss": 0.3408,
"step": 93
},
{
"epoch": 47.0,
"learning_rate": 5.7631578947368425e-06,
"loss": 0.2914,
"step": 94
},
{
"epoch": 47.5,
"learning_rate": 5.723684210526316e-06,
"loss": 0.3105,
"step": 95
},
{
"epoch": 48.0,
"learning_rate": 5.684210526315789e-06,
"loss": 0.3058,
"step": 96
},
{
"epoch": 48.5,
"learning_rate": 5.644736842105263e-06,
"loss": 0.2905,
"step": 97
},
{
"epoch": 49.0,
"learning_rate": 5.6052631578947374e-06,
"loss": 0.307,
"step": 98
},
{
"epoch": 49.5,
"learning_rate": 5.565789473684211e-06,
"loss": 0.3005,
"step": 99
},
{
"epoch": 50.0,
"learning_rate": 5.526315789473685e-06,
"loss": 0.2729,
"step": 100
},
{
"epoch": 50.0,
"eval_accuracy_dropoff": 0.5231817243514759,
"eval_accuracy_undropoff": 0.9872727435553251,
"eval_accuracy_unlabeled": NaN,
"eval_iou_dropoff": 0.41634177326090765,
"eval_iou_undropoff": 0.9677673081732493,
"eval_iou_unlabeled": 0.0,
"eval_loss": 0.29534655809402466,
"eval_mean_accuracy": 0.7552272339534005,
"eval_mean_iou": 0.46136969381138565,
"eval_overall_accuracy": 0.967976188659668,
"eval_runtime": 2.1109,
"eval_samples_per_second": 9.475,
"eval_steps_per_second": 0.947,
"step": 100
},
{
"epoch": 50.5,
"learning_rate": 5.486842105263158e-06,
"loss": 0.2812,
"step": 101
},
{
"epoch": 51.0,
"learning_rate": 5.4473684210526315e-06,
"loss": 0.2825,
"step": 102
},
{
"epoch": 51.5,
"learning_rate": 5.407894736842106e-06,
"loss": 0.266,
"step": 103
},
{
"epoch": 52.0,
"learning_rate": 5.368421052631579e-06,
"loss": 0.294,
"step": 104
},
{
"epoch": 52.5,
"learning_rate": 5.328947368421052e-06,
"loss": 0.2774,
"step": 105
},
{
"epoch": 53.0,
"learning_rate": 5.289473684210527e-06,
"loss": 0.2546,
"step": 106
},
{
"epoch": 53.5,
"learning_rate": 5.2500000000000006e-06,
"loss": 0.26,
"step": 107
},
{
"epoch": 54.0,
"learning_rate": 5.210526315789474e-06,
"loss": 0.253,
"step": 108
},
{
"epoch": 54.5,
"learning_rate": 5.171052631578947e-06,
"loss": 0.2597,
"step": 109
},
{
"epoch": 55.0,
"learning_rate": 5.131578947368421e-06,
"loss": 0.2546,
"step": 110
},
{
"epoch": 55.0,
"eval_accuracy_dropoff": 0.5275900823413381,
"eval_accuracy_undropoff": 0.9882373427451574,
"eval_accuracy_unlabeled": NaN,
"eval_iou_dropoff": 0.42014283365906224,
"eval_iou_undropoff": 0.9685616940402523,
"eval_iou_unlabeled": 0.0,
"eval_loss": 0.27704566717147827,
"eval_mean_accuracy": 0.7579137125432478,
"eval_mean_iou": 0.46290150923310486,
"eval_overall_accuracy": 0.9690839767456054,
"eval_runtime": 2.0931,
"eval_samples_per_second": 9.555,
"eval_steps_per_second": 0.956,
"step": 110
},
{
"epoch": 55.5,
"learning_rate": 5.092105263157895e-06,
"loss": 0.2616,
"step": 111
},
{
"epoch": 56.0,
"learning_rate": 5.052631578947368e-06,
"loss": 0.246,
"step": 112
},
{
"epoch": 56.5,
"learning_rate": 5.013157894736842e-06,
"loss": 0.2653,
"step": 113
},
{
"epoch": 57.0,
"learning_rate": 4.973684210526316e-06,
"loss": 0.2223,
"step": 114
},
{
"epoch": 57.5,
"learning_rate": 4.9342105263157895e-06,
"loss": 0.2316,
"step": 115
},
{
"epoch": 58.0,
"learning_rate": 4.894736842105264e-06,
"loss": 0.2684,
"step": 116
},
{
"epoch": 58.5,
"learning_rate": 4.855263157894737e-06,
"loss": 0.2447,
"step": 117
},
{
"epoch": 59.0,
"learning_rate": 4.81578947368421e-06,
"loss": 0.2209,
"step": 118
},
{
"epoch": 59.5,
"learning_rate": 4.7763157894736844e-06,
"loss": 0.2294,
"step": 119
},
{
"epoch": 60.0,
"learning_rate": 4.736842105263158e-06,
"loss": 0.2281,
"step": 120
},
{
"epoch": 60.0,
"eval_accuracy_dropoff": 0.5235303562008303,
"eval_accuracy_undropoff": 0.9895947867463634,
"eval_accuracy_unlabeled": NaN,
"eval_iou_dropoff": 0.42446118084611806,
"eval_iou_undropoff": 0.9695980338955686,
"eval_iou_unlabeled": 0.0,
"eval_loss": 0.25913122296333313,
"eval_mean_accuracy": 0.7565625714735968,
"eval_mean_iou": 0.4646864049138955,
"eval_overall_accuracy": 0.970216178894043,
"eval_runtime": 4.2351,
"eval_samples_per_second": 4.722,
"eval_steps_per_second": 0.472,
"step": 120
}
],
"max_steps": 240,
"num_train_epochs": 120,
"total_flos": 6.945599099555021e+17,
"trial_name": null,
"trial_params": null
}