sam1120's picture
Training in progress, step 120
dfd7d29
{
"best_metric": 0.42677026987075806,
"best_model_checkpoint": "/robodata/smodak/Projects/nspl/scripts/terrainseg/training/models/dropoff-utcustom-train-SF-RGBD-b0_3/checkpoint-120",
"epoch": 60.0,
"global_step": 120,
"is_hyper_param_search": false,
"is_local_process_zero": true,
"is_world_process_zero": true,
"log_history": [
{
"epoch": 0.5,
"learning_rate": 3.3333333333333333e-06,
"loss": 1.0931,
"step": 1
},
{
"epoch": 1.0,
"learning_rate": 6.666666666666667e-06,
"loss": 1.0897,
"step": 2
},
{
"epoch": 1.5,
"learning_rate": 1e-05,
"loss": 1.0991,
"step": 3
},
{
"epoch": 2.0,
"learning_rate": 1.3333333333333333e-05,
"loss": 1.0969,
"step": 4
},
{
"epoch": 2.5,
"learning_rate": 1.6666666666666667e-05,
"loss": 1.0911,
"step": 5
},
{
"epoch": 3.0,
"learning_rate": 2e-05,
"loss": 1.0789,
"step": 6
},
{
"epoch": 3.5,
"learning_rate": 2.3333333333333336e-05,
"loss": 1.0635,
"step": 7
},
{
"epoch": 4.0,
"learning_rate": 2.6666666666666667e-05,
"loss": 1.0651,
"step": 8
},
{
"epoch": 4.5,
"learning_rate": 3.0000000000000004e-05,
"loss": 1.0608,
"step": 9
},
{
"epoch": 5.0,
"learning_rate": 3.3333333333333335e-05,
"loss": 1.0352,
"step": 10
},
{
"epoch": 5.0,
"eval_accuracy_dropoff": 0.428574049863529,
"eval_accuracy_undropoff": 0.7265983201605609,
"eval_accuracy_unlabeled": NaN,
"eval_iou_dropoff": 0.058895022539432185,
"eval_iou_undropoff": 0.7090215213149824,
"eval_iou_unlabeled": 0.0,
"eval_loss": 1.0675508975982666,
"eval_mean_accuracy": 0.577586185012045,
"eval_mean_iou": 0.25597218128480487,
"eval_overall_accuracy": 0.7142066955566406,
"eval_runtime": 2.8372,
"eval_samples_per_second": 7.049,
"eval_steps_per_second": 0.705,
"step": 10
},
{
"epoch": 5.5,
"learning_rate": 3.6666666666666666e-05,
"loss": 1.037,
"step": 11
},
{
"epoch": 6.0,
"learning_rate": 4e-05,
"loss": 1.0319,
"step": 12
},
{
"epoch": 6.5,
"learning_rate": 3.9824561403508774e-05,
"loss": 1.0252,
"step": 13
},
{
"epoch": 7.0,
"learning_rate": 3.9649122807017545e-05,
"loss": 1.0074,
"step": 14
},
{
"epoch": 7.5,
"learning_rate": 3.9473684210526316e-05,
"loss": 0.9873,
"step": 15
},
{
"epoch": 8.0,
"learning_rate": 3.929824561403509e-05,
"loss": 0.9838,
"step": 16
},
{
"epoch": 8.5,
"learning_rate": 3.9122807017543865e-05,
"loss": 0.9714,
"step": 17
},
{
"epoch": 9.0,
"learning_rate": 3.8947368421052636e-05,
"loss": 0.9773,
"step": 18
},
{
"epoch": 9.5,
"learning_rate": 3.877192982456141e-05,
"loss": 0.9487,
"step": 19
},
{
"epoch": 10.0,
"learning_rate": 3.859649122807018e-05,
"loss": 0.9564,
"step": 20
},
{
"epoch": 10.0,
"eval_accuracy_dropoff": 0.15711828252941581,
"eval_accuracy_undropoff": 0.9581383454546721,
"eval_accuracy_unlabeled": NaN,
"eval_iou_dropoff": 0.08219483807489711,
"eval_iou_undropoff": 0.9243382471663849,
"eval_iou_unlabeled": 0.0,
"eval_loss": 0.9742695093154907,
"eval_mean_accuracy": 0.5576283139920439,
"eval_mean_iou": 0.3355110284137606,
"eval_overall_accuracy": 0.924832534790039,
"eval_runtime": 2.8897,
"eval_samples_per_second": 6.921,
"eval_steps_per_second": 0.692,
"step": 20
},
{
"epoch": 10.5,
"learning_rate": 3.842105263157895e-05,
"loss": 0.9386,
"step": 21
},
{
"epoch": 11.0,
"learning_rate": 3.824561403508772e-05,
"loss": 0.9329,
"step": 22
},
{
"epoch": 11.5,
"learning_rate": 3.807017543859649e-05,
"loss": 0.923,
"step": 23
},
{
"epoch": 12.0,
"learning_rate": 3.789473684210526e-05,
"loss": 0.9097,
"step": 24
},
{
"epoch": 12.5,
"learning_rate": 3.771929824561404e-05,
"loss": 0.9109,
"step": 25
},
{
"epoch": 13.0,
"learning_rate": 3.754385964912281e-05,
"loss": 0.8933,
"step": 26
},
{
"epoch": 13.5,
"learning_rate": 3.736842105263158e-05,
"loss": 0.8754,
"step": 27
},
{
"epoch": 14.0,
"learning_rate": 3.719298245614035e-05,
"loss": 0.8954,
"step": 28
},
{
"epoch": 14.5,
"learning_rate": 3.7017543859649124e-05,
"loss": 0.874,
"step": 29
},
{
"epoch": 15.0,
"learning_rate": 3.6842105263157895e-05,
"loss": 0.8577,
"step": 30
},
{
"epoch": 15.0,
"eval_accuracy_dropoff": 0.07822656482946856,
"eval_accuracy_undropoff": 0.9783656740403014,
"eval_accuracy_unlabeled": NaN,
"eval_iou_dropoff": 0.05451029280143204,
"eval_iou_undropoff": 0.9407458379314891,
"eval_iou_unlabeled": 0.0,
"eval_loss": 0.8504301905632019,
"eval_mean_accuracy": 0.528296119434885,
"eval_mean_iou": 0.3317520435776404,
"eval_overall_accuracy": 0.9409385681152344,
"eval_runtime": 2.8978,
"eval_samples_per_second": 6.902,
"eval_steps_per_second": 0.69,
"step": 30
},
{
"epoch": 15.5,
"learning_rate": 3.6666666666666666e-05,
"loss": 0.859,
"step": 31
},
{
"epoch": 16.0,
"learning_rate": 3.649122807017544e-05,
"loss": 0.8646,
"step": 32
},
{
"epoch": 16.5,
"learning_rate": 3.6315789473684214e-05,
"loss": 0.8326,
"step": 33
},
{
"epoch": 17.0,
"learning_rate": 3.6140350877192985e-05,
"loss": 0.8148,
"step": 34
},
{
"epoch": 17.5,
"learning_rate": 3.5964912280701756e-05,
"loss": 0.8272,
"step": 35
},
{
"epoch": 18.0,
"learning_rate": 3.578947368421053e-05,
"loss": 0.7985,
"step": 36
},
{
"epoch": 18.5,
"learning_rate": 3.56140350877193e-05,
"loss": 0.7725,
"step": 37
},
{
"epoch": 19.0,
"learning_rate": 3.543859649122807e-05,
"loss": 0.8029,
"step": 38
},
{
"epoch": 19.5,
"learning_rate": 3.526315789473684e-05,
"loss": 0.7959,
"step": 39
},
{
"epoch": 20.0,
"learning_rate": 3.508771929824562e-05,
"loss": 0.7512,
"step": 40
},
{
"epoch": 20.0,
"eval_accuracy_dropoff": 0.031794307208880934,
"eval_accuracy_undropoff": 0.9926193335767883,
"eval_accuracy_unlabeled": NaN,
"eval_iou_dropoff": 0.028292336454702055,
"eval_iou_undropoff": 0.9526062992592174,
"eval_iou_unlabeled": 0.0,
"eval_loss": 0.6972397565841675,
"eval_mean_accuracy": 0.5122068203928346,
"eval_mean_iou": 0.3269662119046398,
"eval_overall_accuracy": 0.9526689529418946,
"eval_runtime": 2.8239,
"eval_samples_per_second": 7.082,
"eval_steps_per_second": 0.708,
"step": 40
},
{
"epoch": 20.5,
"learning_rate": 3.491228070175439e-05,
"loss": 0.791,
"step": 41
},
{
"epoch": 21.0,
"learning_rate": 3.473684210526316e-05,
"loss": 0.7408,
"step": 42
},
{
"epoch": 21.5,
"learning_rate": 3.456140350877193e-05,
"loss": 0.7873,
"step": 43
},
{
"epoch": 22.0,
"learning_rate": 3.43859649122807e-05,
"loss": 0.7602,
"step": 44
},
{
"epoch": 22.5,
"learning_rate": 3.421052631578947e-05,
"loss": 0.7512,
"step": 45
},
{
"epoch": 23.0,
"learning_rate": 3.403508771929825e-05,
"loss": 0.7277,
"step": 46
},
{
"epoch": 23.5,
"learning_rate": 3.385964912280702e-05,
"loss": 0.7186,
"step": 47
},
{
"epoch": 24.0,
"learning_rate": 3.368421052631579e-05,
"loss": 0.7219,
"step": 48
},
{
"epoch": 24.5,
"learning_rate": 3.3508771929824564e-05,
"loss": 0.7071,
"step": 49
},
{
"epoch": 25.0,
"learning_rate": 3.3333333333333335e-05,
"loss": 0.6955,
"step": 50
},
{
"epoch": 25.0,
"eval_accuracy_dropoff": 0.0249501135347141,
"eval_accuracy_undropoff": 0.9947805770679329,
"eval_accuracy_unlabeled": NaN,
"eval_iou_dropoff": 0.02343941907819604,
"eval_iou_undropoff": 0.9544097357599161,
"eval_iou_unlabeled": 0.0,
"eval_loss": 0.5760915279388428,
"eval_mean_accuracy": 0.5098653453013234,
"eval_mean_iou": 0.3259497182793707,
"eval_overall_accuracy": 0.9544557571411133,
"eval_runtime": 2.8981,
"eval_samples_per_second": 6.901,
"eval_steps_per_second": 0.69,
"step": 50
},
{
"epoch": 25.5,
"learning_rate": 3.3157894736842106e-05,
"loss": 0.6941,
"step": 51
},
{
"epoch": 26.0,
"learning_rate": 3.298245614035088e-05,
"loss": 0.6858,
"step": 52
},
{
"epoch": 26.5,
"learning_rate": 3.2807017543859655e-05,
"loss": 0.6836,
"step": 53
},
{
"epoch": 27.0,
"learning_rate": 3.2631578947368426e-05,
"loss": 0.6738,
"step": 54
},
{
"epoch": 27.5,
"learning_rate": 3.24561403508772e-05,
"loss": 0.6485,
"step": 55
},
{
"epoch": 28.0,
"learning_rate": 3.228070175438597e-05,
"loss": 0.6463,
"step": 56
},
{
"epoch": 28.5,
"learning_rate": 3.210526315789474e-05,
"loss": 0.6632,
"step": 57
},
{
"epoch": 29.0,
"learning_rate": 3.192982456140351e-05,
"loss": 0.6648,
"step": 58
},
{
"epoch": 29.5,
"learning_rate": 3.175438596491229e-05,
"loss": 0.6512,
"step": 59
},
{
"epoch": 30.0,
"learning_rate": 3.157894736842106e-05,
"loss": 0.6691,
"step": 60
},
{
"epoch": 30.0,
"eval_accuracy_dropoff": 0.06316199912842038,
"eval_accuracy_undropoff": 0.991065666179425,
"eval_accuracy_unlabeled": NaN,
"eval_iou_dropoff": 0.05572689007608871,
"eval_iou_undropoff": 0.9523936200221805,
"eval_iou_unlabeled": 0.0,
"eval_loss": 0.520889163017273,
"eval_mean_accuracy": 0.5271138326539228,
"eval_mean_iou": 0.33604017003275644,
"eval_overall_accuracy": 0.952484130859375,
"eval_runtime": 2.9738,
"eval_samples_per_second": 6.725,
"eval_steps_per_second": 0.673,
"step": 60
},
{
"epoch": 30.5,
"learning_rate": 3.140350877192983e-05,
"loss": 0.655,
"step": 61
},
{
"epoch": 31.0,
"learning_rate": 3.12280701754386e-05,
"loss": 0.6027,
"step": 62
},
{
"epoch": 31.5,
"learning_rate": 3.105263157894737e-05,
"loss": 0.652,
"step": 63
},
{
"epoch": 32.0,
"learning_rate": 3.087719298245614e-05,
"loss": 0.6229,
"step": 64
},
{
"epoch": 32.5,
"learning_rate": 3.070175438596492e-05,
"loss": 0.5883,
"step": 65
},
{
"epoch": 33.0,
"learning_rate": 3.052631578947369e-05,
"loss": 0.5846,
"step": 66
},
{
"epoch": 33.5,
"learning_rate": 3.035087719298246e-05,
"loss": 0.6157,
"step": 67
},
{
"epoch": 34.0,
"learning_rate": 3.017543859649123e-05,
"loss": 0.6156,
"step": 68
},
{
"epoch": 34.5,
"learning_rate": 3.0000000000000004e-05,
"loss": 0.6153,
"step": 69
},
{
"epoch": 35.0,
"learning_rate": 2.9824561403508775e-05,
"loss": 0.626,
"step": 70
},
{
"epoch": 35.0,
"eval_accuracy_dropoff": 0.08440101837198101,
"eval_accuracy_undropoff": 0.9880552490255996,
"eval_accuracy_unlabeled": NaN,
"eval_iou_dropoff": 0.07191744679187757,
"eval_iou_undropoff": 0.9503456175329309,
"eval_iou_unlabeled": 0.0,
"eval_loss": 0.5296640992164612,
"eval_mean_accuracy": 0.5362281336987903,
"eval_mean_iou": 0.34075435477493615,
"eval_overall_accuracy": 0.9504819869995117,
"eval_runtime": 2.9057,
"eval_samples_per_second": 6.883,
"eval_steps_per_second": 0.688,
"step": 70
},
{
"epoch": 35.5,
"learning_rate": 2.9649122807017546e-05,
"loss": 0.5973,
"step": 71
},
{
"epoch": 36.0,
"learning_rate": 2.9473684210526317e-05,
"loss": 0.6262,
"step": 72
},
{
"epoch": 36.5,
"learning_rate": 2.9298245614035088e-05,
"loss": 0.6077,
"step": 73
},
{
"epoch": 37.0,
"learning_rate": 2.9122807017543863e-05,
"loss": 0.5391,
"step": 74
},
{
"epoch": 37.5,
"learning_rate": 2.8947368421052634e-05,
"loss": 0.5766,
"step": 75
},
{
"epoch": 38.0,
"learning_rate": 2.8771929824561408e-05,
"loss": 0.5623,
"step": 76
},
{
"epoch": 38.5,
"learning_rate": 2.859649122807018e-05,
"loss": 0.6004,
"step": 77
},
{
"epoch": 39.0,
"learning_rate": 2.842105263157895e-05,
"loss": 0.5805,
"step": 78
},
{
"epoch": 39.5,
"learning_rate": 2.824561403508772e-05,
"loss": 0.5391,
"step": 79
},
{
"epoch": 40.0,
"learning_rate": 2.8070175438596492e-05,
"loss": 0.5544,
"step": 80
},
{
"epoch": 40.0,
"eval_accuracy_dropoff": 0.16521938576572856,
"eval_accuracy_undropoff": 0.9862325207442558,
"eval_accuracy_unlabeled": NaN,
"eval_iou_dropoff": 0.13295361001701742,
"eval_iou_undropoff": 0.9517702617082298,
"eval_iou_unlabeled": 0.0,
"eval_loss": 0.5263115167617798,
"eval_mean_accuracy": 0.5757259532549922,
"eval_mean_iou": 0.3615746239084157,
"eval_overall_accuracy": 0.9520954132080078,
"eval_runtime": 2.9775,
"eval_samples_per_second": 6.717,
"eval_steps_per_second": 0.672,
"step": 80
},
{
"epoch": 40.5,
"learning_rate": 2.7894736842105263e-05,
"loss": 0.5685,
"step": 81
},
{
"epoch": 41.0,
"learning_rate": 2.771929824561404e-05,
"loss": 0.5441,
"step": 82
},
{
"epoch": 41.5,
"learning_rate": 2.754385964912281e-05,
"loss": 0.545,
"step": 83
},
{
"epoch": 42.0,
"learning_rate": 2.7368421052631583e-05,
"loss": 0.5239,
"step": 84
},
{
"epoch": 42.5,
"learning_rate": 2.7192982456140354e-05,
"loss": 0.5411,
"step": 85
},
{
"epoch": 43.0,
"learning_rate": 2.7017543859649125e-05,
"loss": 0.5304,
"step": 86
},
{
"epoch": 43.5,
"learning_rate": 2.6842105263157896e-05,
"loss": 0.5623,
"step": 87
},
{
"epoch": 44.0,
"learning_rate": 2.6666666666666667e-05,
"loss": 0.5373,
"step": 88
},
{
"epoch": 44.5,
"learning_rate": 2.6491228070175438e-05,
"loss": 0.5685,
"step": 89
},
{
"epoch": 45.0,
"learning_rate": 2.6315789473684215e-05,
"loss": 0.5316,
"step": 90
},
{
"epoch": 45.0,
"eval_accuracy_dropoff": 0.291483749627285,
"eval_accuracy_undropoff": 0.9791696725397696,
"eval_accuracy_unlabeled": NaN,
"eval_iou_dropoff": 0.20086552169968483,
"eval_iou_undropoff": 0.9499704112593605,
"eval_iou_unlabeled": 0.0,
"eval_loss": 0.48249512910842896,
"eval_mean_accuracy": 0.6353267110835273,
"eval_mean_iou": 0.38361197765301513,
"eval_overall_accuracy": 0.9505762100219727,
"eval_runtime": 2.8321,
"eval_samples_per_second": 7.062,
"eval_steps_per_second": 0.706,
"step": 90
},
{
"epoch": 45.5,
"learning_rate": 2.6140350877192986e-05,
"loss": 0.5093,
"step": 91
},
{
"epoch": 46.0,
"learning_rate": 2.5964912280701757e-05,
"loss": 0.5316,
"step": 92
},
{
"epoch": 46.5,
"learning_rate": 2.578947368421053e-05,
"loss": 0.5125,
"step": 93
},
{
"epoch": 47.0,
"learning_rate": 2.56140350877193e-05,
"loss": 0.4953,
"step": 94
},
{
"epoch": 47.5,
"learning_rate": 2.543859649122807e-05,
"loss": 0.5341,
"step": 95
},
{
"epoch": 48.0,
"learning_rate": 2.526315789473684e-05,
"loss": 0.4951,
"step": 96
},
{
"epoch": 48.5,
"learning_rate": 2.5087719298245616e-05,
"loss": 0.5089,
"step": 97
},
{
"epoch": 49.0,
"learning_rate": 2.491228070175439e-05,
"loss": 0.4924,
"step": 98
},
{
"epoch": 49.5,
"learning_rate": 2.473684210526316e-05,
"loss": 0.4839,
"step": 99
},
{
"epoch": 50.0,
"learning_rate": 2.4561403508771932e-05,
"loss": 0.4929,
"step": 100
},
{
"epoch": 50.0,
"eval_accuracy_dropoff": 0.3378059129796555,
"eval_accuracy_undropoff": 0.9797145606317359,
"eval_accuracy_unlabeled": NaN,
"eval_iou_dropoff": 0.23516487727612392,
"eval_iou_undropoff": 0.9523556470025176,
"eval_iou_unlabeled": 0.0,
"eval_loss": 0.4762709140777588,
"eval_mean_accuracy": 0.6587602368056957,
"eval_mean_iou": 0.3958401747595472,
"eval_overall_accuracy": 0.9530244827270508,
"eval_runtime": 2.8553,
"eval_samples_per_second": 7.005,
"eval_steps_per_second": 0.7,
"step": 100
},
{
"epoch": 50.5,
"learning_rate": 2.4385964912280703e-05,
"loss": 0.5066,
"step": 101
},
{
"epoch": 51.0,
"learning_rate": 2.4210526315789474e-05,
"loss": 0.5068,
"step": 102
},
{
"epoch": 51.5,
"learning_rate": 2.403508771929825e-05,
"loss": 0.4604,
"step": 103
},
{
"epoch": 52.0,
"learning_rate": 2.385964912280702e-05,
"loss": 0.4837,
"step": 104
},
{
"epoch": 52.5,
"learning_rate": 2.368421052631579e-05,
"loss": 0.477,
"step": 105
},
{
"epoch": 53.0,
"learning_rate": 2.3508771929824565e-05,
"loss": 0.461,
"step": 106
},
{
"epoch": 53.5,
"learning_rate": 2.3333333333333336e-05,
"loss": 0.4595,
"step": 107
},
{
"epoch": 54.0,
"learning_rate": 2.3157894736842107e-05,
"loss": 0.5052,
"step": 108
},
{
"epoch": 54.5,
"learning_rate": 2.2982456140350878e-05,
"loss": 0.4706,
"step": 109
},
{
"epoch": 55.0,
"learning_rate": 2.2807017543859652e-05,
"loss": 0.468,
"step": 110
},
{
"epoch": 55.0,
"eval_accuracy_dropoff": 0.4188307071263102,
"eval_accuracy_undropoff": 0.9759254191886978,
"eval_accuracy_unlabeled": NaN,
"eval_iou_dropoff": 0.2712588015092546,
"eval_iou_undropoff": 0.9519246242461289,
"eval_iou_unlabeled": 0.0,
"eval_loss": 0.4583040773868561,
"eval_mean_accuracy": 0.697378063157504,
"eval_mean_iou": 0.40772780858512786,
"eval_overall_accuracy": 0.9527618408203125,
"eval_runtime": 2.9198,
"eval_samples_per_second": 6.85,
"eval_steps_per_second": 0.685,
"step": 110
},
{
"epoch": 55.5,
"learning_rate": 2.2631578947368423e-05,
"loss": 0.4494,
"step": 111
},
{
"epoch": 56.0,
"learning_rate": 2.2456140350877194e-05,
"loss": 0.439,
"step": 112
},
{
"epoch": 56.5,
"learning_rate": 2.2280701754385965e-05,
"loss": 0.4833,
"step": 113
},
{
"epoch": 57.0,
"learning_rate": 2.210526315789474e-05,
"loss": 0.4272,
"step": 114
},
{
"epoch": 57.5,
"learning_rate": 2.192982456140351e-05,
"loss": 0.4596,
"step": 115
},
{
"epoch": 58.0,
"learning_rate": 2.1754385964912285e-05,
"loss": 0.441,
"step": 116
},
{
"epoch": 58.5,
"learning_rate": 2.1578947368421056e-05,
"loss": 0.4488,
"step": 117
},
{
"epoch": 59.0,
"learning_rate": 2.1403508771929827e-05,
"loss": 0.4742,
"step": 118
},
{
"epoch": 59.5,
"learning_rate": 2.1228070175438598e-05,
"loss": 0.4408,
"step": 119
},
{
"epoch": 60.0,
"learning_rate": 2.105263157894737e-05,
"loss": 0.429,
"step": 120
},
{
"epoch": 60.0,
"eval_accuracy_dropoff": 0.3199431179614211,
"eval_accuracy_undropoff": 0.985179561323294,
"eval_accuracy_unlabeled": NaN,
"eval_iou_dropoff": 0.2386322423479338,
"eval_iou_undropoff": 0.956946794728302,
"eval_iou_unlabeled": 0.0,
"eval_loss": 0.42677026987075806,
"eval_mean_accuracy": 0.6525613396423575,
"eval_mean_iou": 0.3985263456920786,
"eval_overall_accuracy": 0.95751953125,
"eval_runtime": 2.9496,
"eval_samples_per_second": 6.781,
"eval_steps_per_second": 0.678,
"step": 120
}
],
"max_steps": 240,
"num_train_epochs": 120,
"total_flos": 6.107874830843904e+16,
"trial_name": null,
"trial_params": null
}