sam1120's picture
Training in progress, step 40
0ca38fd
raw
history blame
7.62 kB
{
"best_metric": 0.45899271965026855,
"best_model_checkpoint": "/robodata/smodak/Projects/nspl/scripts/terrainseg/training/models/dropoff-utcustom-train-SF-RGBD-b5_6/checkpoint-40",
"epoch": 20.0,
"global_step": 40,
"is_hyper_param_search": false,
"is_local_process_zero": true,
"is_world_process_zero": true,
"log_history": [
{
"epoch": 0.5,
"learning_rate": 1.6666666666666667e-06,
"loss": 1.2935,
"step": 1
},
{
"epoch": 1.0,
"learning_rate": 3.3333333333333333e-06,
"loss": 1.3048,
"step": 2
},
{
"epoch": 1.5,
"learning_rate": 5e-06,
"loss": 1.2961,
"step": 3
},
{
"epoch": 2.0,
"learning_rate": 6.666666666666667e-06,
"loss": 1.2755,
"step": 4
},
{
"epoch": 2.5,
"learning_rate": 8.333333333333334e-06,
"loss": 1.2586,
"step": 5
},
{
"epoch": 3.0,
"learning_rate": 1e-05,
"loss": 1.246,
"step": 6
},
{
"epoch": 3.5,
"learning_rate": 1.1666666666666668e-05,
"loss": 1.2275,
"step": 7
},
{
"epoch": 4.0,
"learning_rate": 1.3333333333333333e-05,
"loss": 1.2057,
"step": 8
},
{
"epoch": 4.5,
"learning_rate": 1.5000000000000002e-05,
"loss": 1.1914,
"step": 9
},
{
"epoch": 5.0,
"learning_rate": 1.6666666666666667e-05,
"loss": 1.159,
"step": 10
},
{
"epoch": 5.0,
"eval_accuracy_dropoff": 0.5030895204018441,
"eval_accuracy_undropoff": 0.6320608332329993,
"eval_accuracy_unlabeled": NaN,
"eval_iou_dropoff": 0.06443018550460354,
"eval_iou_undropoff": 0.6203390578678918,
"eval_iou_unlabeled": 0.0,
"eval_loss": 1.003960132598877,
"eval_mean_accuracy": 0.5675751768174218,
"eval_mean_iou": 0.22825641445749845,
"eval_overall_accuracy": 0.6266983032226563,
"eval_runtime": 2.8797,
"eval_samples_per_second": 6.945,
"eval_steps_per_second": 0.695,
"step": 10
},
{
"epoch": 5.5,
"learning_rate": 1.8333333333333333e-05,
"loss": 1.138,
"step": 11
},
{
"epoch": 6.0,
"learning_rate": 2e-05,
"loss": 1.1,
"step": 12
},
{
"epoch": 6.5,
"learning_rate": 1.9912280701754387e-05,
"loss": 1.0765,
"step": 13
},
{
"epoch": 7.0,
"learning_rate": 1.9824561403508773e-05,
"loss": 1.0466,
"step": 14
},
{
"epoch": 7.5,
"learning_rate": 1.9736842105263158e-05,
"loss": 1.0032,
"step": 15
},
{
"epoch": 8.0,
"learning_rate": 1.9649122807017544e-05,
"loss": 0.9971,
"step": 16
},
{
"epoch": 8.5,
"learning_rate": 1.9561403508771933e-05,
"loss": 0.9459,
"step": 17
},
{
"epoch": 9.0,
"learning_rate": 1.9473684210526318e-05,
"loss": 0.9112,
"step": 18
},
{
"epoch": 9.5,
"learning_rate": 1.9385964912280704e-05,
"loss": 0.8852,
"step": 19
},
{
"epoch": 10.0,
"learning_rate": 1.929824561403509e-05,
"loss": 0.8345,
"step": 20
},
{
"epoch": 10.0,
"eval_accuracy_dropoff": 0.11338333448014863,
"eval_accuracy_undropoff": 0.9505815555977898,
"eval_accuracy_unlabeled": NaN,
"eval_iou_dropoff": 0.05546940396815957,
"eval_iou_undropoff": 0.9154202339604258,
"eval_iou_unlabeled": 0.0,
"eval_loss": 0.747998058795929,
"eval_mean_accuracy": 0.5319824450389692,
"eval_mean_iou": 0.32362987930952847,
"eval_overall_accuracy": 0.915771484375,
"eval_runtime": 2.8511,
"eval_samples_per_second": 7.015,
"eval_steps_per_second": 0.701,
"step": 20
},
{
"epoch": 10.5,
"learning_rate": 1.9210526315789474e-05,
"loss": 0.8052,
"step": 21
},
{
"epoch": 11.0,
"learning_rate": 1.912280701754386e-05,
"loss": 0.7648,
"step": 22
},
{
"epoch": 11.5,
"learning_rate": 1.9035087719298245e-05,
"loss": 0.7405,
"step": 23
},
{
"epoch": 12.0,
"learning_rate": 1.894736842105263e-05,
"loss": 0.7066,
"step": 24
},
{
"epoch": 12.5,
"learning_rate": 1.885964912280702e-05,
"loss": 0.6675,
"step": 25
},
{
"epoch": 13.0,
"learning_rate": 1.8771929824561405e-05,
"loss": 0.6358,
"step": 26
},
{
"epoch": 13.5,
"learning_rate": 1.868421052631579e-05,
"loss": 0.6127,
"step": 27
},
{
"epoch": 14.0,
"learning_rate": 1.8596491228070176e-05,
"loss": 0.5935,
"step": 28
},
{
"epoch": 14.5,
"learning_rate": 1.8508771929824562e-05,
"loss": 0.5549,
"step": 29
},
{
"epoch": 15.0,
"learning_rate": 1.8421052631578947e-05,
"loss": 0.5406,
"step": 30
},
{
"epoch": 15.0,
"eval_accuracy_dropoff": 0.017945365719397234,
"eval_accuracy_undropoff": 0.9917655826949273,
"eval_accuracy_unlabeled": NaN,
"eval_iou_dropoff": 0.015735932390196418,
"eval_iou_undropoff": 0.9512822485309885,
"eval_iou_unlabeled": 0.0,
"eval_loss": 0.5476881265640259,
"eval_mean_accuracy": 0.5048554742071623,
"eval_mean_iou": 0.322339393640395,
"eval_overall_accuracy": 0.9512748718261719,
"eval_runtime": 5.6484,
"eval_samples_per_second": 3.541,
"eval_steps_per_second": 0.354,
"step": 30
},
{
"epoch": 15.5,
"learning_rate": 1.8333333333333333e-05,
"loss": 0.53,
"step": 31
},
{
"epoch": 16.0,
"learning_rate": 1.824561403508772e-05,
"loss": 0.5004,
"step": 32
},
{
"epoch": 16.5,
"learning_rate": 1.8157894736842107e-05,
"loss": 0.4965,
"step": 33
},
{
"epoch": 17.0,
"learning_rate": 1.8070175438596493e-05,
"loss": 0.4653,
"step": 34
},
{
"epoch": 17.5,
"learning_rate": 1.7982456140350878e-05,
"loss": 0.4434,
"step": 35
},
{
"epoch": 18.0,
"learning_rate": 1.7894736842105264e-05,
"loss": 0.4414,
"step": 36
},
{
"epoch": 18.5,
"learning_rate": 1.780701754385965e-05,
"loss": 0.3915,
"step": 37
},
{
"epoch": 19.0,
"learning_rate": 1.7719298245614035e-05,
"loss": 0.4001,
"step": 38
},
{
"epoch": 19.5,
"learning_rate": 1.763157894736842e-05,
"loss": 0.4204,
"step": 39
},
{
"epoch": 20.0,
"learning_rate": 1.754385964912281e-05,
"loss": 0.3695,
"step": 40
},
{
"epoch": 20.0,
"eval_accuracy_dropoff": 0.014605839583476686,
"eval_accuracy_undropoff": 0.9925813227566402,
"eval_accuracy_unlabeled": NaN,
"eval_iou_dropoff": 0.012491320023381993,
"eval_iou_undropoff": 0.951889892701647,
"eval_iou_unlabeled": 0.0,
"eval_loss": 0.45899271965026855,
"eval_mean_accuracy": 0.5035935811700585,
"eval_mean_iou": 0.3214604042416764,
"eval_overall_accuracy": 0.9519178390502929,
"eval_runtime": 3.3683,
"eval_samples_per_second": 5.938,
"eval_steps_per_second": 0.594,
"step": 40
}
],
"max_steps": 240,
"num_train_epochs": 120,
"total_flos": 4.630914352860365e+17,
"trial_name": null,
"trial_params": null
}