sam1120's picture
Training in progress, step 80
1dcb23d
raw
history blame
15.9 kB
{
"best_metric": 0.16272249817848206,
"best_model_checkpoint": "/robodata/smodak/Projects/nspl/scripts/terrainseg/training/models/safety-userstudy-terrain/checkpoint-80",
"epoch": 80.0,
"global_step": 80,
"is_hyper_param_search": false,
"is_local_process_zero": true,
"is_world_process_zero": true,
"log_history": [
{
"epoch": 1.0,
"learning_rate": 1.6666666666666667e-06,
"loss": 2.8187,
"step": 1
},
{
"epoch": 2.0,
"learning_rate": 3.3333333333333333e-06,
"loss": 2.8228,
"step": 2
},
{
"epoch": 3.0,
"learning_rate": 5e-06,
"loss": 2.821,
"step": 3
},
{
"epoch": 4.0,
"learning_rate": 6.666666666666667e-06,
"loss": 2.7963,
"step": 4
},
{
"epoch": 5.0,
"learning_rate": 8.333333333333334e-06,
"loss": 2.7811,
"step": 5
},
{
"epoch": 6.0,
"learning_rate": 1e-05,
"loss": 2.7249,
"step": 6
},
{
"epoch": 7.0,
"learning_rate": 1.1666666666666668e-05,
"loss": 2.6948,
"step": 7
},
{
"epoch": 8.0,
"learning_rate": 1.3333333333333333e-05,
"loss": 2.6495,
"step": 8
},
{
"epoch": 9.0,
"learning_rate": 1.5e-05,
"loss": 2.594,
"step": 9
},
{
"epoch": 10.0,
"learning_rate": 1.6666666666666667e-05,
"loss": 2.538,
"step": 10
},
{
"epoch": 11.0,
"learning_rate": 1.8333333333333333e-05,
"loss": 2.4672,
"step": 11
},
{
"epoch": 12.0,
"learning_rate": 2e-05,
"loss": 2.4293,
"step": 12
},
{
"epoch": 13.0,
"learning_rate": 2.1666666666666667e-05,
"loss": 2.3298,
"step": 13
},
{
"epoch": 14.0,
"learning_rate": 2.3333333333333336e-05,
"loss": 2.258,
"step": 14
},
{
"epoch": 15.0,
"learning_rate": 2.5e-05,
"loss": 2.1281,
"step": 15
},
{
"epoch": 16.0,
"learning_rate": 2.6666666666666667e-05,
"loss": 2.0277,
"step": 16
},
{
"epoch": 17.0,
"learning_rate": 2.8333333333333335e-05,
"loss": 1.9414,
"step": 17
},
{
"epoch": 18.0,
"learning_rate": 3e-05,
"loss": 1.8256,
"step": 18
},
{
"epoch": 19.0,
"learning_rate": 3.1666666666666666e-05,
"loss": 1.678,
"step": 19
},
{
"epoch": 20.0,
"learning_rate": 3.3333333333333335e-05,
"loss": 1.5957,
"step": 20
},
{
"epoch": 20.0,
"eval_accuracy_NAT": 0.8321817496578309,
"eval_accuracy_REST": NaN,
"eval_accuracy_concrete-road-adj": NaN,
"eval_accuracy_concrete-speedway-side": 0.0071490677229253214,
"eval_accuracy_dark-bricks": NaN,
"eval_accuracy_grass": 0.0,
"eval_accuracy_red-bricks": NaN,
"eval_accuracy_road": NaN,
"eval_accuracy_rough-red-sidewalk": 0.0,
"eval_accuracy_speedway-bricks": 0.9400317341795781,
"eval_accuracy_steel": 0.0,
"eval_accuracy_steel-sidewalk": NaN,
"eval_accuracy_tiles-marble": NaN,
"eval_accuracy_unlabeled": NaN,
"eval_iou_NAT": 0.8227595719506057,
"eval_iou_REST": NaN,
"eval_iou_concrete-road-adj": NaN,
"eval_iou_concrete-speedway-side": 0.006510645785676579,
"eval_iou_dark-bricks": NaN,
"eval_iou_grass": 0.0,
"eval_iou_red-bricks": 0.0,
"eval_iou_road": 0.0,
"eval_iou_rough-red-sidewalk": 0.0,
"eval_iou_speedway-bricks": 0.6827730558356707,
"eval_iou_steel": 0.0,
"eval_iou_steel-sidewalk": 0.0,
"eval_iou_tiles-marble": 0.0,
"eval_iou_unlabeled": 0.0,
"eval_loss": 1.5208492279052734,
"eval_mean_accuracy": 0.2965604252600557,
"eval_mean_iou": 0.13745847941563208,
"eval_overall_accuracy": 0.7877578735351562,
"eval_runtime": 0.7158,
"eval_samples_per_second": 1.397,
"eval_steps_per_second": 1.397,
"step": 20
},
{
"epoch": 21.0,
"learning_rate": 3.5e-05,
"loss": 1.4548,
"step": 21
},
{
"epoch": 22.0,
"learning_rate": 3.6666666666666666e-05,
"loss": 1.3462,
"step": 22
},
{
"epoch": 23.0,
"learning_rate": 3.8333333333333334e-05,
"loss": 1.2189,
"step": 23
},
{
"epoch": 24.0,
"learning_rate": 4e-05,
"loss": 1.1207,
"step": 24
},
{
"epoch": 25.0,
"learning_rate": 4.166666666666667e-05,
"loss": 0.9904,
"step": 25
},
{
"epoch": 26.0,
"learning_rate": 4.3333333333333334e-05,
"loss": 0.9286,
"step": 26
},
{
"epoch": 27.0,
"learning_rate": 4.5e-05,
"loss": 0.8425,
"step": 27
},
{
"epoch": 28.0,
"learning_rate": 4.666666666666667e-05,
"loss": 0.7906,
"step": 28
},
{
"epoch": 29.0,
"learning_rate": 4.8333333333333334e-05,
"loss": 0.7611,
"step": 29
},
{
"epoch": 30.0,
"learning_rate": 5e-05,
"loss": 0.6952,
"step": 30
},
{
"epoch": 31.0,
"learning_rate": 4.991228070175439e-05,
"loss": 0.6077,
"step": 31
},
{
"epoch": 32.0,
"learning_rate": 4.9824561403508773e-05,
"loss": 0.5674,
"step": 32
},
{
"epoch": 33.0,
"learning_rate": 4.973684210526316e-05,
"loss": 0.5183,
"step": 33
},
{
"epoch": 34.0,
"learning_rate": 4.9649122807017544e-05,
"loss": 0.4915,
"step": 34
},
{
"epoch": 35.0,
"learning_rate": 4.956140350877193e-05,
"loss": 0.4782,
"step": 35
},
{
"epoch": 36.0,
"learning_rate": 4.9473684210526315e-05,
"loss": 0.4367,
"step": 36
},
{
"epoch": 37.0,
"learning_rate": 4.93859649122807e-05,
"loss": 0.4252,
"step": 37
},
{
"epoch": 38.0,
"learning_rate": 4.9298245614035086e-05,
"loss": 0.3915,
"step": 38
},
{
"epoch": 39.0,
"learning_rate": 4.921052631578947e-05,
"loss": 0.3803,
"step": 39
},
{
"epoch": 40.0,
"learning_rate": 4.912280701754386e-05,
"loss": 0.3589,
"step": 40
},
{
"epoch": 40.0,
"eval_accuracy_NAT": 0.962933331570143,
"eval_accuracy_REST": NaN,
"eval_accuracy_concrete-road-adj": NaN,
"eval_accuracy_concrete-speedway-side": 0.44266254468167326,
"eval_accuracy_dark-bricks": NaN,
"eval_accuracy_grass": 0.14368837885889768,
"eval_accuracy_red-bricks": NaN,
"eval_accuracy_road": NaN,
"eval_accuracy_rough-red-sidewalk": 0.0,
"eval_accuracy_speedway-bricks": 1.0,
"eval_accuracy_steel": 0.0,
"eval_accuracy_steel-sidewalk": NaN,
"eval_accuracy_tiles-marble": NaN,
"eval_accuracy_unlabeled": NaN,
"eval_iou_NAT": 0.9404239931293225,
"eval_iou_REST": NaN,
"eval_iou_concrete-road-adj": NaN,
"eval_iou_concrete-speedway-side": 0.34041604754829125,
"eval_iou_dark-bricks": NaN,
"eval_iou_grass": 0.11037993543580829,
"eval_iou_red-bricks": NaN,
"eval_iou_road": NaN,
"eval_iou_rough-red-sidewalk": 0.0,
"eval_iou_speedway-bricks": 0.836688077155854,
"eval_iou_steel": 0.0,
"eval_iou_steel-sidewalk": NaN,
"eval_iou_tiles-marble": NaN,
"eval_iou_unlabeled": NaN,
"eval_loss": 0.37735000252723694,
"eval_mean_accuracy": 0.424880709185119,
"eval_mean_iou": 0.37131800887821265,
"eval_overall_accuracy": 0.9033889770507812,
"eval_runtime": 0.5957,
"eval_samples_per_second": 1.679,
"eval_steps_per_second": 1.679,
"step": 40
},
{
"epoch": 41.0,
"learning_rate": 4.903508771929825e-05,
"loss": 0.3388,
"step": 41
},
{
"epoch": 42.0,
"learning_rate": 4.8947368421052635e-05,
"loss": 0.3296,
"step": 42
},
{
"epoch": 43.0,
"learning_rate": 4.885964912280702e-05,
"loss": 0.3273,
"step": 43
},
{
"epoch": 44.0,
"learning_rate": 4.8771929824561406e-05,
"loss": 0.2959,
"step": 44
},
{
"epoch": 45.0,
"learning_rate": 4.868421052631579e-05,
"loss": 0.2805,
"step": 45
},
{
"epoch": 46.0,
"learning_rate": 4.859649122807018e-05,
"loss": 0.2713,
"step": 46
},
{
"epoch": 47.0,
"learning_rate": 4.850877192982456e-05,
"loss": 0.2579,
"step": 47
},
{
"epoch": 48.0,
"learning_rate": 4.842105263157895e-05,
"loss": 0.2477,
"step": 48
},
{
"epoch": 49.0,
"learning_rate": 4.8333333333333334e-05,
"loss": 0.2412,
"step": 49
},
{
"epoch": 50.0,
"learning_rate": 4.824561403508772e-05,
"loss": 0.2234,
"step": 50
},
{
"epoch": 51.0,
"learning_rate": 4.8157894736842105e-05,
"loss": 0.2311,
"step": 51
},
{
"epoch": 52.0,
"learning_rate": 4.807017543859649e-05,
"loss": 0.2132,
"step": 52
},
{
"epoch": 53.0,
"learning_rate": 4.7982456140350876e-05,
"loss": 0.214,
"step": 53
},
{
"epoch": 54.0,
"learning_rate": 4.789473684210526e-05,
"loss": 0.2097,
"step": 54
},
{
"epoch": 55.0,
"learning_rate": 4.780701754385965e-05,
"loss": 0.2033,
"step": 55
},
{
"epoch": 56.0,
"learning_rate": 4.771929824561404e-05,
"loss": 0.192,
"step": 56
},
{
"epoch": 57.0,
"learning_rate": 4.7631578947368424e-05,
"loss": 0.1857,
"step": 57
},
{
"epoch": 58.0,
"learning_rate": 4.754385964912281e-05,
"loss": 0.1799,
"step": 58
},
{
"epoch": 59.0,
"learning_rate": 4.7456140350877195e-05,
"loss": 0.1864,
"step": 59
},
{
"epoch": 60.0,
"learning_rate": 4.736842105263158e-05,
"loss": 0.173,
"step": 60
},
{
"epoch": 60.0,
"eval_accuracy_NAT": 0.9537294781177061,
"eval_accuracy_REST": NaN,
"eval_accuracy_concrete-road-adj": NaN,
"eval_accuracy_concrete-speedway-side": 0.6674717418606898,
"eval_accuracy_dark-bricks": NaN,
"eval_accuracy_grass": 0.5639243575238403,
"eval_accuracy_red-bricks": NaN,
"eval_accuracy_road": NaN,
"eval_accuracy_rough-red-sidewalk": 0.5100017702248185,
"eval_accuracy_speedway-bricks": 1.0,
"eval_accuracy_steel": 0.0,
"eval_accuracy_steel-sidewalk": NaN,
"eval_accuracy_tiles-marble": NaN,
"eval_accuracy_unlabeled": NaN,
"eval_iou_NAT": 0.940778617689453,
"eval_iou_REST": NaN,
"eval_iou_concrete-road-adj": NaN,
"eval_iou_concrete-speedway-side": 0.635193527627103,
"eval_iou_dark-bricks": NaN,
"eval_iou_grass": 0.29731572219855135,
"eval_iou_red-bricks": NaN,
"eval_iou_road": NaN,
"eval_iou_rough-red-sidewalk": 0.4458372021046116,
"eval_iou_speedway-bricks": 0.8960722611941079,
"eval_iou_steel": 0.0,
"eval_iou_steel-sidewalk": NaN,
"eval_iou_tiles-marble": NaN,
"eval_iou_unlabeled": NaN,
"eval_loss": 0.25648337602615356,
"eval_mean_accuracy": 0.6158545579545092,
"eval_mean_iou": 0.5358662218023046,
"eval_overall_accuracy": 0.9278640747070312,
"eval_runtime": 0.5992,
"eval_samples_per_second": 1.669,
"eval_steps_per_second": 1.669,
"step": 60
},
{
"epoch": 61.0,
"learning_rate": 4.728070175438597e-05,
"loss": 0.1641,
"step": 61
},
{
"epoch": 62.0,
"learning_rate": 4.719298245614036e-05,
"loss": 0.1643,
"step": 62
},
{
"epoch": 63.0,
"learning_rate": 4.7105263157894744e-05,
"loss": 0.1636,
"step": 63
},
{
"epoch": 64.0,
"learning_rate": 4.701754385964913e-05,
"loss": 0.16,
"step": 64
},
{
"epoch": 65.0,
"learning_rate": 4.6929824561403515e-05,
"loss": 0.1544,
"step": 65
},
{
"epoch": 66.0,
"learning_rate": 4.68421052631579e-05,
"loss": 0.1483,
"step": 66
},
{
"epoch": 67.0,
"learning_rate": 4.6754385964912286e-05,
"loss": 0.1445,
"step": 67
},
{
"epoch": 68.0,
"learning_rate": 4.666666666666667e-05,
"loss": 0.1532,
"step": 68
},
{
"epoch": 69.0,
"learning_rate": 4.657894736842106e-05,
"loss": 0.1446,
"step": 69
},
{
"epoch": 70.0,
"learning_rate": 4.649122807017544e-05,
"loss": 0.1374,
"step": 70
},
{
"epoch": 71.0,
"learning_rate": 4.640350877192983e-05,
"loss": 0.1345,
"step": 71
},
{
"epoch": 72.0,
"learning_rate": 4.6315789473684214e-05,
"loss": 0.1318,
"step": 72
},
{
"epoch": 73.0,
"learning_rate": 4.62280701754386e-05,
"loss": 0.1328,
"step": 73
},
{
"epoch": 74.0,
"learning_rate": 4.6140350877192985e-05,
"loss": 0.124,
"step": 74
},
{
"epoch": 75.0,
"learning_rate": 4.605263157894737e-05,
"loss": 0.1365,
"step": 75
},
{
"epoch": 76.0,
"learning_rate": 4.5964912280701756e-05,
"loss": 0.1256,
"step": 76
},
{
"epoch": 77.0,
"learning_rate": 4.587719298245614e-05,
"loss": 0.1216,
"step": 77
},
{
"epoch": 78.0,
"learning_rate": 4.5789473684210527e-05,
"loss": 0.124,
"step": 78
},
{
"epoch": 79.0,
"learning_rate": 4.570175438596491e-05,
"loss": 0.1213,
"step": 79
},
{
"epoch": 80.0,
"learning_rate": 4.56140350877193e-05,
"loss": 0.1134,
"step": 80
},
{
"epoch": 80.0,
"eval_accuracy_NAT": 0.9792516579498945,
"eval_accuracy_REST": NaN,
"eval_accuracy_concrete-road-adj": NaN,
"eval_accuracy_concrete-speedway-side": 0.8183750362283837,
"eval_accuracy_dark-bricks": NaN,
"eval_accuracy_grass": 0.49070631970260226,
"eval_accuracy_red-bricks": NaN,
"eval_accuracy_road": NaN,
"eval_accuracy_rough-red-sidewalk": 0.9412285360240751,
"eval_accuracy_speedway-bricks": 0.9946098562628337,
"eval_accuracy_steel": 0.5342876165113183,
"eval_accuracy_steel-sidewalk": NaN,
"eval_accuracy_tiles-marble": NaN,
"eval_accuracy_unlabeled": NaN,
"eval_iou_NAT": 0.9568551695621555,
"eval_iou_REST": NaN,
"eval_iou_concrete-road-adj": NaN,
"eval_iou_concrete-speedway-side": 0.7767284063818082,
"eval_iou_dark-bricks": NaN,
"eval_iou_grass": 0.365739067582219,
"eval_iou_red-bricks": NaN,
"eval_iou_road": NaN,
"eval_iou_rough-red-sidewalk": 0.7858409695536506,
"eval_iou_speedway-bricks": 0.9649559685780907,
"eval_iou_steel": 0.47122724603640637,
"eval_iou_steel-sidewalk": NaN,
"eval_iou_tiles-marble": NaN,
"eval_iou_unlabeled": NaN,
"eval_loss": 0.16272249817848206,
"eval_mean_accuracy": 0.7930765037798513,
"eval_mean_iou": 0.7202244712823885,
"eval_overall_accuracy": 0.9604721069335938,
"eval_runtime": 0.6081,
"eval_samples_per_second": 1.644,
"eval_steps_per_second": 1.644,
"step": 80
}
],
"max_steps": 600,
"num_train_epochs": 600,
"total_flos": 4.151807676776448e+17,
"trial_name": null,
"trial_params": null
}