sam1120's picture
Training in progress, step 80
dc074c5
raw history blame
No virus
16.4 kB
{
"best_metric": 0.4204064607620239,
"best_model_checkpoint": "/robodata/smodak/Projects/lifelong_concept_learner/scripts/terrainseg/training/models/safety-utcustom-terrain-final/checkpoint-80",
"epoch": 4.705882352941177,
"global_step": 80,
"is_hyper_param_search": false,
"is_local_process_zero": true,
"is_world_process_zero": true,
"log_history": [
{
"epoch": 0.06,
"learning_rate": 2.9411764705882356e-07,
"loss": 2.7081,
"step": 1
},
{
"epoch": 0.12,
"learning_rate": 5.882352941176471e-07,
"loss": 2.7218,
"step": 2
},
{
"epoch": 0.18,
"learning_rate": 8.823529411764706e-07,
"loss": 2.709,
"step": 3
},
{
"epoch": 0.24,
"learning_rate": 1.1764705882352942e-06,
"loss": 2.6939,
"step": 4
},
{
"epoch": 0.29,
"learning_rate": 1.4705882352941177e-06,
"loss": 2.6979,
"step": 5
},
{
"epoch": 0.35,
"learning_rate": 1.7647058823529412e-06,
"loss": 2.6788,
"step": 6
},
{
"epoch": 0.41,
"learning_rate": 2.058823529411765e-06,
"loss": 2.6769,
"step": 7
},
{
"epoch": 0.47,
"learning_rate": 2.3529411764705885e-06,
"loss": 2.6464,
"step": 8
},
{
"epoch": 0.53,
"learning_rate": 2.647058823529412e-06,
"loss": 2.6424,
"step": 9
},
{
"epoch": 0.59,
"learning_rate": 2.9411764705882355e-06,
"loss": 2.6198,
"step": 10
},
{
"epoch": 0.65,
"learning_rate": 3.2352941176470594e-06,
"loss": 2.6228,
"step": 11
},
{
"epoch": 0.71,
"learning_rate": 3.5294117647058825e-06,
"loss": 2.6158,
"step": 12
},
{
"epoch": 0.76,
"learning_rate": 3.823529411764706e-06,
"loss": 2.632,
"step": 13
},
{
"epoch": 0.82,
"learning_rate": 4.11764705882353e-06,
"loss": 2.587,
"step": 14
},
{
"epoch": 0.88,
"learning_rate": 4.411764705882353e-06,
"loss": 2.5767,
"step": 15
},
{
"epoch": 0.94,
"learning_rate": 4.705882352941177e-06,
"loss": 2.5638,
"step": 16
},
{
"epoch": 1.0,
"learning_rate": 5e-06,
"loss": 2.517,
"step": 17
},
{
"epoch": 1.06,
"learning_rate": 5.294117647058824e-06,
"loss": 2.5202,
"step": 18
},
{
"epoch": 1.12,
"learning_rate": 5.588235294117647e-06,
"loss": 2.5257,
"step": 19
},
{
"epoch": 1.18,
"learning_rate": 5.882352941176471e-06,
"loss": 2.51,
"step": 20
},
{
"epoch": 1.18,
"eval_accuracy_NAT": 0.0901846148886272,
"eval_accuracy_REST": 0.1772357723577236,
"eval_accuracy_concrete": 0.21836230658804573,
"eval_accuracy_concrete tiles": 0.10897949563987744,
"eval_accuracy_dark bricks": 0.01268591007407691,
"eval_accuracy_grass": 0.027127403891645564,
"eval_accuracy_red bricks": 0.0003844463971880492,
"eval_accuracy_road": 0.0070585682671182835,
"eval_accuracy_rough concrete": 0.002699336785465293,
"eval_accuracy_rough red sidewalk": 0.0,
"eval_accuracy_speedway bricks": 0.6741853129093485,
"eval_accuracy_steel": 0.0007721641086127534,
"eval_accuracy_tiles": 0.2798933624163107,
"eval_accuracy_unlabeled": NaN,
"eval_iou_NAT": 0.09010334674005777,
"eval_iou_REST": 2.227093222444523e-05,
"eval_iou_concrete": 0.07965033823417267,
"eval_iou_concrete tiles": 0.0011714161291131616,
"eval_iou_dark bricks": 0.0030587532148061655,
"eval_iou_grass": 0.019163923774789638,
"eval_iou_red bricks": 5.560869275085081e-06,
"eval_iou_road": 0.006794427417222495,
"eval_iou_rough concrete": 0.00013145523557474666,
"eval_iou_rough red sidewalk": 0.0,
"eval_iou_speedway bricks": 0.4632836614619225,
"eval_iou_steel": 0.0006867765509148131,
"eval_iou_tiles": 0.0032256814378599824,
"eval_iou_unlabeled": 0.0,
"eval_loss": 2.507078170776367,
"eval_mean_accuracy": 0.12304374571723385,
"eval_mean_iou": 0.04766411514270953,
"eval_overall_accuracy": 0.20164948802883342,
"eval_runtime": 18.8176,
"eval_samples_per_second": 6.271,
"eval_steps_per_second": 0.425,
"step": 20
},
{
"epoch": 1.24,
"learning_rate": 6.1764705882352944e-06,
"loss": 2.4771,
"step": 21
},
{
"epoch": 1.29,
"learning_rate": 6.470588235294119e-06,
"loss": 2.4758,
"step": 22
},
{
"epoch": 1.35,
"learning_rate": 6.7647058823529414e-06,
"loss": 2.4283,
"step": 23
},
{
"epoch": 1.41,
"learning_rate": 7.058823529411765e-06,
"loss": 2.3783,
"step": 24
},
{
"epoch": 1.47,
"learning_rate": 7.3529411764705884e-06,
"loss": 2.3059,
"step": 25
},
{
"epoch": 1.53,
"learning_rate": 7.647058823529413e-06,
"loss": 2.3405,
"step": 26
},
{
"epoch": 1.59,
"learning_rate": 7.941176470588235e-06,
"loss": 2.2917,
"step": 27
},
{
"epoch": 1.65,
"learning_rate": 8.23529411764706e-06,
"loss": 2.2857,
"step": 28
},
{
"epoch": 1.71,
"learning_rate": 8.529411764705883e-06,
"loss": 2.1291,
"step": 29
},
{
"epoch": 1.76,
"learning_rate": 8.823529411764707e-06,
"loss": 2.1234,
"step": 30
},
{
"epoch": 1.82,
"learning_rate": 9.117647058823529e-06,
"loss": 2.0916,
"step": 31
},
{
"epoch": 1.88,
"learning_rate": 9.411764705882354e-06,
"loss": 2.0543,
"step": 32
},
{
"epoch": 1.94,
"learning_rate": 9.705882352941177e-06,
"loss": 1.9605,
"step": 33
},
{
"epoch": 2.0,
"learning_rate": 1e-05,
"loss": 1.9835,
"step": 34
},
{
"epoch": 2.06,
"learning_rate": 1.0294117647058824e-05,
"loss": 2.0005,
"step": 35
},
{
"epoch": 2.12,
"learning_rate": 1.0588235294117648e-05,
"loss": 1.8628,
"step": 36
},
{
"epoch": 2.18,
"learning_rate": 1.0882352941176471e-05,
"loss": 1.8321,
"step": 37
},
{
"epoch": 2.24,
"learning_rate": 1.1176470588235295e-05,
"loss": 1.7578,
"step": 38
},
{
"epoch": 2.29,
"learning_rate": 1.1470588235294118e-05,
"loss": 1.7233,
"step": 39
},
{
"epoch": 2.35,
"learning_rate": 1.1764705882352942e-05,
"loss": 1.5504,
"step": 40
},
{
"epoch": 2.35,
"eval_accuracy_NAT": 0.760154401121494,
"eval_accuracy_REST": 0.0,
"eval_accuracy_concrete": 0.3386920222776326,
"eval_accuracy_concrete tiles": 0.007895357058684892,
"eval_accuracy_dark bricks": 0.0014308136975288556,
"eval_accuracy_grass": 0.07454476982063306,
"eval_accuracy_red bricks": 0.0,
"eval_accuracy_road": 0.3332036483520477,
"eval_accuracy_rough concrete": 0.0009832021404012656,
"eval_accuracy_rough red sidewalk": 0.0,
"eval_accuracy_speedway bricks": 0.9879532008612127,
"eval_accuracy_steel": 0.0,
"eval_accuracy_tiles": 0.00875518797903602,
"eval_accuracy_unlabeled": NaN,
"eval_iou_NAT": 0.7577068331202973,
"eval_iou_REST": 0.0,
"eval_iou_concrete": 0.2037716446695068,
"eval_iou_concrete tiles": 0.00033778197232406094,
"eval_iou_dark bricks": 0.0009957339958239116,
"eval_iou_grass": 0.06824458242953867,
"eval_iou_red bricks": 0.0,
"eval_iou_road": 0.31270597640653264,
"eval_iou_rough concrete": 0.00024414351495637967,
"eval_iou_rough red sidewalk": 0.0,
"eval_iou_speedway bricks": 0.571054819328365,
"eval_iou_steel": 0.0,
"eval_iou_tiles": 0.0007685393497465681,
"eval_iou_unlabeled": 0.0,
"eval_loss": 1.840782642364502,
"eval_mean_accuracy": 0.1933548156391286,
"eval_mean_iou": 0.13684500391336366,
"eval_overall_accuracy": 0.6935860908637612,
"eval_runtime": 7.985,
"eval_samples_per_second": 14.778,
"eval_steps_per_second": 1.002,
"step": 40
},
{
"epoch": 2.41,
"learning_rate": 1.2058823529411765e-05,
"loss": 1.7505,
"step": 41
},
{
"epoch": 2.47,
"learning_rate": 1.2352941176470589e-05,
"loss": 1.4808,
"step": 42
},
{
"epoch": 2.53,
"learning_rate": 1.2647058823529412e-05,
"loss": 1.4041,
"step": 43
},
{
"epoch": 2.59,
"learning_rate": 1.2941176470588238e-05,
"loss": 1.3261,
"step": 44
},
{
"epoch": 2.65,
"learning_rate": 1.323529411764706e-05,
"loss": 1.3337,
"step": 45
},
{
"epoch": 2.71,
"learning_rate": 1.3529411764705883e-05,
"loss": 1.3177,
"step": 46
},
{
"epoch": 2.76,
"learning_rate": 1.3823529411764708e-05,
"loss": 1.2551,
"step": 47
},
{
"epoch": 2.82,
"learning_rate": 1.411764705882353e-05,
"loss": 1.119,
"step": 48
},
{
"epoch": 2.88,
"learning_rate": 1.4411764705882352e-05,
"loss": 1.1259,
"step": 49
},
{
"epoch": 2.94,
"learning_rate": 1.4705882352941177e-05,
"loss": 1.0199,
"step": 50
},
{
"epoch": 3.0,
"learning_rate": 1.5e-05,
"loss": 1.1612,
"step": 51
},
{
"epoch": 3.06,
"learning_rate": 1.5294117647058826e-05,
"loss": 0.8821,
"step": 52
},
{
"epoch": 3.12,
"learning_rate": 1.558823529411765e-05,
"loss": 0.8667,
"step": 53
},
{
"epoch": 3.18,
"learning_rate": 1.588235294117647e-05,
"loss": 0.8607,
"step": 54
},
{
"epoch": 3.24,
"learning_rate": 1.6176470588235296e-05,
"loss": 0.8021,
"step": 55
},
{
"epoch": 3.29,
"learning_rate": 1.647058823529412e-05,
"loss": 0.7418,
"step": 56
},
{
"epoch": 3.35,
"learning_rate": 1.676470588235294e-05,
"loss": 0.7114,
"step": 57
},
{
"epoch": 3.41,
"learning_rate": 1.7058823529411767e-05,
"loss": 0.7799,
"step": 58
},
{
"epoch": 3.47,
"learning_rate": 1.735294117647059e-05,
"loss": 0.6974,
"step": 59
},
{
"epoch": 3.53,
"learning_rate": 1.7647058823529414e-05,
"loss": 0.6271,
"step": 60
},
{
"epoch": 3.53,
"eval_accuracy_NAT": 0.9551305368117847,
"eval_accuracy_REST": 0.0,
"eval_accuracy_concrete": 0.8126940616930818,
"eval_accuracy_concrete tiles": 0.0,
"eval_accuracy_dark bricks": 0.0,
"eval_accuracy_grass": 0.25898348643298624,
"eval_accuracy_red bricks": 0.0,
"eval_accuracy_road": 0.8978859032247204,
"eval_accuracy_rough concrete": 0.0,
"eval_accuracy_rough red sidewalk": 0.0,
"eval_accuracy_speedway bricks": 0.9793115987478005,
"eval_accuracy_steel": 0.0,
"eval_accuracy_tiles": 3.0294768093550245e-05,
"eval_accuracy_unlabeled": NaN,
"eval_iou_NAT": 0.9404775296004152,
"eval_iou_REST": 0.0,
"eval_iou_concrete": 0.5259192497701041,
"eval_iou_concrete tiles": 0.0,
"eval_iou_dark bricks": 0.0,
"eval_iou_grass": 0.2391512912549761,
"eval_iou_red bricks": 0.0,
"eval_iou_road": 0.6993177996437074,
"eval_iou_rough concrete": 0.0,
"eval_iou_rough red sidewalk": 0.0,
"eval_iou_speedway bricks": 0.8262999758534085,
"eval_iou_steel": 0.0,
"eval_iou_tiles": 2.891343318105592e-05,
"eval_iou_unlabeled": NaN,
"eval_loss": 0.8082596063613892,
"eval_mean_accuracy": 0.30031045243680515,
"eval_mean_iou": 0.24855344304275326,
"eval_overall_accuracy": 0.8893904281874835,
"eval_runtime": 6.7648,
"eval_samples_per_second": 17.443,
"eval_steps_per_second": 1.183,
"step": 60
},
{
"epoch": 3.59,
"learning_rate": 1.7941176470588237e-05,
"loss": 0.6901,
"step": 61
},
{
"epoch": 3.65,
"learning_rate": 1.8235294117647057e-05,
"loss": 0.5778,
"step": 62
},
{
"epoch": 3.71,
"learning_rate": 1.8529411764705884e-05,
"loss": 0.6319,
"step": 63
},
{
"epoch": 3.76,
"learning_rate": 1.8823529411764708e-05,
"loss": 0.5626,
"step": 64
},
{
"epoch": 3.82,
"learning_rate": 1.9117647058823528e-05,
"loss": 0.7374,
"step": 65
},
{
"epoch": 3.88,
"learning_rate": 1.9411764705882355e-05,
"loss": 0.6702,
"step": 66
},
{
"epoch": 3.94,
"learning_rate": 1.9705882352941178e-05,
"loss": 0.5477,
"step": 67
},
{
"epoch": 4.0,
"learning_rate": 2e-05,
"loss": 0.586,
"step": 68
},
{
"epoch": 4.06,
"learning_rate": 2.0294117647058825e-05,
"loss": 0.505,
"step": 69
},
{
"epoch": 4.12,
"learning_rate": 2.058823529411765e-05,
"loss": 0.5758,
"step": 70
},
{
"epoch": 4.18,
"learning_rate": 2.0882352941176472e-05,
"loss": 0.5694,
"step": 71
},
{
"epoch": 4.24,
"learning_rate": 2.1176470588235296e-05,
"loss": 0.4535,
"step": 72
},
{
"epoch": 4.29,
"learning_rate": 2.1470588235294116e-05,
"loss": 0.5105,
"step": 73
},
{
"epoch": 4.35,
"learning_rate": 2.1764705882352943e-05,
"loss": 0.5525,
"step": 74
},
{
"epoch": 4.41,
"learning_rate": 2.2058823529411766e-05,
"loss": 0.465,
"step": 75
},
{
"epoch": 4.47,
"learning_rate": 2.235294117647059e-05,
"loss": 0.5477,
"step": 76
},
{
"epoch": 4.53,
"learning_rate": 2.2647058823529413e-05,
"loss": 0.3823,
"step": 77
},
{
"epoch": 4.59,
"learning_rate": 2.2941176470588237e-05,
"loss": 0.4698,
"step": 78
},
{
"epoch": 4.65,
"learning_rate": 2.323529411764706e-05,
"loss": 0.4636,
"step": 79
},
{
"epoch": 4.71,
"learning_rate": 2.3529411764705884e-05,
"loss": 0.3963,
"step": 80
},
{
"epoch": 4.71,
"eval_accuracy_NAT": 0.969774210544504,
"eval_accuracy_REST": 0.0,
"eval_accuracy_concrete": 0.7972872258293342,
"eval_accuracy_concrete tiles": 0.04426113598868725,
"eval_accuracy_dark bricks": 0.0,
"eval_accuracy_grass": 0.7826429655535243,
"eval_accuracy_red bricks": 0.0,
"eval_accuracy_road": 0.9650914038630805,
"eval_accuracy_rough concrete": 0.0,
"eval_accuracy_rough red sidewalk": 0.0,
"eval_accuracy_speedway bricks": 0.9776932262095467,
"eval_accuracy_steel": 0.0003548456421456666,
"eval_accuracy_tiles": 0.0,
"eval_accuracy_unlabeled": NaN,
"eval_iou_NAT": 0.9531585041831687,
"eval_iou_REST": 0.0,
"eval_iou_concrete": 0.6900984540650801,
"eval_iou_concrete tiles": 0.04426113598868725,
"eval_iou_dark bricks": 0.0,
"eval_iou_grass": 0.5632131076991799,
"eval_iou_red bricks": 0.0,
"eval_iou_road": 0.727967248165449,
"eval_iou_rough concrete": 0.0,
"eval_iou_rough red sidewalk": 0.0,
"eval_iou_speedway bricks": 0.8641130344685216,
"eval_iou_steel": 0.00035449307490288885,
"eval_iou_tiles": 0.0,
"eval_iou_unlabeled": NaN,
"eval_loss": 0.4204064607620239,
"eval_mean_accuracy": 0.34900807797160177,
"eval_mean_iou": 0.29562815212653765,
"eval_overall_accuracy": 0.9185845003289691,
"eval_runtime": 5.4734,
"eval_samples_per_second": 21.559,
"eval_steps_per_second": 1.462,
"step": 80
}
],
"max_steps": 3400,
"num_train_epochs": 200,
"total_flos": 5.0620116674543616e+17,
"trial_name": null,
"trial_params": null
}