{ "best_metric": 0.6009885668754578, "best_model_checkpoint": "/robodata/smodak/Projects/nspl/scripts/terrainseg/training/models/dropoff-utcustom-train-SF-RGB-b5_2/checkpoint-120", "epoch": 60.0, "global_step": 120, "is_hyper_param_search": false, "is_local_process_zero": true, "is_world_process_zero": true, "log_history": [ { "epoch": 0.5, "learning_rate": 2.5e-07, "loss": 1.031, "step": 1 }, { "epoch": 1.0, "learning_rate": 5e-07, "loss": 1.0251, "step": 2 }, { "epoch": 1.5, "learning_rate": 7.5e-07, "loss": 1.036, "step": 3 }, { "epoch": 2.0, "learning_rate": 1e-06, "loss": 1.0235, "step": 4 }, { "epoch": 2.5, "learning_rate": 1.25e-06, "loss": 1.0287, "step": 5 }, { "epoch": 3.0, "learning_rate": 1.5e-06, "loss": 1.0206, "step": 6 }, { "epoch": 3.5, "learning_rate": 1.7500000000000002e-06, "loss": 1.0211, "step": 7 }, { "epoch": 4.0, "learning_rate": 2e-06, "loss": 1.0216, "step": 8 }, { "epoch": 4.5, "learning_rate": 2.25e-06, "loss": 1.0167, "step": 9 }, { "epoch": 5.0, "learning_rate": 2.5e-06, "loss": 1.0108, "step": 10 }, { "epoch": 5.0, "eval_accuracy_dropoff": 0.6706117112777816, "eval_accuracy_undropoff": 0.40962748401207194, "eval_accuracy_unlabeled": NaN, "eval_iou_dropoff": 0.049414973896495254, "eval_iou_undropoff": 0.404722218999734, "eval_iou_unlabeled": 0.0, "eval_loss": 1.0721067190170288, "eval_mean_accuracy": 0.5401195976449268, "eval_mean_iou": 0.15137906429874307, "eval_overall_accuracy": 0.42047901153564454, "eval_runtime": 2.4375, "eval_samples_per_second": 8.205, "eval_steps_per_second": 0.821, "step": 10 }, { "epoch": 5.5, "learning_rate": 2.75e-06, "loss": 1.0088, "step": 11 }, { "epoch": 6.0, "learning_rate": 3e-06, "loss": 0.9999, "step": 12 }, { "epoch": 6.5, "learning_rate": 2.986842105263158e-06, "loss": 0.999, "step": 13 }, { "epoch": 7.0, "learning_rate": 2.973684210526316e-06, "loss": 0.9936, "step": 14 }, { "epoch": 7.5, "learning_rate": 2.960526315789474e-06, "loss": 0.9839, "step": 15 }, { "epoch": 8.0, "learning_rate": 2.9473684210526313e-06, "loss": 0.9799, "step": 16 }, { "epoch": 8.5, "learning_rate": 2.9342105263157896e-06, "loss": 0.9709, "step": 17 }, { "epoch": 9.0, "learning_rate": 2.9210526315789475e-06, "loss": 0.9737, "step": 18 }, { "epoch": 9.5, "learning_rate": 2.9078947368421054e-06, "loss": 0.9662, "step": 19 }, { "epoch": 10.0, "learning_rate": 2.8947368421052634e-06, "loss": 0.9654, "step": 20 }, { "epoch": 10.0, "eval_accuracy_dropoff": 0.7253239753205348, "eval_accuracy_undropoff": 0.5887255131211958, "eval_accuracy_unlabeled": NaN, "eval_iou_dropoff": 0.07447834260405004, "eval_iou_undropoff": 0.5825809692740065, "eval_iou_unlabeled": 0.0, "eval_loss": 0.9802306890487671, "eval_mean_accuracy": 0.6570247442208653, "eval_mean_iou": 0.21901977062601885, "eval_overall_accuracy": 0.5944051742553711, "eval_runtime": 2.3079, "eval_samples_per_second": 8.666, "eval_steps_per_second": 0.867, "step": 20 }, { "epoch": 10.5, "learning_rate": 2.8815789473684213e-06, "loss": 0.9466, "step": 21 }, { "epoch": 11.0, "learning_rate": 2.8684210526315787e-06, "loss": 0.9606, "step": 22 }, { "epoch": 11.5, "learning_rate": 2.8552631578947367e-06, "loss": 0.9498, "step": 23 }, { "epoch": 12.0, "learning_rate": 2.8421052631578946e-06, "loss": 0.9443, "step": 24 }, { "epoch": 12.5, "learning_rate": 2.828947368421053e-06, "loss": 0.941, "step": 25 }, { "epoch": 13.0, "learning_rate": 2.815789473684211e-06, "loss": 0.9356, "step": 26 }, { "epoch": 13.5, "learning_rate": 2.8026315789473687e-06, "loss": 0.937, "step": 27 }, { "epoch": 14.0, "learning_rate": 2.789473684210526e-06, "loss": 0.9221, "step": 28 }, { "epoch": 14.5, "learning_rate": 2.776315789473684e-06, "loss": 0.9297, "step": 29 }, { "epoch": 15.0, "learning_rate": 2.763157894736842e-06, "loss": 0.9175, "step": 30 }, { "epoch": 15.0, "eval_accuracy_dropoff": 0.7959632101653707, "eval_accuracy_undropoff": 0.6741250794794309, "eval_accuracy_unlabeled": NaN, "eval_iou_dropoff": 0.09733724441512442, "eval_iou_undropoff": 0.6686192564420758, "eval_iou_unlabeled": 0.0, "eval_loss": 0.9046744108200073, "eval_mean_accuracy": 0.7350441448224008, "eval_mean_iou": 0.25531883361906677, "eval_overall_accuracy": 0.6791910171508789, "eval_runtime": 2.4227, "eval_samples_per_second": 8.255, "eval_steps_per_second": 0.826, "step": 30 }, { "epoch": 15.5, "learning_rate": 2.75e-06, "loss": 0.9184, "step": 31 }, { "epoch": 16.0, "learning_rate": 2.736842105263158e-06, "loss": 0.9095, "step": 32 }, { "epoch": 16.5, "learning_rate": 2.723684210526316e-06, "loss": 0.913, "step": 33 }, { "epoch": 17.0, "learning_rate": 2.7105263157894737e-06, "loss": 0.9078, "step": 34 }, { "epoch": 17.5, "learning_rate": 2.6973684210526316e-06, "loss": 0.9012, "step": 35 }, { "epoch": 18.0, "learning_rate": 2.6842105263157895e-06, "loss": 0.9088, "step": 36 }, { "epoch": 18.5, "learning_rate": 2.6710526315789474e-06, "loss": 0.8996, "step": 37 }, { "epoch": 19.0, "learning_rate": 2.6578947368421053e-06, "loss": 0.8885, "step": 38 }, { "epoch": 19.5, "learning_rate": 2.644736842105263e-06, "loss": 0.8754, "step": 39 }, { "epoch": 20.0, "learning_rate": 2.631578947368421e-06, "loss": 0.9052, "step": 40 }, { "epoch": 20.0, "eval_accuracy_dropoff": 0.7970503910640152, "eval_accuracy_undropoff": 0.7351326448266975, "eval_accuracy_unlabeled": NaN, "eval_iou_dropoff": 0.11461529175767018, "eval_iou_undropoff": 0.7289714259214126, "eval_iou_unlabeled": 0.0, "eval_loss": 0.8427202105522156, "eval_mean_accuracy": 0.7660915179453563, "eval_mean_iou": 0.28119557255969424, "eval_overall_accuracy": 0.7377071380615234, "eval_runtime": 3.0382, "eval_samples_per_second": 6.583, "eval_steps_per_second": 0.658, "step": 40 }, { "epoch": 20.5, "learning_rate": 2.618421052631579e-06, "loss": 0.8779, "step": 41 }, { "epoch": 21.0, "learning_rate": 2.605263157894737e-06, "loss": 0.8824, "step": 42 }, { "epoch": 21.5, "learning_rate": 2.592105263157895e-06, "loss": 0.8687, "step": 43 }, { "epoch": 22.0, "learning_rate": 2.5789473684210527e-06, "loss": 0.8856, "step": 44 }, { "epoch": 22.5, "learning_rate": 2.5657894736842107e-06, "loss": 0.8608, "step": 45 }, { "epoch": 23.0, "learning_rate": 2.5526315789473686e-06, "loss": 0.8797, "step": 46 }, { "epoch": 23.5, "learning_rate": 2.5394736842105265e-06, "loss": 0.8685, "step": 47 }, { "epoch": 24.0, "learning_rate": 2.526315789473684e-06, "loss": 0.8506, "step": 48 }, { "epoch": 24.5, "learning_rate": 2.5131578947368423e-06, "loss": 0.8546, "step": 49 }, { "epoch": 25.0, "learning_rate": 2.5e-06, "loss": 0.8555, "step": 50 }, { "epoch": 25.0, "eval_accuracy_dropoff": 0.7747884125782701, "eval_accuracy_undropoff": 0.7906153076140051, "eval_accuracy_unlabeled": NaN, "eval_iou_dropoff": 0.13566559274779452, "eval_iou_undropoff": 0.7832490132868888, "eval_iou_unlabeled": 0.0, "eval_loss": 0.7969633340835571, "eval_mean_accuracy": 0.7827018600961376, "eval_mean_iou": 0.30630486867822776, "eval_overall_accuracy": 0.7899572372436523, "eval_runtime": 2.2898, "eval_samples_per_second": 8.734, "eval_steps_per_second": 0.873, "step": 50 }, { "epoch": 25.5, "learning_rate": 2.486842105263158e-06, "loss": 0.8414, "step": 51 }, { "epoch": 26.0, "learning_rate": 2.473684210526316e-06, "loss": 0.8424, "step": 52 }, { "epoch": 26.5, "learning_rate": 2.460526315789474e-06, "loss": 0.8449, "step": 53 }, { "epoch": 27.0, "learning_rate": 2.4473684210526314e-06, "loss": 0.84, "step": 54 }, { "epoch": 27.5, "learning_rate": 2.4342105263157893e-06, "loss": 0.8236, "step": 55 }, { "epoch": 28.0, "learning_rate": 2.4210526315789472e-06, "loss": 0.8379, "step": 56 }, { "epoch": 28.5, "learning_rate": 2.4078947368421056e-06, "loss": 0.8143, "step": 57 }, { "epoch": 29.0, "learning_rate": 2.3947368421052635e-06, "loss": 0.8481, "step": 58 }, { "epoch": 29.5, "learning_rate": 2.3815789473684214e-06, "loss": 0.8188, "step": 59 }, { "epoch": 30.0, "learning_rate": 2.368421052631579e-06, "loss": 0.8291, "step": 60 }, { "epoch": 30.0, "eval_accuracy_dropoff": 0.7410261703250075, "eval_accuracy_undropoff": 0.8372145830203079, "eval_accuracy_unlabeled": NaN, "eval_iou_dropoff": 0.1586154483630815, "eval_iou_undropoff": 0.8281769467608451, "eval_iou_unlabeled": 0.0, "eval_loss": 0.7542923092842102, "eval_mean_accuracy": 0.7891203766726578, "eval_mean_iou": 0.3289307983746422, "eval_overall_accuracy": 0.8332151412963867, "eval_runtime": 2.7005, "eval_samples_per_second": 7.406, "eval_steps_per_second": 0.741, "step": 60 }, { "epoch": 30.5, "learning_rate": 2.3552631578947368e-06, "loss": 0.8126, "step": 61 }, { "epoch": 31.0, "learning_rate": 2.3421052631578947e-06, "loss": 0.8208, "step": 62 }, { "epoch": 31.5, "learning_rate": 2.3289473684210526e-06, "loss": 0.8139, "step": 63 }, { "epoch": 32.0, "learning_rate": 2.3157894736842105e-06, "loss": 0.7982, "step": 64 }, { "epoch": 32.5, "learning_rate": 2.302631578947369e-06, "loss": 0.8028, "step": 65 }, { "epoch": 33.0, "learning_rate": 2.2894736842105263e-06, "loss": 0.8027, "step": 66 }, { "epoch": 33.5, "learning_rate": 2.2763157894736842e-06, "loss": 0.8009, "step": 67 }, { "epoch": 34.0, "learning_rate": 2.263157894736842e-06, "loss": 0.7866, "step": 68 }, { "epoch": 34.5, "learning_rate": 2.25e-06, "loss": 0.7977, "step": 69 }, { "epoch": 35.0, "learning_rate": 2.236842105263158e-06, "loss": 0.7923, "step": 70 }, { "epoch": 35.0, "eval_accuracy_dropoff": 0.7404940480286245, "eval_accuracy_undropoff": 0.8517160094211111, "eval_accuracy_unlabeled": NaN, "eval_iou_dropoff": 0.17008041268396298, "eval_iou_undropoff": 0.8425017633425929, "eval_iou_unlabeled": 0.0, "eval_loss": 0.7326571345329285, "eval_mean_accuracy": 0.7961050287248679, "eval_mean_iou": 0.33752739200885196, "eval_overall_accuracy": 0.8470914840698243, "eval_runtime": 2.5567, "eval_samples_per_second": 7.823, "eval_steps_per_second": 0.782, "step": 70 }, { "epoch": 35.5, "learning_rate": 2.223684210526316e-06, "loss": 0.7766, "step": 71 }, { "epoch": 36.0, "learning_rate": 2.2105263157894738e-06, "loss": 0.7847, "step": 72 }, { "epoch": 36.5, "learning_rate": 2.1973684210526313e-06, "loss": 0.7581, "step": 73 }, { "epoch": 37.0, "learning_rate": 2.1842105263157896e-06, "loss": 0.8063, "step": 74 }, { "epoch": 37.5, "learning_rate": 2.1710526315789475e-06, "loss": 0.7767, "step": 75 }, { "epoch": 38.0, "learning_rate": 2.1578947368421054e-06, "loss": 0.7536, "step": 76 }, { "epoch": 38.5, "learning_rate": 2.1447368421052633e-06, "loss": 0.7688, "step": 77 }, { "epoch": 39.0, "learning_rate": 2.1315789473684212e-06, "loss": 0.7566, "step": 78 }, { "epoch": 39.5, "learning_rate": 2.1184210526315787e-06, "loss": 0.7464, "step": 79 }, { "epoch": 40.0, "learning_rate": 2.1052631578947366e-06, "loss": 0.7724, "step": 80 }, { "epoch": 40.0, "eval_accuracy_dropoff": 0.7148650198399046, "eval_accuracy_undropoff": 0.8786919501640336, "eval_accuracy_unlabeled": NaN, "eval_iou_dropoff": 0.1906032174696887, "eval_iou_undropoff": 0.8682203681454491, "eval_iou_unlabeled": 0.0, "eval_loss": 0.6994231343269348, "eval_mean_accuracy": 0.796778485001969, "eval_mean_iou": 0.3529411952050459, "eval_overall_accuracy": 0.8718801498413086, "eval_runtime": 2.2913, "eval_samples_per_second": 8.729, "eval_steps_per_second": 0.873, "step": 80 }, { "epoch": 40.5, "learning_rate": 2.0921052631578945e-06, "loss": 0.7366, "step": 81 }, { "epoch": 41.0, "learning_rate": 2.078947368421053e-06, "loss": 0.7775, "step": 82 }, { "epoch": 41.5, "learning_rate": 2.0657894736842108e-06, "loss": 0.75, "step": 83 }, { "epoch": 42.0, "learning_rate": 2.0526315789473687e-06, "loss": 0.7481, "step": 84 }, { "epoch": 42.5, "learning_rate": 2.039473684210526e-06, "loss": 0.7443, "step": 85 }, { "epoch": 43.0, "learning_rate": 2.026315789473684e-06, "loss": 0.7342, "step": 86 }, { "epoch": 43.5, "learning_rate": 2.013157894736842e-06, "loss": 0.7305, "step": 87 }, { "epoch": 44.0, "learning_rate": 2e-06, "loss": 0.7546, "step": 88 }, { "epoch": 44.5, "learning_rate": 1.986842105263158e-06, "loss": 0.7398, "step": 89 }, { "epoch": 45.0, "learning_rate": 1.973684210526316e-06, "loss": 0.7215, "step": 90 }, { "epoch": 45.0, "eval_accuracy_dropoff": 0.6824101470217206, "eval_accuracy_undropoff": 0.9046774204782796, "eval_accuracy_unlabeled": NaN, "eval_iou_dropoff": 0.2156646138501328, "eval_iou_undropoff": 0.8926325561996815, "eval_iou_unlabeled": 0.0, "eval_loss": 0.6674811840057373, "eval_mean_accuracy": 0.7935437837500001, "eval_mean_iou": 0.3694323900166048, "eval_overall_accuracy": 0.8954357147216797, "eval_runtime": 2.2961, "eval_samples_per_second": 8.711, "eval_steps_per_second": 0.871, "step": 90 }, { "epoch": 45.5, "learning_rate": 1.9605263157894736e-06, "loss": 0.7146, "step": 91 }, { "epoch": 46.0, "learning_rate": 1.9473684210526315e-06, "loss": 0.7334, "step": 92 }, { "epoch": 46.5, "learning_rate": 1.9342105263157895e-06, "loss": 0.7439, "step": 93 }, { "epoch": 47.0, "learning_rate": 1.9210526315789474e-06, "loss": 0.6884, "step": 94 }, { "epoch": 47.5, "learning_rate": 1.9078947368421053e-06, "loss": 0.7154, "step": 95 }, { "epoch": 48.0, "learning_rate": 1.8947368421052632e-06, "loss": 0.7148, "step": 96 }, { "epoch": 48.5, "learning_rate": 1.8815789473684209e-06, "loss": 0.7036, "step": 97 }, { "epoch": 49.0, "learning_rate": 1.8684210526315792e-06, "loss": 0.7242, "step": 98 }, { "epoch": 49.5, "learning_rate": 1.855263157894737e-06, "loss": 0.7023, "step": 99 }, { "epoch": 50.0, "learning_rate": 1.8421052631578948e-06, "loss": 0.6907, "step": 100 }, { "epoch": 50.0, "eval_accuracy_dropoff": 0.6903919814674648, "eval_accuracy_undropoff": 0.9091346369120885, "eval_accuracy_unlabeled": NaN, "eval_iou_dropoff": 0.22523428544063023, "eval_iou_undropoff": 0.8973130482009237, "eval_iou_unlabeled": 0.0, "eval_loss": 0.6520694494247437, "eval_mean_accuracy": 0.7997633091897767, "eval_mean_iou": 0.37418244454718463, "eval_overall_accuracy": 0.9000394821166993, "eval_runtime": 2.3891, "eval_samples_per_second": 8.371, "eval_steps_per_second": 0.837, "step": 100 }, { "epoch": 50.5, "learning_rate": 1.8289473684210527e-06, "loss": 0.6913, "step": 101 }, { "epoch": 51.0, "learning_rate": 1.8157894736842106e-06, "loss": 0.6938, "step": 102 }, { "epoch": 51.5, "learning_rate": 1.8026315789473683e-06, "loss": 0.668, "step": 103 }, { "epoch": 52.0, "learning_rate": 1.7894736842105262e-06, "loss": 0.7141, "step": 104 }, { "epoch": 52.5, "learning_rate": 1.7763157894736842e-06, "loss": 0.6951, "step": 105 }, { "epoch": 53.0, "learning_rate": 1.7631578947368423e-06, "loss": 0.6731, "step": 106 }, { "epoch": 53.5, "learning_rate": 1.7500000000000002e-06, "loss": 0.6823, "step": 107 }, { "epoch": 54.0, "learning_rate": 1.736842105263158e-06, "loss": 0.6726, "step": 108 }, { "epoch": 54.5, "learning_rate": 1.7236842105263158e-06, "loss": 0.6788, "step": 109 }, { "epoch": 55.0, "learning_rate": 1.7105263157894737e-06, "loss": 0.6768, "step": 110 }, { "epoch": 55.0, "eval_accuracy_dropoff": 0.6827450170875479, "eval_accuracy_undropoff": 0.9217414925913727, "eval_accuracy_unlabeled": NaN, "eval_iou_dropoff": 0.24547430712697668, "eval_iou_undropoff": 0.9094276508794655, "eval_iou_unlabeled": 0.0, "eval_loss": 0.625967800617218, "eval_mean_accuracy": 0.8022432548394602, "eval_mean_iou": 0.38496731933548073, "eval_overall_accuracy": 0.91180419921875, "eval_runtime": 2.383, "eval_samples_per_second": 8.393, "eval_steps_per_second": 0.839, "step": 110 }, { "epoch": 55.5, "learning_rate": 1.6973684210526316e-06, "loss": 0.6745, "step": 111 }, { "epoch": 56.0, "learning_rate": 1.6842105263157895e-06, "loss": 0.6584, "step": 112 }, { "epoch": 56.5, "learning_rate": 1.6710526315789472e-06, "loss": 0.6862, "step": 113 }, { "epoch": 57.0, "learning_rate": 1.6578947368421056e-06, "loss": 0.6477, "step": 114 }, { "epoch": 57.5, "learning_rate": 1.6447368421052632e-06, "loss": 0.6478, "step": 115 }, { "epoch": 58.0, "learning_rate": 1.6315789473684212e-06, "loss": 0.6812, "step": 116 }, { "epoch": 58.5, "learning_rate": 1.618421052631579e-06, "loss": 0.6602, "step": 117 }, { "epoch": 59.0, "learning_rate": 1.605263157894737e-06, "loss": 0.6479, "step": 118 }, { "epoch": 59.5, "learning_rate": 1.5921052631578947e-06, "loss": 0.6531, "step": 119 }, { "epoch": 60.0, "learning_rate": 1.5789473684210526e-06, "loss": 0.659, "step": 120 }, { "epoch": 60.0, "eval_accuracy_dropoff": 0.6586389596091653, "eval_accuracy_undropoff": 0.9358954483535444, "eval_accuracy_unlabeled": NaN, "eval_iou_dropoff": 0.2670917911780463, "eval_iou_undropoff": 0.9224199158424202, "eval_iou_unlabeled": 0.0, "eval_loss": 0.6009885668754578, "eval_mean_accuracy": 0.7972672039813549, "eval_mean_iou": 0.39650390234015553, "eval_overall_accuracy": 0.9243673324584961, "eval_runtime": 2.3866, "eval_samples_per_second": 8.38, "eval_steps_per_second": 0.838, "step": 120 } ], "max_steps": 240, "num_train_epochs": 120, "total_flos": 6.945599099555021e+17, "trial_name": null, "trial_params": null }