Training in progress, epoch 60, checkpoint

last-checkpoint/model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9d8e3908d4c24a5c3ae195e58d5883214734533f624cbaaf5bf57b362ddc57e2
+oid sha256:f383c6a569aa1757f42c068466838ac2ad34601c735c42e76c4b0eccec7ec251
 size 166496880

last-checkpoint/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3694a0f8e0e6ba5d58eed8663d397675a566634b850069de2ff56c27472be6be
+oid sha256:7985126ab91293b3d360f1dc2d798595b418d44dce13ead2ff7d367e826b6e16
 size 330495866

last-checkpoint/rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7021b7f290ce2ecee8960d7d8842b39007015ec1c207fbea7eb79b547a00e6ea
+oid sha256:521682efd6d19f4a9e6242718069a7e8eb91340f3ce03656684bc97ecd02f9fa
 size 14244

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:cbb398868d9495d1d98703c67e6d962238241ad82fc84d6f95dd75ff664b33ec
+oid sha256:6c5b71df2955d83314469d7d171689f62e78d85dd182a8d60c5a5ca2b526ce72
 size 1064

last-checkpoint/trainer_state.json

@@ -1,9 +1,9 @@
 {
   "best_metric": 0.28454330563545227,
   "best_model_checkpoint": "joe611/chickens-60-epoch-1000-images-aug/checkpoint-28500",
-  "epoch": 59.0,
+  "epoch": 60.0,
   "eval_steps": 500,
-  "global_step": 29500,
+  "global_step": 30000,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -8422,6 +8422,151 @@
       "eval_samples_per_second": 17.921,
       "eval_steps_per_second": 2.33,
       "step": 29500
+    },
+    {
+      "epoch": 59.04,
+      "grad_norm": 29.91386604309082,
+      "learning_rate": 6.315216969912663e-09,
+      "loss": 0.3851,
+      "step": 29520
+    },
+    {
+      "epoch": 59.1,
+      "grad_norm": 505.2102355957031,
+      "learning_rate": 5.5506251901504825e-09,
+      "loss": 0.3666,
+      "step": 29550
+    },
+    {
+      "epoch": 59.16,
+      "grad_norm": 45.7656135559082,
+      "learning_rate": 4.835326609376468e-09,
+      "loss": 0.4178,
+      "step": 29580
+    },
+    {
+      "epoch": 59.22,
+      "grad_norm": 56.061153411865234,
+      "learning_rate": 4.169328287299545e-09,
+      "loss": 0.4697,
+      "step": 29610
+    },
+    {
+      "epoch": 59.28,
+      "grad_norm": 93.67945098876953,
+      "learning_rate": 3.5526367970539765e-09,
+      "loss": 0.3581,
+      "step": 29640
+    },
+    {
+      "epoch": 59.34,
+      "grad_norm": 56.09961700439453,
+      "learning_rate": 2.9852582251355124e-09,
+      "loss": 0.4007,
+      "step": 29670
+    },
+    {
+      "epoch": 59.4,
+      "grad_norm": 95.72340393066406,
+      "learning_rate": 2.4671981713420003e-09,
+      "loss": 0.4338,
+      "step": 29700
+    },
+    {
+      "epoch": 59.46,
+      "grad_norm": 73.806884765625,
+      "learning_rate": 1.9984617487173174e-09,
+      "loss": 0.4488,
+      "step": 29730
+    },
+    {
+      "epoch": 59.52,
+      "grad_norm": 63.850746154785156,
+      "learning_rate": 1.5790535835003006e-09,
+      "loss": 0.4424,
+      "step": 29760
+    },
+    {
+      "epoch": 59.58,
+      "grad_norm": 54.269371032714844,
+      "learning_rate": 1.2089778150797816e-09,
+      "loss": 0.4145,
+      "step": 29790
+    },
+    {
+      "epoch": 59.64,
+      "grad_norm": 45.143714904785156,
+      "learning_rate": 8.88238095955174e-10,
+      "loss": 0.4422,
+      "step": 29820
+    },
+    {
+      "epoch": 59.7,
+      "grad_norm": 32.879642486572266,
+      "learning_rate": 6.168375916970615e-10,
+      "loss": 0.3915,
+      "step": 29850
+    },
+    {
+      "epoch": 59.76,
+      "grad_norm": 68.58180236816406,
+      "learning_rate": 3.9477898091944135e-10,
+      "loss": 0.423,
+      "step": 29880
+    },
+    {
+      "epoch": 59.82,
+      "grad_norm": 57.879634857177734,
+      "learning_rate": 2.2206445525085886e-10,
+      "loss": 0.4619,
+      "step": 29910
+    },
+    {
+      "epoch": 59.88,
+      "grad_norm": 133.5286102294922,
+      "learning_rate": 9.869571931442334e-11,
+      "loss": 0.3987,
+      "step": 29940
+    },
+    {
+      "epoch": 59.94,
+      "grad_norm": 63.78921127319336,
+      "learning_rate": 2.467399070893439e-11,
+      "loss": 0.3533,
+      "step": 29970
+    },
+    {
+      "epoch": 60.0,
+      "grad_norm": 65.49594116210938,
+      "learning_rate": 0.0,
+      "loss": 0.4504,
+      "step": 30000
+    },
+    {
+      "epoch": 60.0,
+      "eval_loss": 0.28507041931152344,
+      "eval_map": 0.8003,
+      "eval_map_50": 0.9645,
+      "eval_map_75": 0.9196,
+      "eval_map_chicken": 0.8072,
+      "eval_map_duck": 0.7689,
+      "eval_map_large": 0.8772,
+      "eval_map_medium": 0.7999,
+      "eval_map_plant": 0.8248,
+      "eval_map_small": 0.2246,
+      "eval_mar_1": 0.3086,
+      "eval_mar_10": 0.8346,
+      "eval_mar_100": 0.8384,
+      "eval_mar_100_chicken": 0.8413,
+      "eval_mar_100_duck": 0.799,
+      "eval_mar_100_plant": 0.8749,
+      "eval_mar_large": 0.918,
+      "eval_mar_medium": 0.8496,
+      "eval_mar_small": 0.3614,
+      "eval_runtime": 5.2733,
+      "eval_samples_per_second": 18.963,
+      "eval_steps_per_second": 2.465,
+      "step": 30000
     }
   ],
   "logging_steps": 30,
@@ -8436,12 +8581,12 @@
         "should_evaluate": false,
         "should_log": false,
         "should_save": true,
-        "should_training_stop": false
+        "should_training_stop": true
       },
       "attributes": {}
     }
   },
-  "total_flos": 1.01486818308096e+19,
+  "total_flos": 1.0320693387264e+19,
   "train_batch_size": 2,
   "trial_name": null,
   "trial_params": null