Training in progress, epoch 1

all_results.json

@@ -1,17 +1,17 @@
 {
-    "epoch": 12.0,
+    "epoch": 15.0,
     "eval_AUC": 0.5,
-    "eval_F1": 0.47572815533980584,
-    "eval_Precision": 0.9074074074074074,
+    "eval_F1": 0.47540983606557374,
+    "eval_Precision": 0.90625,
     "eval_Recall": 1.0,
-    "eval_accuracy": 0.9074074074074074,
-    "eval_loss": 0.3346473276615143,
-    "eval_runtime": 5.474,
-    "eval_samples_per_second": 59.189,
-    "eval_steps_per_second": 3.836,
-    "total_flos": 6.862690600268268e+17,
-    "train_loss": 0.03526242822408676,
-    "train_runtime": 251.233,
-    "train_samples_per_second": 35.25,
-    "train_steps_per_second": 2.245
+    "eval_accuracy": 0.90625,
+    "eval_loss": 0.33828961849212646,
+    "eval_runtime": 5.0159,
+    "eval_samples_per_second": 76.557,
+    "eval_steps_per_second": 4.785,
+    "total_flos": 1.0112704644704256e+18,
+    "train_loss": 0.06119063453240828,
+    "train_runtime": 257.3401,
+    "train_samples_per_second": 50.711,
+    "train_steps_per_second": 3.206
 }

eval_results.json

@@ -1,12 +1,12 @@
 {
-    "epoch": 12.0,
+    "epoch": 15.0,
     "eval_AUC": 0.5,
-    "eval_F1": 0.47572815533980584,
-    "eval_Precision": 0.9074074074074074,
+    "eval_F1": 0.47540983606557374,
+    "eval_Precision": 0.90625,
     "eval_Recall": 1.0,
-    "eval_accuracy": 0.9074074074074074,
-    "eval_loss": 0.3346473276615143,
-    "eval_runtime": 5.474,
-    "eval_samples_per_second": 59.189,
-    "eval_steps_per_second": 3.836
+    "eval_accuracy": 0.90625,
+    "eval_loss": 0.33828961849212646,
+    "eval_runtime": 5.0159,
+    "eval_samples_per_second": 76.557,
+    "eval_steps_per_second": 4.785
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:760373362287b69643cb264294c3cdca3c69105ed8734ed2b2cb181e5b3d2411
+oid sha256:4884df1de3f4f54ce6cdaeef73244807d3600eed0a76713c72478f0fb60cd851
 size 343223968

runs/May28_16-57-52_RTX3090/events.out.tfevents.1716883342.RTX3090.462376.1

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:845cd2569a3c1adfb6a17d0eb48ba0c95677eccc9c9fb4a0c5a2ec8f0b20a465
+size 607

runs/May29_01-21-51_RTX3090/events.out.tfevents.1716913322.RTX3090.597255.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:00bba5d4ad77fbd27f6c0ab08e3f7f63a5773ce2655780c8eaff95de30eb1ccb
+size 5634

train_results.json

@@ -1,8 +1,8 @@
 {
-    "epoch": 12.0,
-    "total_flos": 6.862690600268268e+17,
-    "train_loss": 0.03526242822408676,
-    "train_runtime": 251.233,
-    "train_samples_per_second": 35.25,
-    "train_steps_per_second": 2.245
+    "epoch": 15.0,
+    "total_flos": 1.0112704644704256e+18,
+    "train_loss": 0.06119063453240828,
+    "train_runtime": 257.3401,
+    "train_samples_per_second": 50.711,
+    "train_steps_per_second": 3.206
 }

trainer_state.json

@@ -1,197 +1,236 @@
 {
-  "best_metric": 0.3346473276615143,
-  "best_model_checkpoint": "google/vit-base-patch16-224-in21k_covid_19_ct_scans/checkpoint-47",
-  "epoch": 12.0,
+  "best_metric": 0.33828961849212646,
+  "best_model_checkpoint": "google/vit-base-patch16-224-in21k_covid_19_ct_scans/checkpoint-55",
+  "epoch": 15.0,
   "eval_steps": 500,
-  "global_step": 564,
+  "global_step": 825,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
-      "epoch": 0.02127659574468085,
-      "grad_norm": 4.243431091308594,
-      "learning_rate": 0.000199645390070922,
-      "loss": 0.7767,
+      "epoch": 0.01818181818181818,
+      "grad_norm": 3.793684244155884,
+      "learning_rate": 0.00019975757575757576,
+      "loss": 0.7218,
       "step": 1
     },
     {
       "epoch": 1.0,
       "eval_AUC": 0.5,
-      "eval_F1": 0.47572815533980584,
-      "eval_Precision": 0.9074074074074074,
+      "eval_F1": 0.47540983606557374,
+      "eval_Precision": 0.90625,
       "eval_Recall": 1.0,
-      "eval_accuracy": 0.9074074074074074,
-      "eval_loss": 0.3346473276615143,
-      "eval_runtime": 5.5299,
-      "eval_samples_per_second": 58.591,
-      "eval_steps_per_second": 3.798,
-      "step": 47
+      "eval_accuracy": 0.90625,
+      "eval_loss": 0.33828961849212646,
+      "eval_runtime": 4.7492,
+      "eval_samples_per_second": 80.855,
+      "eval_steps_per_second": 5.053,
+      "step": 55
     },
     {
       "epoch": 2.0,
-      "eval_AUC": 0.6204081632653061,
-      "eval_F1": 0.5919028340080972,
-      "eval_Precision": 0.9311594202898551,
-      "eval_Recall": 0.8741496598639455,
-      "eval_accuracy": 0.8271604938271605,
-      "eval_loss": 0.5513330698013306,
-      "eval_runtime": 5.5902,
-      "eval_samples_per_second": 57.959,
-      "eval_steps_per_second": 3.757,
-      "step": 94
+      "eval_AUC": 0.5,
+      "eval_F1": 0.47540983606557374,
+      "eval_Precision": 0.90625,
+      "eval_Recall": 1.0,
+      "eval_accuracy": 0.90625,
+      "eval_loss": 0.382261723279953,
+      "eval_runtime": 4.831,
+      "eval_samples_per_second": 79.486,
+      "eval_steps_per_second": 4.968,
+      "step": 110
     },
     {
       "epoch": 3.0,
       "eval_AUC": 0.5,
-      "eval_F1": 0.47572815533980584,
-      "eval_Precision": 0.9074074074074074,
+      "eval_F1": 0.47540983606557374,
+      "eval_Precision": 0.90625,
       "eval_Recall": 1.0,
-      "eval_accuracy": 0.9074074074074074,
-      "eval_loss": 0.4290040135383606,
-      "eval_runtime": 5.491,
-      "eval_samples_per_second": 59.005,
-      "eval_steps_per_second": 3.824,
-      "step": 141
+      "eval_accuracy": 0.90625,
+      "eval_loss": 0.3956604301929474,
+      "eval_runtime": 4.8897,
+      "eval_samples_per_second": 78.532,
+      "eval_steps_per_second": 4.908,
+      "step": 165
     },
     {
       "epoch": 4.0,
-      "eval_AUC": 0.5965986394557823,
-      "eval_F1": 0.6365832614322692,
-      "eval_Precision": 0.9240506329113924,
-      "eval_Recall": 0.9931972789115646,
-      "eval_accuracy": 0.9197530864197531,
-      "eval_loss": 0.433300256729126,
-      "eval_runtime": 5.3718,
-      "eval_samples_per_second": 60.315,
-      "eval_steps_per_second": 3.909,
-      "step": 188
+      "eval_AUC": 0.5,
+      "eval_F1": 0.47540983606557374,
+      "eval_Precision": 0.90625,
+      "eval_Recall": 1.0,
+      "eval_accuracy": 0.90625,
+      "eval_loss": 0.44851580262184143,
+      "eval_runtime": 4.8632,
+      "eval_samples_per_second": 78.96,
+      "eval_steps_per_second": 4.935,
+      "step": 220
     },
     {
       "epoch": 5.0,
-      "eval_AUC": 0.5897959183673469,
-      "eval_F1": 0.618104667609618,
-      "eval_Precision": 0.9230769230769231,
-      "eval_Recall": 0.9795918367346939,
-      "eval_accuracy": 0.9074074074074074,
-      "eval_loss": 0.5040868520736694,
-      "eval_runtime": 5.759,
-      "eval_samples_per_second": 56.26,
-      "eval_steps_per_second": 3.646,
-      "step": 235
+      "eval_AUC": 0.4942528735632184,
+      "eval_F1": 0.4725274725274725,
+      "eval_Precision": 0.9052631578947369,
+      "eval_Recall": 0.9885057471264368,
+      "eval_accuracy": 0.8958333333333334,
+      "eval_loss": 0.4786018133163452,
+      "eval_runtime": 4.8854,
+      "eval_samples_per_second": 78.601,
+      "eval_steps_per_second": 4.913,
+      "step": 275
     },
     {
       "epoch": 6.0,
-      "eval_AUC": 0.5948979591836735,
-      "eval_F1": 0.6316786661614249,
-      "eval_Precision": 0.9238095238095239,
-      "eval_Recall": 0.9897959183673469,
-      "eval_accuracy": 0.9166666666666666,
-      "eval_loss": 0.4848092496395111,
-      "eval_runtime": 13.7532,
-      "eval_samples_per_second": 23.558,
-      "eval_steps_per_second": 1.527,
-      "step": 282
+      "eval_AUC": 0.49712643678160917,
+      "eval_F1": 0.473972602739726,
+      "eval_Precision": 0.9057591623036649,
+      "eval_Recall": 0.9942528735632183,
+      "eval_accuracy": 0.9010416666666666,
+      "eval_loss": 0.5316488146781921,
+      "eval_runtime": 5.3191,
+      "eval_samples_per_second": 72.193,
+      "eval_steps_per_second": 4.512,
+      "step": 330
     },
     {
       "epoch": 7.0,
-      "eval_AUC": 0.5965986394557823,
-      "eval_F1": 0.6365832614322692,
-      "eval_Precision": 0.9240506329113924,
-      "eval_Recall": 0.9931972789115646,
-      "eval_accuracy": 0.9197530864197531,
-      "eval_loss": 0.48772257566452026,
-      "eval_runtime": 5.5683,
-      "eval_samples_per_second": 58.186,
-      "eval_steps_per_second": 3.771,
-      "step": 329
+      "eval_AUC": 0.49712643678160917,
+      "eval_F1": 0.473972602739726,
+      "eval_Precision": 0.9057591623036649,
+      "eval_Recall": 0.9942528735632183,
+      "eval_accuracy": 0.9010416666666666,
+      "eval_loss": 0.553883969783783,
+      "eval_runtime": 4.8592,
+      "eval_samples_per_second": 79.026,
+      "eval_steps_per_second": 4.939,
+      "step": 385
     },
     {
       "epoch": 8.0,
-      "eval_AUC": 0.5965986394557823,
-      "eval_F1": 0.6365832614322692,
-      "eval_Precision": 0.9240506329113924,
-      "eval_Recall": 0.9931972789115646,
-      "eval_accuracy": 0.9197530864197531,
-      "eval_loss": 0.49257075786590576,
-      "eval_runtime": 5.2567,
-      "eval_samples_per_second": 61.635,
-      "eval_steps_per_second": 3.995,
-      "step": 376
+      "eval_AUC": 0.49712643678160917,
+      "eval_F1": 0.473972602739726,
+      "eval_Precision": 0.9057591623036649,
+      "eval_Recall": 0.9942528735632183,
+      "eval_accuracy": 0.9010416666666666,
+      "eval_loss": 0.5799620747566223,
+      "eval_runtime": 4.7649,
+      "eval_samples_per_second": 80.589,
+      "eval_steps_per_second": 5.037,
+      "step": 440
     },
     {
       "epoch": 9.0,
-      "eval_AUC": 0.5965986394557823,
-      "eval_F1": 0.6365832614322692,
-      "eval_Precision": 0.9240506329113924,
-      "eval_Recall": 0.9931972789115646,
-      "eval_accuracy": 0.9197530864197531,
-      "eval_loss": 0.49581947922706604,
-      "eval_runtime": 5.6218,
-      "eval_samples_per_second": 57.633,
-      "eval_steps_per_second": 3.735,
-      "step": 423
+      "eval_AUC": 0.49712643678160917,
+      "eval_F1": 0.473972602739726,
+      "eval_Precision": 0.9057591623036649,
+      "eval_Recall": 0.9942528735632183,
+      "eval_accuracy": 0.9010416666666666,
+      "eval_loss": 0.5976574420928955,
+      "eval_runtime": 4.7034,
+      "eval_samples_per_second": 81.644,
+      "eval_steps_per_second": 5.103,
+      "step": 495
     },
     {
-      "epoch": 10.0,
-      "eval_AUC": 0.5965986394557823,
-      "eval_F1": 0.6365832614322692,
-      "eval_Precision": 0.9240506329113924,
-      "eval_Recall": 0.9931972789115646,
-      "eval_accuracy": 0.9197530864197531,
-      "eval_loss": 0.4980551600456238,
-      "eval_runtime": 5.5112,
-      "eval_samples_per_second": 58.79,
-      "eval_steps_per_second": 3.81,
-      "step": 470
-    },
-    {
-      "epoch": 10.638297872340425,
-      "grad_norm": 0.012480547651648521,
-      "learning_rate": 2.269503546099291e-05,
-      "loss": 0.0381,
+      "epoch": 9.090909090909092,
+      "grad_norm": 0.018258079886436462,
+      "learning_rate": 7.878787878787879e-05,
+      "loss": 0.0987,
       "step": 500
     },
+    {
+      "epoch": 10.0,
+      "eval_AUC": 0.49712643678160917,
+      "eval_F1": 0.473972602739726,
+      "eval_Precision": 0.9057591623036649,
+      "eval_Recall": 0.9942528735632183,
+      "eval_accuracy": 0.9010416666666666,
+      "eval_loss": 0.6109693646430969,
+      "eval_runtime": 4.7422,
+      "eval_samples_per_second": 80.974,
+      "eval_steps_per_second": 5.061,
+      "step": 550
+    },
     {
       "epoch": 11.0,
-      "eval_AUC": 0.5965986394557823,
-      "eval_F1": 0.6365832614322692,
-      "eval_Precision": 0.9240506329113924,
-      "eval_Recall": 0.9931972789115646,
-      "eval_accuracy": 0.9197530864197531,
-      "eval_loss": 0.4996081292629242,
-      "eval_runtime": 5.5015,
-      "eval_samples_per_second": 58.893,
-      "eval_steps_per_second": 3.817,
-      "step": 517
+      "eval_AUC": 0.49712643678160917,
+      "eval_F1": 0.473972602739726,
+      "eval_Precision": 0.9057591623036649,
+      "eval_Recall": 0.9942528735632183,
+      "eval_accuracy": 0.9010416666666666,
+      "eval_loss": 0.6211499571800232,
+      "eval_runtime": 4.7723,
+      "eval_samples_per_second": 80.464,
+      "eval_steps_per_second": 5.029,
+      "step": 605
     },
     {
       "epoch": 12.0,
-      "eval_AUC": 0.5965986394557823,
-      "eval_F1": 0.6365832614322692,
-      "eval_Precision": 0.9240506329113924,
-      "eval_Recall": 0.9931972789115646,
-      "eval_accuracy": 0.9197530864197531,
-      "eval_loss": 0.5001835823059082,
-      "eval_runtime": 5.5423,
-      "eval_samples_per_second": 58.46,
-      "eval_steps_per_second": 3.789,
-      "step": 564
+      "eval_AUC": 0.49712643678160917,
+      "eval_F1": 0.473972602739726,
+      "eval_Precision": 0.9057591623036649,
+      "eval_Recall": 0.9942528735632183,
+      "eval_accuracy": 0.9010416666666666,
+      "eval_loss": 0.6287771463394165,
+      "eval_runtime": 4.9768,
+      "eval_samples_per_second": 77.159,
+      "eval_steps_per_second": 4.822,
+      "step": 660
     },
     {
-      "epoch": 12.0,
-      "step": 564,
-      "total_flos": 6.862690600268268e+17,
-      "train_loss": 0.03526242822408676,
-      "train_runtime": 251.233,
-      "train_samples_per_second": 35.25,
-      "train_steps_per_second": 2.245
+      "epoch": 13.0,
+      "eval_AUC": 0.49712643678160917,
+      "eval_F1": 0.473972602739726,
+      "eval_Precision": 0.9057591623036649,
+      "eval_Recall": 0.9942528735632183,
+      "eval_accuracy": 0.9010416666666666,
+      "eval_loss": 0.6341301798820496,
+      "eval_runtime": 4.7422,
+      "eval_samples_per_second": 80.974,
+      "eval_steps_per_second": 5.061,
+      "step": 715
+    },
+    {
+      "epoch": 14.0,
+      "eval_AUC": 0.49712643678160917,
+      "eval_F1": 0.473972602739726,
+      "eval_Precision": 0.9057591623036649,
+      "eval_Recall": 0.9942528735632183,
+      "eval_accuracy": 0.9010416666666666,
+      "eval_loss": 0.6373991370201111,
+      "eval_runtime": 4.8768,
+      "eval_samples_per_second": 78.74,
+      "eval_steps_per_second": 4.921,
+      "step": 770
+    },
+    {
+      "epoch": 15.0,
+      "eval_AUC": 0.49712643678160917,
+      "eval_F1": 0.473972602739726,
+      "eval_Precision": 0.9057591623036649,
+      "eval_Recall": 0.9942528735632183,
+      "eval_accuracy": 0.9010416666666666,
+      "eval_loss": 0.6385161280632019,
+      "eval_runtime": 4.6939,
+      "eval_samples_per_second": 81.808,
+      "eval_steps_per_second": 5.113,
+      "step": 825
+    },
+    {
+      "epoch": 15.0,
+      "step": 825,
+      "total_flos": 1.0112704644704256e+18,
+      "train_loss": 0.06119063453240828,
+      "train_runtime": 257.3401,
+      "train_samples_per_second": 50.711,
+      "train_steps_per_second": 3.206
     }
   ],
   "logging_steps": 500,
-  "max_steps": 564,
+  "max_steps": 825,
   "num_input_tokens_seen": 0,
-  "num_train_epochs": 12,
+  "num_train_epochs": 15,
   "save_steps": 500,
   "stateful_callbacks": {
     "TrainerControl": {
@@ -205,7 +244,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 6.862690600268268e+17,
+  "total_flos": 1.0112704644704256e+18,
   "train_batch_size": 16,
   "trial_name": null,
   "trial_params": null

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:860ac9cc2700a6acca98170286a713aa05b047089895c0cc3d2037638dcd625e
+oid sha256:f570a2a2407a22a4528fa092d0409008681075d1ae3fba96df20687f008eccf6
 size 4731