Training in progress, epoch 0

all_results.json

@@ -1,8 +1,8 @@
 {
-    "epoch": 19.2,
-    "total_flos": 6.063680104641331e+17,
-    "train_loss": 0.059417286076692714,
-    "train_runtime": 389.1501,
-    "train_samples_per_second": 20.558,
-    "train_steps_per_second": 0.308
+    "epoch": 18.46153846153846,
+    "total_flos": 6.035309694497341e+17,
+    "train_loss": 0.09444785690047866,
+    "train_runtime": 374.1698,
+    "train_samples_per_second": 22.129,
+    "train_steps_per_second": 0.321
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2838fadb867240e176028870f536aa974297d04766f71557ecd1fe9ab88467e2
+oid sha256:285b88f17873b74a93a630cf43553eacf28648e1c373dbccbea976e9a04636a6
 size 349857196

runs/Jul03_12-38-35_e0133a370a2e/events.out.tfevents.1720010332.e0133a370a2e.34.2

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:82ba278461f45a4f201bb17c5aff573cb89f27ed28dba6f5b725949e8705223f
+size 5170

train_results.json

@@ -1,8 +1,8 @@
 {
-    "epoch": 19.2,
-    "total_flos": 6.063680104641331e+17,
-    "train_loss": 0.059417286076692714,
-    "train_runtime": 389.1501,
-    "train_samples_per_second": 20.558,
-    "train_steps_per_second": 0.308
+    "epoch": 18.46153846153846,
+    "total_flos": 6.035309694497341e+17,
+    "train_loss": 0.09444785690047866,
+    "train_runtime": 374.1698,
+    "train_samples_per_second": 22.129,
+    "train_steps_per_second": 0.321
 }

trainer_state.json

@@ -1,7 +1,7 @@
 {
   "best_metric": 1.0,
-  "best_model_checkpoint": "ktp-not-ktp-clip/checkpoint-37",
-  "epoch": 19.2,
+  "best_model_checkpoint": "ktp-not-ktp-clip/checkpoint-52",
+  "epoch": 18.46153846153846,
   "eval_steps": 500,
   "global_step": 120,
   "is_hyper_param_search": false,
@@ -9,277 +9,268 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "epoch": 0.96,
-      "eval_accuracy": 0.8829787234042553,
-      "eval_loss": 0.40863433480262756,
-      "eval_runtime": 5.5907,
-      "eval_samples_per_second": 33.627,
-      "eval_steps_per_second": 2.146,
+      "epoch": 0.9230769230769231,
+      "eval_accuracy": 0.897196261682243,
+      "eval_loss": 0.4882933795452118,
+      "eval_runtime": 6.4817,
+      "eval_samples_per_second": 33.016,
+      "eval_steps_per_second": 2.16,
       "step": 6
     },
     {
-      "epoch": 1.6,
-      "grad_norm": 9.341768264770508,
+      "epoch": 1.5384615384615383,
+      "grad_norm": 8.5100679397583,
       "learning_rate": 4.166666666666667e-05,
-      "loss": 0.4605,
+      "loss": 0.6748,
       "step": 10
     },
     {
-      "epoch": 1.92,
-      "eval_accuracy": 0.9680851063829787,
-      "eval_loss": 0.05953093245625496,
-      "eval_runtime": 4.4357,
-      "eval_samples_per_second": 42.384,
-      "eval_steps_per_second": 2.705,
-      "step": 12
+      "epoch": 2.0,
+      "eval_accuracy": 0.9719626168224299,
+      "eval_loss": 0.07545875012874603,
+      "eval_runtime": 4.9236,
+      "eval_samples_per_second": 43.464,
+      "eval_steps_per_second": 2.843,
+      "step": 13
     },
     {
-      "epoch": 2.88,
-      "eval_accuracy": 0.9946808510638298,
-      "eval_loss": 0.03500603511929512,
-      "eval_runtime": 4.3384,
-      "eval_samples_per_second": 43.334,
-      "eval_steps_per_second": 2.766,
-      "step": 18
+      "epoch": 2.9230769230769234,
+      "eval_accuracy": 0.9345794392523364,
+      "eval_loss": 0.13731282949447632,
+      "eval_runtime": 4.9997,
+      "eval_samples_per_second": 42.803,
+      "eval_steps_per_second": 2.8,
+      "step": 19
     },
     {
-      "epoch": 3.2,
-      "grad_norm": 30.609560012817383,
+      "epoch": 3.076923076923077,
+      "grad_norm": 55.80315399169922,
       "learning_rate": 4.62962962962963e-05,
-      "loss": 0.1012,
+      "loss": 0.1779,
       "step": 20
     },
     {
       "epoch": 4.0,
-      "eval_accuracy": 0.9787234042553191,
-      "eval_loss": 0.05293251574039459,
-      "eval_runtime": 4.4455,
-      "eval_samples_per_second": 42.29,
-      "eval_steps_per_second": 2.699,
-      "step": 25
+      "eval_accuracy": 0.9719626168224299,
+      "eval_loss": 0.06819602847099304,
+      "eval_runtime": 5.1079,
+      "eval_samples_per_second": 41.896,
+      "eval_steps_per_second": 2.741,
+      "step": 26
     },
     {
-      "epoch": 4.8,
-      "grad_norm": 15.118265151977539,
+      "epoch": 4.615384615384615,
+      "grad_norm": 5.013700008392334,
       "learning_rate": 4.166666666666667e-05,
-      "loss": 0.0975,
+      "loss": 0.1511,
       "step": 30
     },
     {
-      "epoch": 4.96,
-      "eval_accuracy": 0.9946808510638298,
-      "eval_loss": 0.016444995999336243,
-      "eval_runtime": 4.5421,
-      "eval_samples_per_second": 41.39,
-      "eval_steps_per_second": 2.642,
-      "step": 31
+      "epoch": 4.923076923076923,
+      "eval_accuracy": 0.9766355140186916,
+      "eval_loss": 0.03988885134458542,
+      "eval_runtime": 5.0626,
+      "eval_samples_per_second": 42.271,
+      "eval_steps_per_second": 2.765,
+      "step": 32
     },
     {
-      "epoch": 5.92,
-      "eval_accuracy": 1.0,
-      "eval_loss": 0.002412277739495039,
-      "eval_runtime": 4.4217,
-      "eval_samples_per_second": 42.518,
-      "eval_steps_per_second": 2.714,
-      "step": 37
+      "epoch": 6.0,
+      "eval_accuracy": 0.9953271028037384,
+      "eval_loss": 0.011155444197356701,
+      "eval_runtime": 5.129,
+      "eval_samples_per_second": 41.724,
+      "eval_steps_per_second": 2.73,
+      "step": 39
     },
     {
-      "epoch": 6.4,
-      "grad_norm": 0.07197532057762146,
+      "epoch": 6.153846153846154,
+      "grad_norm": 22.189523696899414,
       "learning_rate": 3.7037037037037037e-05,
-      "loss": 0.0107,
+      "loss": 0.0248,
       "step": 40
     },
     {
-      "epoch": 6.88,
-      "eval_accuracy": 0.9840425531914894,
-      "eval_loss": 0.06681745499372482,
-      "eval_runtime": 4.4511,
-      "eval_samples_per_second": 42.237,
-      "eval_steps_per_second": 2.696,
-      "step": 43
+      "epoch": 6.923076923076923,
+      "eval_accuracy": 0.9766355140186916,
+      "eval_loss": 0.05554972589015961,
+      "eval_runtime": 5.1142,
+      "eval_samples_per_second": 41.844,
+      "eval_steps_per_second": 2.737,
+      "step": 45
     },
     {
-      "epoch": 8.0,
-      "grad_norm": 0.005246018059551716,
+      "epoch": 7.6923076923076925,
+      "grad_norm": 49.896671295166016,
       "learning_rate": 3.240740740740741e-05,
-      "loss": 0.0415,
+      "loss": 0.057,
       "step": 50
     },
     {
       "epoch": 8.0,
       "eval_accuracy": 1.0,
-      "eval_loss": 0.002033823635429144,
-      "eval_runtime": 4.4719,
-      "eval_samples_per_second": 42.041,
-      "eval_steps_per_second": 2.683,
-      "step": 50
+      "eval_loss": 0.005110082216560841,
+      "eval_runtime": 5.1693,
+      "eval_samples_per_second": 41.398,
+      "eval_steps_per_second": 2.708,
+      "step": 52
     },
     {
-      "epoch": 8.96,
-      "eval_accuracy": 0.9946808510638298,
-      "eval_loss": 0.021679291501641273,
-      "eval_runtime": 4.4307,
-      "eval_samples_per_second": 42.432,
-      "eval_steps_per_second": 2.708,
-      "step": 56
+      "epoch": 8.923076923076923,
+      "eval_accuracy": 0.985981308411215,
+      "eval_loss": 0.029344480484724045,
+      "eval_runtime": 5.1265,
+      "eval_samples_per_second": 41.744,
+      "eval_steps_per_second": 2.731,
+      "step": 58
     },
     {
-      "epoch": 9.6,
-      "grad_norm": 30.30540657043457,
+      "epoch": 9.23076923076923,
+      "grad_norm": 22.167015075683594,
       "learning_rate": 2.777777777777778e-05,
-      "loss": 0.0016,
+      "loss": 0.0361,
       "step": 60
     },
     {
-      "epoch": 9.92,
-      "eval_accuracy": 1.0,
-      "eval_loss": 0.005527271423488855,
-      "eval_runtime": 4.4668,
-      "eval_samples_per_second": 42.089,
-      "eval_steps_per_second": 2.687,
-      "step": 62
-    },
-    {
-      "epoch": 10.88,
-      "eval_accuracy": 0.9893617021276596,
-      "eval_loss": 0.040382999926805496,
-      "eval_runtime": 4.5274,
-      "eval_samples_per_second": 41.525,
-      "eval_steps_per_second": 2.651,
-      "step": 68
+      "epoch": 10.0,
+      "eval_accuracy": 0.985981308411215,
+      "eval_loss": 0.027151916176080704,
+      "eval_runtime": 5.1474,
+      "eval_samples_per_second": 41.574,
+      "eval_steps_per_second": 2.72,
+      "step": 65
     },
     {
-      "epoch": 11.2,
-      "grad_norm": 0.0002783833770081401,
+      "epoch": 10.76923076923077,
+      "grad_norm": 0.32469525933265686,
       "learning_rate": 2.314814814814815e-05,
-      "loss": 0.0,
+      "loss": 0.011,
       "step": 70
     },
+    {
+      "epoch": 10.923076923076923,
+      "eval_accuracy": 0.9906542056074766,
+      "eval_loss": 0.01700090989470482,
+      "eval_runtime": 5.0367,
+      "eval_samples_per_second": 42.488,
+      "eval_steps_per_second": 2.78,
+      "step": 71
+    },
     {
       "epoch": 12.0,
-      "eval_accuracy": 1.0,
-      "eval_loss": 0.003910040948539972,
-      "eval_runtime": 4.4428,
-      "eval_samples_per_second": 42.315,
-      "eval_steps_per_second": 2.701,
-      "step": 75
+      "eval_accuracy": 0.985981308411215,
+      "eval_loss": 0.07101369649171829,
+      "eval_runtime": 4.9654,
+      "eval_samples_per_second": 43.099,
+      "eval_steps_per_second": 2.82,
+      "step": 78
     },
     {
-      "epoch": 12.8,
-      "grad_norm": 0.0002681456971913576,
+      "epoch": 12.307692307692308,
+      "grad_norm": 0.06220458447933197,
       "learning_rate": 1.8518518518518518e-05,
-      "loss": 0.0,
+      "loss": 0.0006,
       "step": 80
     },
     {
-      "epoch": 12.96,
-      "eval_accuracy": 0.9946808510638298,
-      "eval_loss": 0.006880749948322773,
-      "eval_runtime": 4.5249,
-      "eval_samples_per_second": 41.548,
-      "eval_steps_per_second": 2.652,
-      "step": 81
-    },
-    {
-      "epoch": 13.92,
-      "eval_accuracy": 0.9946808510638298,
-      "eval_loss": 0.01297041680663824,
-      "eval_runtime": 4.4227,
-      "eval_samples_per_second": 42.508,
-      "eval_steps_per_second": 2.713,
-      "step": 87
+      "epoch": 12.923076923076923,
+      "eval_accuracy": 0.9813084112149533,
+      "eval_loss": 0.07214021682739258,
+      "eval_runtime": 4.9054,
+      "eval_samples_per_second": 43.625,
+      "eval_steps_per_second": 2.854,
+      "step": 84
     },
     {
-      "epoch": 14.4,
-      "grad_norm": 0.00011915920185856521,
+      "epoch": 13.846153846153847,
+      "grad_norm": 0.0023754581343382597,
       "learning_rate": 1.388888888888889e-05,
-      "loss": 0.0,
+      "loss": 0.0001,
       "step": 90
     },
     {
-      "epoch": 14.88,
-      "eval_accuracy": 0.9946808510638298,
-      "eval_loss": 0.015453271567821503,
-      "eval_runtime": 4.4271,
-      "eval_samples_per_second": 42.465,
-      "eval_steps_per_second": 2.711,
-      "step": 93
+      "epoch": 14.0,
+      "eval_accuracy": 0.9906542056074766,
+      "eval_loss": 0.02770264819264412,
+      "eval_runtime": 4.9176,
+      "eval_samples_per_second": 43.517,
+      "eval_steps_per_second": 2.847,
+      "step": 91
     },
     {
-      "epoch": 16.0,
-      "grad_norm": 0.00199418468400836,
-      "learning_rate": 9.259259259259259e-06,
-      "loss": 0.0,
-      "step": 100
+      "epoch": 14.923076923076923,
+      "eval_accuracy": 0.9953271028037384,
+      "eval_loss": 0.022401457652449608,
+      "eval_runtime": 4.8851,
+      "eval_samples_per_second": 43.806,
+      "eval_steps_per_second": 2.866,
+      "step": 97
     },
     {
-      "epoch": 16.0,
-      "eval_accuracy": 0.9946808510638298,
-      "eval_loss": 0.015377724543213844,
-      "eval_runtime": 4.4729,
-      "eval_samples_per_second": 42.03,
-      "eval_steps_per_second": 2.683,
+      "epoch": 15.384615384615385,
+      "grad_norm": 0.0014737015590071678,
+      "learning_rate": 9.259259259259259e-06,
+      "loss": 0.0001,
       "step": 100
     },
     {
-      "epoch": 16.96,
-      "eval_accuracy": 0.9946808510638298,
-      "eval_loss": 0.01379456277936697,
-      "eval_runtime": 4.3656,
-      "eval_samples_per_second": 43.064,
-      "eval_steps_per_second": 2.749,
-      "step": 106
+      "epoch": 16.0,
+      "eval_accuracy": 0.9953271028037384,
+      "eval_loss": 0.02016393281519413,
+      "eval_runtime": 5.2559,
+      "eval_samples_per_second": 40.716,
+      "eval_steps_per_second": 2.664,
+      "step": 104
     },
     {
-      "epoch": 17.6,
-      "grad_norm": 3.829578054137528e-05,
+      "epoch": 16.923076923076923,
+      "grad_norm": 0.0013604339910671115,
       "learning_rate": 4.6296296296296296e-06,
       "loss": 0.0,
       "step": 110
     },
     {
-      "epoch": 17.92,
-      "eval_accuracy": 0.9946808510638298,
-      "eval_loss": 0.013120871968567371,
-      "eval_runtime": 4.534,
-      "eval_samples_per_second": 41.464,
-      "eval_steps_per_second": 2.647,
-      "step": 112
+      "epoch": 16.923076923076923,
+      "eval_accuracy": 0.9953271028037384,
+      "eval_loss": 0.023425478488206863,
+      "eval_runtime": 5.0917,
+      "eval_samples_per_second": 42.029,
+      "eval_steps_per_second": 2.75,
+      "step": 110
     },
     {
-      "epoch": 18.88,
-      "eval_accuracy": 0.9946808510638298,
-      "eval_loss": 0.012892493978142738,
-      "eval_runtime": 4.5207,
-      "eval_samples_per_second": 41.586,
-      "eval_steps_per_second": 2.654,
-      "step": 118
+      "epoch": 18.0,
+      "eval_accuracy": 0.9953271028037384,
+      "eval_loss": 0.02447943389415741,
+      "eval_runtime": 4.9857,
+      "eval_samples_per_second": 42.923,
+      "eval_steps_per_second": 2.808,
+      "step": 117
     },
     {
-      "epoch": 19.2,
-      "grad_norm": 0.00024270155699923635,
+      "epoch": 18.46153846153846,
+      "grad_norm": 0.009677406400442123,
       "learning_rate": 0.0,
       "loss": 0.0,
       "step": 120
     },
     {
-      "epoch": 19.2,
-      "eval_accuracy": 0.9946808510638298,
-      "eval_loss": 0.012876511551439762,
-      "eval_runtime": 4.5819,
-      "eval_samples_per_second": 41.031,
-      "eval_steps_per_second": 2.619,
+      "epoch": 18.46153846153846,
+      "eval_accuracy": 0.9953271028037384,
+      "eval_loss": 0.024529017508029938,
+      "eval_runtime": 5.1559,
+      "eval_samples_per_second": 41.506,
+      "eval_steps_per_second": 2.715,
       "step": 120
     },
     {
-      "epoch": 19.2,
+      "epoch": 18.46153846153846,
       "step": 120,
-      "total_flos": 6.063680104641331e+17,
-      "train_loss": 0.059417286076692714,
-      "train_runtime": 389.1501,
-      "train_samples_per_second": 20.558,
-      "train_steps_per_second": 0.308
+      "total_flos": 6.035309694497341e+17,
+      "train_loss": 0.09444785690047866,
+      "train_runtime": 374.1698,
+      "train_samples_per_second": 22.129,
+      "train_steps_per_second": 0.321
     }
   ],
   "logging_steps": 10,
@@ -299,7 +290,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 6.063680104641331e+17,
+  "total_flos": 6.035309694497341e+17,
   "train_batch_size": 16,
   "trial_name": null,
   "trial_params": null

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:234698a21a3ee38e3970458d513d29bf92da7ee921a19acdb43031f7f8dec258
+oid sha256:06137faeaa484e835b530ab7917f824549233122bd7a33c6728f43780ec71c5c
 size 5112