Training in progress, step 21000

last-checkpoint/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3070aeb72d29088489c0fb6e92c23b06f3de39671f0b9c078f18da734e95bf6b
+oid sha256:8c1e93fa65fde9ba67832b5fde4ad330506d95a23edfec7be98020d37f17559b
 size 12347192855

last-checkpoint/pytorch_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:47bc5d383a893937307b6252269bc95121112ddd7c888bf951e25cc0f6233b96
+oid sha256:f07b5f25c323d9a36c85d57f748183f9d3d14494dbfa6b195ed15224d9deafeb
 size 6173655480

last-checkpoint/rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:4979e2334c039bb7332b1f1ca1ec119b85c014a960ba0dbab629d4dd618f0ad2
+oid sha256:1f36f42cb84b389111430939ff75a7277c877ebb6534c5db685d643483c1e7ae
 size 14575

last-checkpoint/scaler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0e95ea4d51ce3dbab01e59ff8912c2f7044fb7bd804fa1afc7ea77264e8c7d84
+oid sha256:75b2d5c8b12724e300963a719034bfe44737705d2de06deafe54683c78564862
 size 557

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e664770f258aac914190f0cd016b820f86cc722eb7db2dbd9de5aff987606154
+oid sha256:e7394c13a986a89cb46fa0877cb8091fc4bf0e4f28ce1608b81abde4cc024dfc
 size 627

last-checkpoint/trainer_state.json

@@ -1,8 +1,8 @@
 {
   "best_metric": 0.11536111854119403,
   "best_model_checkpoint": "/storage/elsayedissa/whisper-large-v2-spanish/checkpoint-1000",
-  "epoch": 0.6942274983512097,
-  "global_step": 20000,
+  "epoch": 0.7289388732687702,
+  "global_step": 21000,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -4986,11 +4986,260 @@
       "eval_steps_per_second": 0.063,
       "eval_wer": 0.08736786835874422,
       "step": 20000
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 2.0351020408163267e-06,
+      "loss": 0.1463,
+      "step": 20025
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 2.024897959183674e-06,
+      "loss": 0.1206,
+      "step": 20050
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 2.0146938775510204e-06,
+      "loss": 0.1498,
+      "step": 20075
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 2.0044897959183675e-06,
+      "loss": 0.1191,
+      "step": 20100
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 1.9942857142857146e-06,
+      "loss": 0.1674,
+      "step": 20125
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 1.984081632653061e-06,
+      "loss": 0.1326,
+      "step": 20150
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 1.9738775510204083e-06,
+      "loss": 0.1602,
+      "step": 20175
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 1.9636734693877553e-06,
+      "loss": 0.1298,
+      "step": 20200
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 1.9534693877551024e-06,
+      "loss": 0.1644,
+      "step": 20225
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 1.943265306122449e-06,
+      "loss": 0.1277,
+      "step": 20250
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 1.933061224489796e-06,
+      "loss": 0.1593,
+      "step": 20275
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 1.922857142857143e-06,
+      "loss": 0.113,
+      "step": 20300
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 1.91265306122449e-06,
+      "loss": 0.142,
+      "step": 20325
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 1.9024489795918368e-06,
+      "loss": 0.1197,
+      "step": 20350
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 1.8922448979591837e-06,
+      "loss": 0.1487,
+      "step": 20375
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 1.8820408163265308e-06,
+      "loss": 0.1309,
+      "step": 20400
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 1.8718367346938776e-06,
+      "loss": 0.1531,
+      "step": 20425
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 1.8616326530612247e-06,
+      "loss": 0.135,
+      "step": 20450
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 1.8514285714285715e-06,
+      "loss": 0.1591,
+      "step": 20475
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 1.8412244897959186e-06,
+      "loss": 0.1402,
+      "step": 20500
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 1.8310204081632654e-06,
+      "loss": 0.1577,
+      "step": 20525
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 1.8208163265306125e-06,
+      "loss": 0.1218,
+      "step": 20550
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 1.8106122448979593e-06,
+      "loss": 0.1638,
+      "step": 20575
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 1.8004081632653062e-06,
+      "loss": 0.1376,
+      "step": 20600
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 1.7902040816326533e-06,
+      "loss": 0.1609,
+      "step": 20625
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 1.7800000000000001e-06,
+      "loss": 0.1099,
+      "step": 20650
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 1.7697959183673472e-06,
+      "loss": 0.1863,
+      "step": 20675
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 1.759591836734694e-06,
+      "loss": 0.1185,
+      "step": 20700
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 1.749387755102041e-06,
+      "loss": 0.1616,
+      "step": 20725
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 1.739183673469388e-06,
+      "loss": 0.1208,
+      "step": 20750
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 1.728979591836735e-06,
+      "loss": 0.1588,
+      "step": 20775
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 1.7187755102040818e-06,
+      "loss": 0.1268,
+      "step": 20800
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 1.7085714285714287e-06,
+      "loss": 0.1506,
+      "step": 20825
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 1.6983673469387758e-06,
+      "loss": 0.1412,
+      "step": 20850
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 1.6881632653061226e-06,
+      "loss": 0.1547,
+      "step": 20875
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 1.6779591836734697e-06,
+      "loss": 0.1386,
+      "step": 20900
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 1.6677551020408165e-06,
+      "loss": 0.1597,
+      "step": 20925
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 1.6575510204081636e-06,
+      "loss": 0.103,
+      "step": 20950
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 1.6473469387755102e-06,
+      "loss": 0.1609,
+      "step": 20975
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 1.637142857142857e-06,
+      "loss": 0.1479,
+      "step": 21000
+    },
+    {
+      "epoch": 0.73,
+      "eval_loss": 0.15253788232803345,
+      "eval_runtime": 31232.4021,
+      "eval_samples_per_second": 0.994,
+      "eval_steps_per_second": 0.062,
+      "eval_wer": 0.08848403192624074,
+      "step": 21000
     }
   ],
   "max_steps": 25000,
   "num_train_epochs": 1,
-  "total_flos": 6.79414726656e+20,
+  "total_flos": 7.133854629888e+20,
   "trial_name": null,
   "trial_params": null
 }

pytorch_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:47bc5d383a893937307b6252269bc95121112ddd7c888bf951e25cc0f6233b96
+oid sha256:f07b5f25c323d9a36c85d57f748183f9d3d14494dbfa6b195ed15224d9deafeb
 size 6173655480

runs/Dec26_19-57-19_gpu07.cyverse.org/events.out.tfevents.1672109970.gpu07.cyverse.org.126369.0

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:4222f5599f934e25008ca9c74695648d0516086fca35c66323f38f875ddff30c
-size 136808
+oid sha256:a05d2b060ee61e97fa7da66f905940a11d0a5b14104c16f577979a5e29d6fa2d
+size 143532