Training in progress, step 7000

last-checkpoint/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f0460400a279e7766ba1a5c0ca78380731e0774cf4d291edfbc84654ee4a6aca
+oid sha256:b6193d4dcef6d888921a1166b7db92e1165f3686f778873a319aa454e17ea622
 size 12347192855

last-checkpoint/pytorch_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5e0ff690a614ad4feb1584be1b22e3316419b9b45d45121399a24f7b7785e65c
+oid sha256:7e91ae1bbae769e9ce17ed73669048b57cd296ecd5ff7d7b0071e74ca1c28a4f
 size 6173655480

last-checkpoint/rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3788ae40196f57221f45ac4d182ab0f36e1ff61517502c4d51687db9b9b71e1f
-size 14511
+oid sha256:49ab9d2b8499c20583cc80db00868f9477a33e6dd8363c0d4ff3b9c6bb471b6b
+size 14575

last-checkpoint/scaler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:da3defc4121d1071efa04d9a3c24a084fc57cd835b7f2d21ed3bc283ad291901
+oid sha256:cafc80e01cebde114db040b18403e6c790dd6f4096e4412ec1d94153944f97a8
 size 557

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6c16fd3410fdd3660983e7e486d0b41995f7675ed13e0a940e3532e9d1a7db51
+oid sha256:9d0b2313199b9b45d740a2bffd20986f78b2b72606b98ad7849fa54f34c2f5de
 size 627

last-checkpoint/trainer_state.json

@@ -1,8 +1,8 @@
 {
   "best_metric": 0.11536111854119403,
   "best_model_checkpoint": "/storage/elsayedissa/whisper-large-v2-spanish/checkpoint-1000",
-  "epoch": 0.2082682495053629,
-  "global_step": 6000,
+  "epoch": 0.2429796244229234,
+  "global_step": 7000,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -1500,11 +1500,260 @@
       "eval_steps_per_second": 0.062,
       "eval_wer": 0.10863441680453598,
       "step": 6000
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 7.747755102040817e-06,
+      "loss": 0.1818,
+      "step": 6025
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 7.737551020408164e-06,
+      "loss": 0.1595,
+      "step": 6050
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 7.727346938775512e-06,
+      "loss": 0.199,
+      "step": 6075
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 7.717142857142857e-06,
+      "loss": 0.1747,
+      "step": 6100
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 7.706938775510205e-06,
+      "loss": 0.1888,
+      "step": 6125
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 7.69673469387755e-06,
+      "loss": 0.1553,
+      "step": 6150
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 7.6865306122449e-06,
+      "loss": 0.1924,
+      "step": 6175
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 7.676326530612246e-06,
+      "loss": 0.1832,
+      "step": 6200
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 7.666122448979593e-06,
+      "loss": 0.2015,
+      "step": 6225
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 7.655918367346939e-06,
+      "loss": 0.1587,
+      "step": 6250
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 7.645714285714286e-06,
+      "loss": 0.1847,
+      "step": 6275
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 7.635510204081634e-06,
+      "loss": 0.1667,
+      "step": 6300
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 7.62530612244898e-06,
+      "loss": 0.1848,
+      "step": 6325
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 7.615102040816327e-06,
+      "loss": 0.1763,
+      "step": 6350
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 7.604897959183674e-06,
+      "loss": 0.1918,
+      "step": 6375
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 7.594693877551021e-06,
+      "loss": 0.1782,
+      "step": 6400
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 7.584489795918369e-06,
+      "loss": 0.2043,
+      "step": 6425
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 7.574285714285715e-06,
+      "loss": 0.1687,
+      "step": 6450
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 7.564081632653062e-06,
+      "loss": 0.1986,
+      "step": 6475
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 7.553877551020409e-06,
+      "loss": 0.1545,
+      "step": 6500
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 7.543673469387755e-06,
+      "loss": 0.1946,
+      "step": 6525
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 7.533469387755103e-06,
+      "loss": 0.1792,
+      "step": 6550
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 7.523265306122449e-06,
+      "loss": 0.206,
+      "step": 6575
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 7.513061224489796e-06,
+      "loss": 0.1653,
+      "step": 6600
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 7.502857142857144e-06,
+      "loss": 0.2081,
+      "step": 6625
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 7.492653061224491e-06,
+      "loss": 0.1745,
+      "step": 6650
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 7.482448979591838e-06,
+      "loss": 0.212,
+      "step": 6675
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 7.472244897959184e-06,
+      "loss": 0.1625,
+      "step": 6700
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 7.462040816326531e-06,
+      "loss": 0.2008,
+      "step": 6725
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 7.451836734693878e-06,
+      "loss": 0.1892,
+      "step": 6750
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 7.441632653061225e-06,
+      "loss": 0.1979,
+      "step": 6775
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 7.431428571428572e-06,
+      "loss": 0.1695,
+      "step": 6800
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 7.421224489795918e-06,
+      "loss": 0.1914,
+      "step": 6825
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 7.411020408163267e-06,
+      "loss": 0.1687,
+      "step": 6850
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 7.400816326530613e-06,
+      "loss": 0.1805,
+      "step": 6875
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 7.39061224489796e-06,
+      "loss": 0.1902,
+      "step": 6900
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 7.3804081632653066e-06,
+      "loss": 0.212,
+      "step": 6925
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 7.370204081632654e-06,
+      "loss": 0.1943,
+      "step": 6950
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 7.360000000000001e-06,
+      "loss": 0.1934,
+      "step": 6975
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 7.349795918367347e-06,
+      "loss": 0.1854,
+      "step": 7000
+    },
+    {
+      "epoch": 0.24,
+      "eval_loss": 0.19266490638256073,
+      "eval_runtime": 31023.933,
+      "eval_samples_per_second": 1.0,
+      "eval_steps_per_second": 0.063,
+      "eval_wer": 0.10483351660871203,
+      "step": 7000
     }
   ],
   "max_steps": 25000,
   "num_train_epochs": 1,
-  "total_flos": 2.038244179968e+20,
+  "total_flos": 2.377951543296e+20,
   "trial_name": null,
   "trial_params": null
 }

pytorch_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5e0ff690a614ad4feb1584be1b22e3316419b9b45d45121399a24f7b7785e65c
+oid sha256:7e91ae1bbae769e9ce17ed73669048b57cd296ecd5ff7d7b0071e74ca1c28a4f
 size 6173655480

runs/Dec26_19-57-19_gpu07.cyverse.org/events.out.tfevents.1672109970.gpu07.cyverse.org.126369.0

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5c471ec8b7d00e44f926ec8e72e6951f01dad13c670f4fcd66f750928299dbb9
-size 43977
+oid sha256:6e2bff7f5195a10fc570bcd185faa14302e6ca8f8cd84b4a9fa8001f74a0166e
+size 50575