Training in progress, step 5000

last-checkpoint/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:eb55c91b103628f5be95896e1614ad2bc546605cf6236721f0622634adcd689e
+oid sha256:c171984694c48427b034af7134433824c6ffd4f9f28188dda965bf3de0c15fb0
 size 12347192855

last-checkpoint/pytorch_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:43e47f260dd1b8c823dbd2ec8239c2bf713fcf544791f3f0b436803f94dc5f60
+oid sha256:d172c474a443c85f3cb4d2a327f379678abab48c79f8e28e462b950711cc5684
 size 6173655480

last-checkpoint/rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:640548085210c8490898cfba017d8bd1baf3553a19e27f9821a34801ec2b3a88
+oid sha256:e0a9edbae95b39e125cb13cf9656d66377898a7639a1540109f9acf0f641de62
 size 14575

last-checkpoint/scaler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2f245e4bee87e5915781db8b8b7a3b00fd2ef949a525de0dd949823efc21b2f4
+oid sha256:02bfa0a41730a41e7e9ac0b2cafb5ad11cb816439654260867a463c2e192b332
 size 557

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:455fa7fbc6657e0c30aabe935c34b61e115426f0bbb7f1106461aac75d23221b
+oid sha256:183d1da7e0c968851cacd55db8383570bdd99f0154f18e1b72c4b435effad44d
 size 627

last-checkpoint/trainer_state.json

@@ -1,8 +1,8 @@
 {
   "best_metric": 0.4207764409383577,
   "best_model_checkpoint": "/storage/elsayedissa/whisper-large-v2-arabic-24h/checkpoint-3000",
-  "epoch": 7.1301247771836005,
-  "global_step": 4000,
+  "epoch": 8.9126559714795,
+  "global_step": 5000,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -1002,11 +1002,260 @@
       "eval_steps_per_second": 0.149,
       "eval_wer": 0.4251609673577187,
       "step": 4000
+    },
+    {
+      "epoch": 7.17,
+      "learning_rate": 2.1800000000000003e-06,
+      "loss": 0.002,
+      "step": 4025
+    },
+    {
+      "epoch": 7.22,
+      "learning_rate": 2.1244444444444443e-06,
+      "loss": 0.0042,
+      "step": 4050
+    },
+    {
+      "epoch": 7.26,
+      "learning_rate": 2.0688888888888892e-06,
+      "loss": 0.0021,
+      "step": 4075
+    },
+    {
+      "epoch": 7.31,
+      "learning_rate": 2.0133333333333337e-06,
+      "loss": 0.0035,
+      "step": 4100
+    },
+    {
+      "epoch": 7.35,
+      "learning_rate": 1.9577777777777777e-06,
+      "loss": 0.0026,
+      "step": 4125
+    },
+    {
+      "epoch": 7.4,
+      "learning_rate": 1.9022222222222222e-06,
+      "loss": 0.0037,
+      "step": 4150
+    },
+    {
+      "epoch": 7.44,
+      "learning_rate": 1.8466666666666668e-06,
+      "loss": 0.0024,
+      "step": 4175
+    },
+    {
+      "epoch": 7.49,
+      "learning_rate": 1.7911111111111113e-06,
+      "loss": 0.0038,
+      "step": 4200
+    },
+    {
+      "epoch": 7.53,
+      "learning_rate": 1.7355555555555555e-06,
+      "loss": 0.0024,
+      "step": 4225
+    },
+    {
+      "epoch": 7.58,
+      "learning_rate": 1.6800000000000002e-06,
+      "loss": 0.0036,
+      "step": 4250
+    },
+    {
+      "epoch": 7.62,
+      "learning_rate": 1.6244444444444447e-06,
+      "loss": 0.0028,
+      "step": 4275
+    },
+    {
+      "epoch": 7.66,
+      "learning_rate": 1.568888888888889e-06,
+      "loss": 0.0043,
+      "step": 4300
+    },
+    {
+      "epoch": 7.71,
+      "learning_rate": 1.5133333333333334e-06,
+      "loss": 0.0017,
+      "step": 4325
+    },
+    {
+      "epoch": 7.75,
+      "learning_rate": 1.457777777777778e-06,
+      "loss": 0.0035,
+      "step": 4350
+    },
+    {
+      "epoch": 7.8,
+      "learning_rate": 1.4022222222222223e-06,
+      "loss": 0.003,
+      "step": 4375
+    },
+    {
+      "epoch": 7.84,
+      "learning_rate": 1.3466666666666668e-06,
+      "loss": 0.0037,
+      "step": 4400
+    },
+    {
+      "epoch": 7.89,
+      "learning_rate": 1.2911111111111112e-06,
+      "loss": 0.0018,
+      "step": 4425
+    },
+    {
+      "epoch": 7.93,
+      "learning_rate": 1.2355555555555557e-06,
+      "loss": 0.004,
+      "step": 4450
+    },
+    {
+      "epoch": 7.98,
+      "learning_rate": 1.1800000000000001e-06,
+      "loss": 0.002,
+      "step": 4475
+    },
+    {
+      "epoch": 8.02,
+      "learning_rate": 1.1244444444444446e-06,
+      "loss": 0.0025,
+      "step": 4500
+    },
+    {
+      "epoch": 8.07,
+      "learning_rate": 1.068888888888889e-06,
+      "loss": 0.0014,
+      "step": 4525
+    },
+    {
+      "epoch": 8.11,
+      "learning_rate": 1.0133333333333333e-06,
+      "loss": 0.0018,
+      "step": 4550
+    },
+    {
+      "epoch": 8.16,
+      "learning_rate": 9.57777777777778e-07,
+      "loss": 0.0012,
+      "step": 4575
+    },
+    {
+      "epoch": 8.2,
+      "learning_rate": 9.022222222222222e-07,
+      "loss": 0.0022,
+      "step": 4600
+    },
+    {
+      "epoch": 8.24,
+      "learning_rate": 8.466666666666668e-07,
+      "loss": 0.0011,
+      "step": 4625
+    },
+    {
+      "epoch": 8.29,
+      "learning_rate": 7.911111111111111e-07,
+      "loss": 0.0018,
+      "step": 4650
+    },
+    {
+      "epoch": 8.33,
+      "learning_rate": 7.355555555555556e-07,
+      "loss": 0.0013,
+      "step": 4675
+    },
+    {
+      "epoch": 8.38,
+      "learning_rate": 6.800000000000001e-07,
+      "loss": 0.0016,
+      "step": 4700
+    },
+    {
+      "epoch": 8.42,
+      "learning_rate": 6.244444444444445e-07,
+      "loss": 0.0013,
+      "step": 4725
+    },
+    {
+      "epoch": 8.47,
+      "learning_rate": 5.68888888888889e-07,
+      "loss": 0.0019,
+      "step": 4750
+    },
+    {
+      "epoch": 8.51,
+      "learning_rate": 5.133333333333334e-07,
+      "loss": 0.0013,
+      "step": 4775
+    },
+    {
+      "epoch": 8.56,
+      "learning_rate": 4.5777777777777784e-07,
+      "loss": 0.0017,
+      "step": 4800
+    },
+    {
+      "epoch": 8.6,
+      "learning_rate": 4.0222222222222224e-07,
+      "loss": 0.0013,
+      "step": 4825
+    },
+    {
+      "epoch": 8.65,
+      "learning_rate": 3.466666666666667e-07,
+      "loss": 0.0021,
+      "step": 4850
+    },
+    {
+      "epoch": 8.69,
+      "learning_rate": 2.9111111111111116e-07,
+      "loss": 0.0012,
+      "step": 4875
+    },
+    {
+      "epoch": 8.73,
+      "learning_rate": 2.3555555555555556e-07,
+      "loss": 0.0026,
+      "step": 4900
+    },
+    {
+      "epoch": 8.78,
+      "learning_rate": 1.8e-07,
+      "loss": 0.0012,
+      "step": 4925
+    },
+    {
+      "epoch": 8.82,
+      "learning_rate": 1.2444444444444446e-07,
+      "loss": 0.0017,
+      "step": 4950
+    },
+    {
+      "epoch": 8.87,
+      "learning_rate": 6.888888888888889e-08,
+      "loss": 0.0012,
+      "step": 4975
+    },
+    {
+      "epoch": 8.91,
+      "learning_rate": 1.3333333333333334e-08,
+      "loss": 0.0016,
+      "step": 5000
+    },
+    {
+      "epoch": 8.91,
+      "eval_loss": 0.3434337377548218,
+      "eval_runtime": 4386.6592,
+      "eval_samples_per_second": 2.379,
+      "eval_steps_per_second": 0.149,
+      "eval_wer": 0.4239008475600283,
+      "step": 5000
     }
   ],
   "max_steps": 5000,
   "num_train_epochs": 9,
-  "total_flos": 4.245301687799808e+20,
+  "total_flos": 5.306738576228352e+20,
   "trial_name": null,
   "trial_params": null
 }

pytorch_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:43e47f260dd1b8c823dbd2ec8239c2bf713fcf544791f3f0b436803f94dc5f60
+oid sha256:d172c474a443c85f3cb4d2a327f379678abab48c79f8e28e462b950711cc5684
 size 6173655480

runs/Jan28_23-19-38_gpu07.cyverse.org/events.out.tfevents.1674973221.gpu07.cyverse.org.79624.0

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c20785d93f929c6776fb55f65c13ef81d9ad4f46d8a0e0fdba027f22601ec447
-size 30790
+oid sha256:22f944f653d930146c1944127978e6c5f4f499b2926b2d171485fe46cc8860c4
+size 37388