Training in progress, step 9500, checkpoint

last-checkpoint/global_step9500/mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:aab2566649265c7de72eb81e39871412c46701e5a9afb54d2abbd528e4af9ae9
+size 197282509

last-checkpoint/global_step9500/zero_pp_rank_0_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0a9c534408566520ab01f7ba38671c1252a73c896b865991d72666d511bbf8b8
+size 180416968

last-checkpoint/global_step9500/zero_pp_rank_1_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b772aff2ef9f6dcf3cf0937c7327745c5a787daee8c3a60c47d9aa8d9c12c0ff
+size 180416776

last-checkpoint/global_step9500/zero_pp_rank_2_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f50c2ba6ba9b89de6f5aa85d79e1b6506f2f234d007302258c6ea5848b0380cd
+size 180416776

last-checkpoint/global_step9500/zero_pp_rank_3_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cdc06a26b12090b4a97a5a80554b8acd37ec0759310f81fb8afa96e3cd562e1a
+size 180416904

last-checkpoint/global_step9500/zero_pp_rank_4_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:06a70ea1a7650f92d8377c26244524f98dc1942526181eb972d2bd744bfccb00
+size 180416712

last-checkpoint/global_step9500/zero_pp_rank_5_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4cb48bec2ebb45cbe335f3d4a474fe7ddf7f49e0f12b92fa513d07facafe39f9
+size 180417096

last-checkpoint/latest

@@ -1 +1 @@
-global_step9000
+global_step9500

last-checkpoint/model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2ab65fcd6eef541b0eab0651d0717f1badda42d7fd92bfb4075b331da2037828
+oid sha256:360912bb66b5ef43b6217bdaf80a08845f2208809b73bea201e43e6402ec2cc3
 size 188836816

last-checkpoint/rng_state_0.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:04a7c505ffdf7349443f4571264a08aa5d00e509401833cfbc8ee36e3f78ae54
+oid sha256:d79ce11b93dbde86948b98c9ce1fa175f7a568c470e9afcefe72b95bef17a002
 size 15536

last-checkpoint/rng_state_1.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:95d7ca58acb8d3876c3a7f75f1478ea294c4dc3f3a656bcf55937f21b39bafeb
+oid sha256:429ad9026ad7a94e536302987a593f0bac5e320bb5b4eac33f49d6dd3d14350d
 size 15536

last-checkpoint/rng_state_2.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2b892c9e25d561484c4d46a997b69468880c163891890ad309278d371355e9ba
+oid sha256:e4f799dffa70912ee5c070fe5ffb374a83158856242756a00c555654374f01da
 size 15536

last-checkpoint/rng_state_3.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a8a81bb1b67d6e9cc674626ce2fca3b1db95526692490a7cca8768ee12d6e2e7
+oid sha256:54ecabfa330e288c44f0ddbde659a7cb50fb9698004a8036194b92ee4dd46064
 size 15536

last-checkpoint/rng_state_4.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:bfa5c5c42cdaccb27d45e096ddb8bbb4eafd2783ecc504d34c4ed6a4e70b1732
-size 15472
+oid sha256:f1dbdc413b81880e152a22beadfb7765b2eb1b5148311b1cd2ec696f7323e1db
+size 15536

last-checkpoint/rng_state_5.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:abd2f2fb50fa21042e610837b6b9a95060d6b9c41e23ab7ce19a6228e9dd5bb9
+oid sha256:44530a10c71833f6b70a1d042fd27fbc157e7885451a81a8b6656842c24fa6f3
 size 15536

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:dc228f89b1f6600cf303d1bda56efe53cce614e2f06ad7a65194b3b38bd74bd7
+oid sha256:44b6f2c1812fce23d5731b76df787ac1b0733fa7e4172a4feeceed05b16ec8ff
 size 1256

last-checkpoint/trainer_state.json

@@ -1,9 +1,9 @@
 {
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 243.24324324324326,
+  "epoch": 256.7567567567568,
   "eval_steps": 1500,
-  "global_step": 9000,
+  "global_step": 9500,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -1312,6 +1312,76 @@
       "eval_steps_per_second": 0.226,
       "eval_wer": 0.19295266397792402,
       "step": 9000
+    },
+    {
+      "epoch": 244.59459459459458,
+      "grad_norm": 0.2917761206626892,
+      "learning_rate": 0.0001,
+      "loss": 0.0194,
+      "step": 9050
+    },
+    {
+      "epoch": 245.94594594594594,
+      "grad_norm": 0.28261467814445496,
+      "learning_rate": 0.0001,
+      "loss": 0.0186,
+      "step": 9100
+    },
+    {
+      "epoch": 247.2972972972973,
+      "grad_norm": 0.44025787711143494,
+      "learning_rate": 0.0001,
+      "loss": 0.0191,
+      "step": 9150
+    },
+    {
+      "epoch": 248.64864864864865,
+      "grad_norm": 0.26063069701194763,
+      "learning_rate": 0.0001,
+      "loss": 0.0182,
+      "step": 9200
+    },
+    {
+      "epoch": 250.0,
+      "grad_norm": 0.3038322329521179,
+      "learning_rate": 0.0001,
+      "loss": 0.0185,
+      "step": 9250
+    },
+    {
+      "epoch": 251.35135135135135,
+      "grad_norm": 0.30964452028274536,
+      "learning_rate": 0.0001,
+      "loss": 0.0188,
+      "step": 9300
+    },
+    {
+      "epoch": 252.7027027027027,
+      "grad_norm": 0.34113481640815735,
+      "learning_rate": 0.0001,
+      "loss": 0.0189,
+      "step": 9350
+    },
+    {
+      "epoch": 254.05405405405406,
+      "grad_norm": 0.28624454140663147,
+      "learning_rate": 0.0001,
+      "loss": 0.0186,
+      "step": 9400
+    },
+    {
+      "epoch": 255.40540540540542,
+      "grad_norm": 0.28637397289276123,
+      "learning_rate": 0.0001,
+      "loss": 0.0189,
+      "step": 9450
+    },
+    {
+      "epoch": 256.7567567567568,
+      "grad_norm": 0.3362099230289459,
+      "learning_rate": 0.0001,
+      "loss": 0.0194,
+      "step": 9500
     }
   ],
   "logging_steps": 50,
@@ -1331,7 +1401,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 2.5263079974749025e+20,
+  "total_flos": 2.6668736569018207e+20,
   "train_batch_size": 64,
   "trial_name": null,
   "trial_params": null