End of training

all_results.json

@@ -1,14 +1,14 @@
 {
     "epoch": 100.0,
-    "eval_loss": 0.5928943157196045,
-    "eval_runtime": 66.288,
+    "eval_loss": 3.384493350982666,
+    "eval_runtime": 65.2378,
     "eval_samples": 1112,
-    "eval_samples_per_second": 16.775,
-    "eval_steps_per_second": 4.194,
-    "eval_wer": 0.7180269058295964,
-    "train_loss": 1.2273276989276594,
-    "train_runtime": 32753.7935,
+    "eval_samples_per_second": 17.045,
+    "eval_steps_per_second": 4.261,
+    "eval_wer": 0.9869058295964126,
+    "train_loss": 2.424026096050556,
+    "train_runtime": 30676.4279,
     "train_samples": 3336,
-    "train_samples_per_second": 10.185,
-    "train_steps_per_second": 0.159
+    "train_samples_per_second": 10.875,
+    "train_steps_per_second": 0.17
 }

eval_results.json

@@ -1,9 +1,9 @@
 {
     "epoch": 100.0,
-    "eval_loss": 0.5928943157196045,
-    "eval_runtime": 66.288,
+    "eval_loss": 3.384493350982666,
+    "eval_runtime": 65.2378,
     "eval_samples": 1112,
-    "eval_samples_per_second": 16.775,
-    "eval_steps_per_second": 4.194,
-    "eval_wer": 0.7180269058295964
+    "eval_samples_per_second": 17.045,
+    "eval_steps_per_second": 4.261,
+    "eval_wer": 0.9869058295964126
 }

pytorch_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:810b36f805c9412b3235dab58050d680cb8084ef2efd1ac272ab78267cd6d389
+oid sha256:06fe919eab98bc12ffc7b9ac5f1bb776f71d1502f7036df84461f078d7f4357d
 size 3850717489

runs/Jan28_06-54-16_job-e3562c4a-f6a3-47dc-8131-7e578f1139a3/events.out.tfevents.1643352878.job-e3562c4a-f6a3-47dc-8131-7e578f1139a3.778557.0

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:bcd59225c940f726389353f59ffa22210a77d66451ca8236a21b03dddaaf4477
-size 5779
+oid sha256:79086fce91a69cadecf971dbba0baaa955ef5be4371bf2a9647257c72a27483b
+size 16374

runs/Jan28_06-54-16_job-e3562c4a-f6a3-47dc-8131-7e578f1139a3/events.out.tfevents.1643383627.job-e3562c4a-f6a3-47dc-8131-7e578f1139a3.778557.2

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:33e0fa954a0844e51d914b007036c4780bb75a9cc5b86a1de1b2859a91eafb04
+size 358

train_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 100.0,
-    "train_loss": 1.2273276989276594,
-    "train_runtime": 32753.7935,
+    "train_loss": 2.424026096050556,
+    "train_runtime": 30676.4279,
     "train_samples": 3336,
-    "train_samples_per_second": 10.185,
-    "train_steps_per_second": 0.159
+    "train_samples_per_second": 10.875,
+    "train_steps_per_second": 0.17
 }

trainer_state.json

@@ -9,414 +9,414 @@
   "log_history": [
     {
       "epoch": 1.92,
-      "learning_rate": 3.2333333333333334e-06,
-      "loss": 13.5304,
+      "learning_rate": 1.6166666666666665e-07,
+      "loss": 24.0656,
       "step": 100
     },
     {
       "epoch": 3.84,
-      "learning_rate": 6.566666666666667e-06,
-      "loss": 3.5412,
+      "learning_rate": 3.283333333333333e-07,
+      "loss": 19.0268,
       "step": 200
     },
     {
       "epoch": 5.77,
-      "learning_rate": 9.9e-06,
-      "loss": 3.1425,
+      "learning_rate": 4.95e-07,
+      "loss": 6.1995,
       "step": 300
     },
     {
       "epoch": 7.69,
-      "learning_rate": 9.80204081632653e-06,
-      "loss": 2.0204,
+      "learning_rate": 4.901020408163265e-07,
+      "loss": 3.6484,
       "step": 400
     },
     {
       "epoch": 9.61,
-      "learning_rate": 9.59795918367347e-06,
-      "loss": 1.5794,
+      "learning_rate": 4.798979591836734e-07,
+      "loss": 3.5131,
       "step": 500
     },
     {
       "epoch": 9.61,
-      "eval_loss": 0.9859886169433594,
-      "eval_runtime": 65.6063,
-      "eval_samples_per_second": 16.95,
-      "eval_steps_per_second": 4.237,
-      "eval_wer": 0.9582062780269058,
+      "eval_loss": 3.5294487476348877,
+      "eval_runtime": 64.5425,
+      "eval_samples_per_second": 17.229,
+      "eval_steps_per_second": 4.307,
+      "eval_wer": 1.0,
       "step": 500
     },
     {
       "epoch": 11.54,
-      "learning_rate": 9.39387755102041e-06,
-      "loss": 1.4015,
+      "learning_rate": 4.696938775510204e-07,
+      "loss": 3.4699,
       "step": 600
     },
     {
       "epoch": 13.46,
-      "learning_rate": 9.189795918367347e-06,
-      "loss": 1.2803,
+      "learning_rate": 4.5948979591836735e-07,
+      "loss": 3.3973,
       "step": 700
     },
     {
       "epoch": 15.38,
-      "learning_rate": 8.985714285714287e-06,
-      "loss": 1.1969,
+      "learning_rate": 4.4928571428571426e-07,
+      "loss": 3.2876,
       "step": 800
     },
     {
       "epoch": 17.31,
-      "learning_rate": 8.781632653061225e-06,
-      "loss": 1.1458,
+      "learning_rate": 4.390816326530612e-07,
+      "loss": 3.0902,
       "step": 900
     },
     {
       "epoch": 19.23,
-      "learning_rate": 8.577551020408163e-06,
-      "loss": 1.0985,
+      "learning_rate": 4.288775510204081e-07,
+      "loss": 2.8596,
       "step": 1000
     },
     {
       "epoch": 19.23,
-      "eval_loss": 0.5948446989059448,
-      "eval_runtime": 63.7089,
-      "eval_samples_per_second": 17.454,
-      "eval_steps_per_second": 4.364,
-      "eval_wer": 0.7533632286995515,
+      "eval_loss": 3.5708465576171875,
+      "eval_runtime": 66.2433,
+      "eval_samples_per_second": 16.787,
+      "eval_steps_per_second": 4.197,
+      "eval_wer": 1.0,
       "step": 1000
     },
     {
       "epoch": 21.15,
-      "learning_rate": 8.373469387755104e-06,
-      "loss": 1.0639,
+      "learning_rate": 4.1867346938775513e-07,
+      "loss": 2.6397,
       "step": 1100
     },
     {
       "epoch": 23.08,
-      "learning_rate": 8.169387755102042e-06,
-      "loss": 1.0214,
+      "learning_rate": 4.0846938775510203e-07,
+      "loss": 2.4302,
       "step": 1200
     },
     {
       "epoch": 25.0,
-      "learning_rate": 7.96530612244898e-06,
-      "loss": 1.0047,
+      "learning_rate": 3.98265306122449e-07,
+      "loss": 2.2341,
       "step": 1300
     },
     {
       "epoch": 26.92,
-      "learning_rate": 7.76122448979592e-06,
-      "loss": 0.971,
+      "learning_rate": 3.880612244897959e-07,
+      "loss": 2.071,
       "step": 1400
     },
     {
       "epoch": 28.84,
-      "learning_rate": 7.557142857142857e-06,
-      "loss": 0.9551,
+      "learning_rate": 3.778571428571428e-07,
+      "loss": 1.9055,
       "step": 1500
     },
     {
       "epoch": 28.84,
-      "eval_loss": 0.5892378091812134,
-      "eval_runtime": 64.7699,
-      "eval_samples_per_second": 17.168,
-      "eval_steps_per_second": 4.292,
-      "eval_wer": 0.7391928251121076,
+      "eval_loss": 3.6432793140411377,
+      "eval_runtime": 65.1639,
+      "eval_samples_per_second": 17.065,
+      "eval_steps_per_second": 4.266,
+      "eval_wer": 1.0007174887892376,
       "step": 1500
     },
     {
       "epoch": 30.77,
-      "learning_rate": 7.353061224489797e-06,
-      "loss": 0.9387,
+      "learning_rate": 3.676530612244898e-07,
+      "loss": 1.7787,
       "step": 1600
     },
     {
       "epoch": 32.69,
-      "learning_rate": 7.148979591836735e-06,
-      "loss": 0.9272,
+      "learning_rate": 3.574489795918367e-07,
+      "loss": 1.6655,
       "step": 1700
     },
     {
       "epoch": 34.61,
-      "learning_rate": 6.944897959183674e-06,
-      "loss": 0.9192,
+      "learning_rate": 3.4724489795918366e-07,
+      "loss": 1.5696,
       "step": 1800
     },
     {
       "epoch": 36.54,
-      "learning_rate": 6.740816326530613e-06,
-      "loss": 0.892,
+      "learning_rate": 3.3704081632653057e-07,
+      "loss": 1.4902,
       "step": 1900
     },
     {
       "epoch": 38.46,
-      "learning_rate": 6.5367346938775515e-06,
-      "loss": 0.8699,
+      "learning_rate": 3.268367346938775e-07,
+      "loss": 1.4239,
       "step": 2000
     },
     {
       "epoch": 38.46,
-      "eval_loss": 0.5926958322525024,
-      "eval_runtime": 64.9144,
-      "eval_samples_per_second": 17.13,
-      "eval_steps_per_second": 4.283,
-      "eval_wer": 0.72,
+      "eval_loss": 3.6568963527679443,
+      "eval_runtime": 66.0028,
+      "eval_samples_per_second": 16.848,
+      "eval_steps_per_second": 4.212,
+      "eval_wer": 0.9994618834080717,
       "step": 2000
     },
     {
       "epoch": 40.38,
-      "learning_rate": 6.332653061224491e-06,
-      "loss": 0.8779,
+      "learning_rate": 3.166326530612245e-07,
+      "loss": 1.3735,
       "step": 2100
     },
     {
       "epoch": 42.31,
-      "learning_rate": 6.128571428571429e-06,
-      "loss": 0.8511,
+      "learning_rate": 3.0642857142857144e-07,
+      "loss": 1.3228,
       "step": 2200
     },
     {
       "epoch": 44.23,
-      "learning_rate": 5.924489795918368e-06,
-      "loss": 0.8532,
+      "learning_rate": 2.9622448979591834e-07,
+      "loss": 1.2834,
       "step": 2300
     },
     {
       "epoch": 46.15,
-      "learning_rate": 5.720408163265306e-06,
-      "loss": 0.8442,
+      "learning_rate": 2.860204081632653e-07,
+      "loss": 1.2438,
       "step": 2400
     },
     {
       "epoch": 48.08,
-      "learning_rate": 5.516326530612245e-06,
-      "loss": 0.8372,
+      "learning_rate": 2.758163265306122e-07,
+      "loss": 1.2168,
       "step": 2500
     },
     {
       "epoch": 48.08,
-      "eval_loss": 0.5938696265220642,
-      "eval_runtime": 64.5641,
-      "eval_samples_per_second": 17.223,
-      "eval_steps_per_second": 4.306,
-      "eval_wer": 0.73847533632287,
+      "eval_loss": 3.6079351902008057,
+      "eval_runtime": 65.3166,
+      "eval_samples_per_second": 17.025,
+      "eval_steps_per_second": 4.256,
+      "eval_wer": 0.995695067264574,
       "step": 2500
     },
     {
       "epoch": 50.0,
-      "learning_rate": 5.312244897959185e-06,
-      "loss": 0.8112,
+      "learning_rate": 2.656122448979592e-07,
+      "loss": 1.1792,
       "step": 2600
     },
     {
       "epoch": 51.92,
-      "learning_rate": 5.108163265306123e-06,
-      "loss": 0.8165,
+      "learning_rate": 2.554081632653061e-07,
+      "loss": 1.1706,
       "step": 2700
     },
     {
       "epoch": 53.84,
-      "learning_rate": 4.904081632653061e-06,
-      "loss": 0.8045,
+      "learning_rate": 2.4520408163265307e-07,
+      "loss": 1.1429,
       "step": 2800
     },
     {
       "epoch": 55.77,
-      "learning_rate": 4.7e-06,
-      "loss": 0.7974,
+      "learning_rate": 2.3499999999999997e-07,
+      "loss": 1.1318,
       "step": 2900
     },
     {
       "epoch": 57.69,
-      "learning_rate": 4.4959183673469394e-06,
-      "loss": 0.7794,
+      "learning_rate": 2.2479591836734693e-07,
+      "loss": 1.1063,
       "step": 3000
     },
     {
       "epoch": 57.69,
-      "eval_loss": 0.5800846815109253,
-      "eval_runtime": 63.4747,
-      "eval_samples_per_second": 17.519,
-      "eval_steps_per_second": 4.38,
-      "eval_wer": 0.7388340807174888,
+      "eval_loss": 3.5737504959106445,
+      "eval_runtime": 64.9336,
+      "eval_samples_per_second": 17.125,
+      "eval_steps_per_second": 4.281,
+      "eval_wer": 0.9924663677130044,
       "step": 3000
     },
     {
       "epoch": 59.61,
-      "learning_rate": 4.291836734693878e-06,
-      "loss": 0.7752,
+      "learning_rate": 2.1459183673469386e-07,
+      "loss": 1.0893,
       "step": 3100
     },
     {
       "epoch": 61.54,
-      "learning_rate": 4.087755102040817e-06,
-      "loss": 0.7765,
+      "learning_rate": 2.0438775510204082e-07,
+      "loss": 1.0744,
       "step": 3200
     },
     {
       "epoch": 63.46,
-      "learning_rate": 3.8836734693877556e-06,
-      "loss": 0.7584,
+      "learning_rate": 1.9418367346938775e-07,
+      "loss": 1.0591,
       "step": 3300
     },
     {
       "epoch": 65.38,
-      "learning_rate": 3.6795918367346943e-06,
-      "loss": 0.7589,
+      "learning_rate": 1.839795918367347e-07,
+      "loss": 1.05,
       "step": 3400
     },
     {
       "epoch": 67.31,
-      "learning_rate": 3.475510204081633e-06,
-      "loss": 0.7565,
+      "learning_rate": 1.7377551020408163e-07,
+      "loss": 1.0404,
       "step": 3500
     },
     {
       "epoch": 67.31,
-      "eval_loss": 0.582656979560852,
-      "eval_runtime": 71.1025,
-      "eval_samples_per_second": 15.639,
-      "eval_steps_per_second": 3.91,
-      "eval_wer": 0.7183856502242153,
+      "eval_loss": 3.4857470989227295,
+      "eval_runtime": 64.9602,
+      "eval_samples_per_second": 17.118,
+      "eval_steps_per_second": 4.28,
+      "eval_wer": 0.9888789237668162,
       "step": 3500
     },
     {
       "epoch": 69.23,
-      "learning_rate": 3.2714285714285717e-06,
-      "loss": 0.7595,
+      "learning_rate": 1.6357142857142856e-07,
+      "loss": 1.0325,
       "step": 3600
     },
     {
       "epoch": 71.15,
-      "learning_rate": 3.0673469387755104e-06,
-      "loss": 0.747,
+      "learning_rate": 1.5336734693877552e-07,
+      "loss": 1.0266,
       "step": 3700
     },
     {
       "epoch": 73.08,
-      "learning_rate": 2.863265306122449e-06,
-      "loss": 0.7373,
+      "learning_rate": 1.4316326530612245e-07,
+      "loss": 1.0164,
       "step": 3800
     },
     {
       "epoch": 75.0,
-      "learning_rate": 2.6591836734693882e-06,
-      "loss": 0.7268,
+      "learning_rate": 1.329591836734694e-07,
+      "loss": 1.0002,
       "step": 3900
     },
     {
       "epoch": 76.92,
-      "learning_rate": 2.455102040816327e-06,
-      "loss": 0.7303,
+      "learning_rate": 1.227551020408163e-07,
+      "loss": 1.001,
       "step": 4000
     },
     {
       "epoch": 76.92,
-      "eval_loss": 0.580017626285553,
-      "eval_runtime": 63.495,
-      "eval_samples_per_second": 17.513,
-      "eval_steps_per_second": 4.378,
-      "eval_wer": 0.7296860986547086,
+      "eval_loss": 3.4881510734558105,
+      "eval_runtime": 64.7811,
+      "eval_samples_per_second": 17.165,
+      "eval_steps_per_second": 4.291,
+      "eval_wer": 0.9858295964125561,
       "step": 4000
     },
     {
       "epoch": 78.84,
-      "learning_rate": 2.2510204081632656e-06,
-      "loss": 0.7239,
+      "learning_rate": 1.1255102040816327e-07,
+      "loss": 0.997,
       "step": 4100
     },
     {
       "epoch": 80.77,
-      "learning_rate": 2.0469387755102044e-06,
-      "loss": 0.7201,
+      "learning_rate": 1.0234693877551021e-07,
+      "loss": 0.9896,
       "step": 4200
     },
     {
       "epoch": 82.69,
-      "learning_rate": 1.8428571428571428e-06,
-      "loss": 0.7185,
+      "learning_rate": 9.214285714285714e-08,
+      "loss": 0.986,
       "step": 4300
     },
     {
       "epoch": 84.61,
-      "learning_rate": 1.6387755102040818e-06,
-      "loss": 0.7149,
+      "learning_rate": 8.193877551020407e-08,
+      "loss": 0.9823,
       "step": 4400
     },
     {
       "epoch": 86.54,
-      "learning_rate": 1.4346938775510205e-06,
-      "loss": 0.7128,
+      "learning_rate": 7.173469387755101e-08,
+      "loss": 0.982,
       "step": 4500
     },
     {
       "epoch": 86.54,
-      "eval_loss": 0.5977014899253845,
-      "eval_runtime": 64.1516,
-      "eval_samples_per_second": 17.334,
-      "eval_steps_per_second": 4.333,
-      "eval_wer": 0.7149775784753363,
+      "eval_loss": 3.3850555419921875,
+      "eval_runtime": 65.2436,
+      "eval_samples_per_second": 17.044,
+      "eval_steps_per_second": 4.261,
+      "eval_wer": 0.987085201793722,
       "step": 4500
     },
     {
       "epoch": 88.46,
-      "learning_rate": 1.2306122448979594e-06,
-      "loss": 0.705,
+      "learning_rate": 6.153061224489796e-08,
+      "loss": 0.9675,
       "step": 4600
     },
     {
       "epoch": 90.38,
-      "learning_rate": 1.026530612244898e-06,
-      "loss": 0.7129,
+      "learning_rate": 5.132653061224489e-08,
+      "loss": 0.9774,
       "step": 4700
     },
     {
       "epoch": 92.31,
-      "learning_rate": 8.224489795918368e-07,
-      "loss": 0.7076,
+      "learning_rate": 4.1122448979591836e-08,
+      "loss": 0.9687,
       "step": 4800
     },
     {
       "epoch": 94.23,
-      "learning_rate": 6.183673469387755e-07,
-      "loss": 0.7126,
+      "learning_rate": 3.091836734693877e-08,
+      "loss": 0.9771,
       "step": 4900
     },
     {
       "epoch": 96.15,
-      "learning_rate": 4.142857142857143e-07,
-      "loss": 0.6972,
+      "learning_rate": 2.0714285714285713e-08,
+      "loss": 0.9612,
       "step": 5000
     },
     {
       "epoch": 96.15,
-      "eval_loss": 0.5900700092315674,
-      "eval_runtime": 63.2743,
-      "eval_samples_per_second": 17.574,
-      "eval_steps_per_second": 4.394,
-      "eval_wer": 0.7176681614349776,
+      "eval_loss": 3.386908531188965,
+      "eval_runtime": 65.5128,
+      "eval_samples_per_second": 16.974,
+      "eval_steps_per_second": 4.243,
+      "eval_wer": 0.9872645739910314,
       "step": 5000
     },
     {
       "epoch": 98.08,
-      "learning_rate": 2.1020408163265306e-07,
-      "loss": 0.7001,
+      "learning_rate": 1.0510204081632651e-08,
+      "loss": 0.9667,
       "step": 5100
     },
     {
       "epoch": 100.0,
-      "learning_rate": 6.1224489795918365e-09,
-      "loss": 0.6965,
+      "learning_rate": 3.061224489795918e-10,
+      "loss": 0.9664,
       "step": 5200
     },
     {
       "epoch": 100.0,
       "step": 5200,
       "total_flos": 1.4653197993726655e+20,
-      "train_loss": 1.2273276989276594,
-      "train_runtime": 32753.7935,
-      "train_samples_per_second": 10.185,
-      "train_steps_per_second": 0.159
+      "train_loss": 2.424026096050556,
+      "train_runtime": 30676.4279,
+      "train_samples_per_second": 10.875,
+      "train_steps_per_second": 0.17
     }
   ],
   "max_steps": 5200,