End of training

all_results.json

@@ -1,14 +1,14 @@
 {
     "epoch": 15.0,
-    "eval_loss": 0.37937504053115845,
-    "eval_runtime": 67.5206,
+    "eval_loss": 0.37925440073013306,
+    "eval_runtime": 67.0208,
     "eval_samples": 1647,
-    "eval_samples_per_second": 24.393,
-    "eval_steps_per_second": 3.051,
-    "eval_wer": 0.342048820345215,
-    "train_loss": 1.0777230644809361,
-    "train_runtime": 3812.1297,
+    "eval_samples_per_second": 24.574,
+    "eval_steps_per_second": 3.074,
+    "eval_wer": 0.34950464712491064,
+    "train_loss": 1.0876544129957846,
+    "train_runtime": 3803.1822,
     "train_samples": 3478,
-    "train_samples_per_second": 13.685,
-    "train_steps_per_second": 0.429
+    "train_samples_per_second": 13.717,
+    "train_steps_per_second": 0.43
 }

eval_results.json

@@ -1,9 +1,9 @@
 {
     "epoch": 15.0,
-    "eval_loss": 0.37937504053115845,
-    "eval_runtime": 67.5206,
+    "eval_loss": 0.37925440073013306,
+    "eval_runtime": 67.0208,
     "eval_samples": 1647,
-    "eval_samples_per_second": 24.393,
-    "eval_steps_per_second": 3.051,
-    "eval_wer": 0.342048820345215
+    "eval_samples_per_second": 24.574,
+    "eval_steps_per_second": 3.074,
+    "eval_wer": 0.34950464712491064
 }

pytorch_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a364cb1fa5839ec73cef456a2b868c16845a58e01cb3e6b0257b65d652456d89
+oid sha256:676cf1b13df5d5b133df645585b7f4f4d765e89d9dd3a7b22128124e2bca242e
 size 1262091761

runs/Dec09_09-01-28_cuda11-renwei-docker/events.out.tfevents.1639012200.cuda11-renwei-docker.296.0

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f095fadee554b40e27ba1bf9c3df16892f87f6c1f50b14bafa0ad3cc60ef0294
-size 6148
+oid sha256:f6b7ebacf789cae7f1d5a8832e6ac259f2729cd2c6948d9a29d627fe6405d8ec
+size 10789

runs/Dec09_09-01-28_cuda11-renwei-docker/events.out.tfevents.1639016073.cuda11-renwei-docker.296.2

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a082e79574a9b019342c37d54b8ca2ff29dba17c918dbb5ddcb882b051425643
+size 358

train_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 15.0,
-    "train_loss": 1.0777230644809361,
-    "train_runtime": 3812.1297,
+    "train_loss": 1.0876544129957846,
+    "train_runtime": 3803.1822,
     "train_samples": 3478,
-    "train_samples_per_second": 13.685,
-    "train_steps_per_second": 0.429
+    "train_samples_per_second": 13.717,
+    "train_steps_per_second": 0.43
 }

trainer_state.json

@@ -9,174 +9,174 @@
   "log_history": [
     {
       "epoch": 0.92,
-      "eval_loss": 3.5880308151245117,
-      "eval_runtime": 70.0803,
-      "eval_samples_per_second": 23.502,
-      "eval_steps_per_second": 2.939,
+      "eval_loss": 3.6067986488342285,
+      "eval_runtime": 67.1306,
+      "eval_samples_per_second": 24.534,
+      "eval_steps_per_second": 3.069,
       "eval_wer": 1.0,
       "step": 100
     },
     {
       "epoch": 1.83,
-      "eval_loss": 3.012378454208374,
-      "eval_runtime": 67.4593,
-      "eval_samples_per_second": 24.415,
-      "eval_steps_per_second": 3.054,
+      "eval_loss": 3.041459321975708,
+      "eval_runtime": 67.0963,
+      "eval_samples_per_second": 24.547,
+      "eval_steps_per_second": 3.07,
       "eval_wer": 0.9998978653865795,
       "step": 200
     },
     {
       "epoch": 2.75,
-      "eval_loss": 0.8977468609809875,
-      "eval_runtime": 68.3285,
-      "eval_samples_per_second": 24.104,
-      "eval_steps_per_second": 3.015,
-      "eval_wer": 0.7639669083852517,
+      "eval_loss": 0.8838828206062317,
+      "eval_runtime": 66.9473,
+      "eval_samples_per_second": 24.601,
+      "eval_steps_per_second": 3.077,
+      "eval_wer": 0.7398631396180165,
       "step": 300
     },
     {
       "epoch": 3.67,
-      "eval_loss": 0.5706909894943237,
-      "eval_runtime": 67.9906,
-      "eval_samples_per_second": 24.224,
-      "eval_steps_per_second": 3.03,
-      "eval_wer": 0.6114799305484628,
+      "eval_loss": 0.5810018181800842,
+      "eval_runtime": 65.6828,
+      "eval_samples_per_second": 25.075,
+      "eval_steps_per_second": 3.136,
+      "eval_wer": 0.5892145848227964,
       "step": 400
     },
     {
       "epoch": 4.59,
-      "learning_rate": 0.0002988,
-      "loss": 3.1774,
+      "learning_rate": 0.0002982,
+      "loss": 3.2096,
       "step": 500
     },
     {
       "epoch": 4.59,
-      "eval_loss": 0.5086258053779602,
-      "eval_runtime": 68.5993,
-      "eval_samples_per_second": 24.009,
-      "eval_steps_per_second": 3.003,
-      "eval_wer": 0.5162904708405679,
+      "eval_loss": 0.5231282711029053,
+      "eval_runtime": 67.353,
+      "eval_samples_per_second": 24.453,
+      "eval_steps_per_second": 3.059,
+      "eval_wer": 0.5485650086814421,
       "step": 500
     },
     {
       "epoch": 5.5,
-      "eval_loss": 0.4572145342826843,
-      "eval_runtime": 67.7684,
-      "eval_samples_per_second": 24.303,
-      "eval_steps_per_second": 3.04,
-      "eval_wer": 0.4830967214789092,
+      "eval_loss": 0.449034720659256,
+      "eval_runtime": 66.057,
+      "eval_samples_per_second": 24.933,
+      "eval_steps_per_second": 3.119,
+      "eval_wer": 0.5066898171790419,
       "step": 600
     },
     {
       "epoch": 6.42,
-      "eval_loss": 0.4250616133213043,
-      "eval_runtime": 68.0474,
-      "eval_samples_per_second": 24.204,
-      "eval_steps_per_second": 3.027,
-      "eval_wer": 0.45143499131855785,
+      "eval_loss": 0.4039786159992218,
+      "eval_runtime": 65.8832,
+      "eval_samples_per_second": 24.999,
+      "eval_steps_per_second": 3.127,
+      "eval_wer": 0.4405065876825656,
       "step": 700
     },
     {
       "epoch": 7.34,
-      "eval_loss": 0.3982156217098236,
-      "eval_runtime": 67.5727,
-      "eval_samples_per_second": 24.374,
-      "eval_steps_per_second": 3.049,
-      "eval_wer": 0.4190583188642631,
+      "eval_loss": 0.4107377827167511,
+      "eval_runtime": 65.8157,
+      "eval_samples_per_second": 25.024,
+      "eval_steps_per_second": 3.13,
+      "eval_wer": 0.43080379940761926,
       "step": 800
     },
     {
       "epoch": 8.26,
-      "eval_loss": 0.41418781876564026,
-      "eval_runtime": 67.8213,
-      "eval_samples_per_second": 24.284,
-      "eval_steps_per_second": 3.037,
-      "eval_wer": 0.41068328056378306,
+      "eval_loss": 0.4097311794757843,
+      "eval_runtime": 65.3892,
+      "eval_samples_per_second": 25.188,
+      "eval_steps_per_second": 3.15,
+      "eval_wer": 0.4143601266469206,
       "step": 900
     },
     {
       "epoch": 9.17,
-      "learning_rate": 0.00016837004405286342,
-      "loss": 0.2241,
+      "learning_rate": 0.00016863436123348017,
+      "loss": 0.224,
       "step": 1000
     },
     {
       "epoch": 9.17,
-      "eval_loss": 0.3958446979522705,
-      "eval_runtime": 67.1127,
-      "eval_samples_per_second": 24.541,
-      "eval_steps_per_second": 3.069,
-      "eval_wer": 0.39638443468491474,
+      "eval_loss": 0.3750542402267456,
+      "eval_runtime": 65.3056,
+      "eval_samples_per_second": 25.22,
+      "eval_steps_per_second": 3.154,
+      "eval_wer": 0.3914819732407313,
       "step": 1000
     },
     {
       "epoch": 10.09,
-      "eval_loss": 0.40702584385871887,
-      "eval_runtime": 66.7245,
-      "eval_samples_per_second": 24.684,
-      "eval_steps_per_second": 3.087,
-      "eval_wer": 0.38800939638443466,
+      "eval_loss": 0.4037274420261383,
+      "eval_runtime": 66.4673,
+      "eval_samples_per_second": 24.779,
+      "eval_steps_per_second": 3.099,
+      "eval_wer": 0.3872944540904913,
       "step": 1100
     },
     {
       "epoch": 11.01,
-      "eval_loss": 0.3864419162273407,
-      "eval_runtime": 67.8823,
-      "eval_samples_per_second": 24.263,
-      "eval_steps_per_second": 3.035,
-      "eval_wer": 0.3799407619242161,
+      "eval_loss": 0.39948078989982605,
+      "eval_runtime": 65.7895,
+      "eval_samples_per_second": 25.034,
+      "eval_steps_per_second": 3.131,
+      "eval_wer": 0.37943008885711366,
       "step": 1200
     },
     {
       "epoch": 11.93,
-      "eval_loss": 0.39367178082466125,
-      "eval_runtime": 66.9603,
-      "eval_samples_per_second": 24.597,
-      "eval_steps_per_second": 3.076,
-      "eval_wer": 0.367786742927178,
+      "eval_loss": 0.380903959274292,
+      "eval_runtime": 66.0515,
+      "eval_samples_per_second": 24.935,
+      "eval_steps_per_second": 3.119,
+      "eval_wer": 0.36737820447349606,
       "step": 1300
     },
     {
       "epoch": 12.84,
-      "eval_loss": 0.39223712682724,
-      "eval_runtime": 66.9432,
-      "eval_samples_per_second": 24.603,
-      "eval_steps_per_second": 3.077,
-      "eval_wer": 0.3560412623838219,
+      "eval_loss": 0.3864530920982361,
+      "eval_runtime": 65.226,
+      "eval_samples_per_second": 25.251,
+      "eval_steps_per_second": 3.158,
+      "eval_wer": 0.3573690123582882,
       "step": 1400
     },
     {
       "epoch": 13.76,
-      "learning_rate": 3.621145374449339e-05,
-      "loss": 0.102,
+      "learning_rate": 3.647577092511013e-05,
+      "loss": 0.1027,
       "step": 1500
     },
     {
       "epoch": 13.76,
-      "eval_loss": 0.39097627997398376,
-      "eval_runtime": 64.6816,
-      "eval_samples_per_second": 25.463,
-      "eval_steps_per_second": 3.185,
-      "eval_wer": 0.34664487794913695,
+      "eval_loss": 0.38098737597465515,
+      "eval_runtime": 66.3137,
+      "eval_samples_per_second": 24.837,
+      "eval_steps_per_second": 3.106,
+      "eval_wer": 0.3534878970483097,
       "step": 1500
     },
     {
       "epoch": 14.68,
-      "eval_loss": 0.3765156865119934,
-      "eval_runtime": 70.3564,
-      "eval_samples_per_second": 23.409,
-      "eval_steps_per_second": 2.928,
-      "eval_wer": 0.3411296088244306,
+      "eval_loss": 0.37903106212615967,
+      "eval_runtime": 65.9553,
+      "eval_samples_per_second": 24.971,
+      "eval_steps_per_second": 3.123,
+      "eval_wer": 0.35021958941885406,
       "step": 1600
     },
     {
       "epoch": 15.0,
       "step": 1635,
       "total_flos": 6.429866751096261e+18,
-      "train_loss": 1.0777230644809361,
-      "train_runtime": 3812.1297,
-      "train_samples_per_second": 13.685,
-      "train_steps_per_second": 0.429
+      "train_loss": 1.0876544129957846,
+      "train_runtime": 3803.1822,
+      "train_samples_per_second": 13.717,
+      "train_steps_per_second": 0.43
     }
   ],
   "max_steps": 1635,