End of training

all_results.json

@@ -1,14 +1,14 @@
 {
-    "epoch": 4.0,
+    "epoch": 6.0,
     "eval_loss": Infinity,
-    "eval_runtime": 982.1257,
+    "eval_runtime": 970.1291,
     "eval_samples": 16021,
-    "eval_samples_per_second": 16.313,
-    "eval_steps_per_second": 1.02,
+    "eval_samples_per_second": 16.514,
+    "eval_steps_per_second": 1.033,
     "eval_wer": 0.2936861897030796,
-    "train_loss": 0.9901066277552089,
-    "train_runtime": 182144.7146,
+    "train_loss": 0.30566064197718185,
+    "train_runtime": 111829.698,
     "train_samples": 446485,
-    "train_samples_per_second": 9.805,
-    "train_steps_per_second": 0.077
+    "train_samples_per_second": 23.955,
+    "train_steps_per_second": 0.187
 }

eval_results.json

@@ -1,9 +1,9 @@
 {
-    "epoch": 4.0,
+    "epoch": 6.0,
     "eval_loss": Infinity,
-    "eval_runtime": 982.1257,
+    "eval_runtime": 970.1291,
     "eval_samples": 16021,
-    "eval_samples_per_second": 16.313,
-    "eval_steps_per_second": 1.02,
+    "eval_samples_per_second": 16.514,
+    "eval_steps_per_second": 1.033,
     "eval_wer": 0.2936861897030796
 }

train_results.json

@@ -1,8 +1,8 @@
 {
-    "epoch": 4.0,
-    "train_loss": 0.9901066277552089,
-    "train_runtime": 182144.7146,
+    "epoch": 6.0,
+    "train_loss": 0.30566064197718185,
+    "train_runtime": 111829.698,
     "train_samples": 446485,
-    "train_samples_per_second": 9.805,
-    "train_steps_per_second": 0.077
+    "train_samples_per_second": 23.955,
+    "train_steps_per_second": 0.187
 }

trainer_state.json

@@ -1,8 +1,8 @@
 {
   "best_metric": Infinity,
   "best_model_checkpoint": "./checkpoint-1000",
-  "epoch": 3.999928330824912,
-  "global_step": 13952,
+  "epoch": 5.999928330824912,
+  "global_step": 20928,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -906,71 +906,554 @@
     },
     {
       "epoch": 3.76,
-      "learning_rate": 5.390311244979919e-06,
-      "loss": 0.7832,
+      "learning_rate": 3.1045276838546063e-05,
+      "loss": 0.805,
       "step": 13100
     },
     {
       "epoch": 3.78,
-      "learning_rate": 4.762801204819277e-06,
-      "loss": 0.7755,
+      "learning_rate": 3.0649038461538456e-05,
+      "loss": 0.816,
       "step": 13200
     },
     {
       "epoch": 3.81,
-      "learning_rate": 4.135291164658634e-06,
-      "loss": 0.7608,
+      "learning_rate": 3.025280008453085e-05,
+      "loss": 0.8112,
       "step": 13300
     },
     {
       "epoch": 3.84,
-      "learning_rate": 3.514056224899598e-06,
-      "loss": 0.7738,
+      "learning_rate": 2.9860524091293318e-05,
+      "loss": 0.8274,
       "step": 13400
     },
     {
       "epoch": 3.87,
-      "learning_rate": 2.8865461847389554e-06,
-      "loss": 0.7732,
+      "learning_rate": 2.946428571428571e-05,
+      "loss": 0.8249,
       "step": 13500
     },
     {
       "epoch": 3.9,
-      "learning_rate": 2.259036144578313e-06,
-      "loss": 0.7756,
+      "learning_rate": 2.9068047337278103e-05,
+      "loss": 0.8276,
       "step": 13600
     },
     {
       "epoch": 3.93,
-      "learning_rate": 1.637801204819277e-06,
-      "loss": 0.7675,
+      "learning_rate": 2.8675771344040573e-05,
+      "loss": 0.8227,
       "step": 13700
     },
     {
       "epoch": 3.96,
-      "learning_rate": 1.0102911646586344e-06,
-      "loss": 0.7653,
+      "learning_rate": 2.8279532967032966e-05,
+      "loss": 0.8246,
       "step": 13800
     },
     {
       "epoch": 3.99,
-      "learning_rate": 3.8278112449799194e-07,
-      "loss": 0.7627,
+      "learning_rate": 2.7883294590025355e-05,
+      "loss": 0.8198,
       "step": 13900
     },
     {
-      "epoch": 4.0,
-      "step": 13952,
-      "total_flos": 8.511794559686228e+20,
-      "train_loss": 0.9901066277552089,
-      "train_runtime": 182144.7146,
-      "train_samples_per_second": 9.805,
-      "train_steps_per_second": 0.077
+      "epoch": 4.01,
+      "learning_rate": 2.748705621301775e-05,
+      "loss": 0.8372,
+      "step": 14000
+    },
+    {
+      "epoch": 4.01,
+      "eval_loss": Infinity,
+      "eval_runtime": 973.1623,
+      "eval_samples_per_second": 16.463,
+      "eval_steps_per_second": 1.03,
+      "eval_wer": 0.19336725853959047,
+      "step": 14000
+    },
+    {
+      "epoch": 4.04,
+      "learning_rate": 2.7090817836010143e-05,
+      "loss": 0.8228,
+      "step": 14100
+    },
+    {
+      "epoch": 4.07,
+      "learning_rate": 2.6694579459002535e-05,
+      "loss": 0.8197,
+      "step": 14200
+    },
+    {
+      "epoch": 4.1,
+      "learning_rate": 2.6302303465765005e-05,
+      "loss": 0.8295,
+      "step": 14300
+    },
+    {
+      "epoch": 4.13,
+      "learning_rate": 2.5906065088757394e-05,
+      "loss": 0.8147,
+      "step": 14400
+    },
+    {
+      "epoch": 4.16,
+      "learning_rate": 2.5509826711749787e-05,
+      "loss": 0.8153,
+      "step": 14500
+    },
+    {
+      "epoch": 4.19,
+      "learning_rate": 2.511358833474218e-05,
+      "loss": 0.8155,
+      "step": 14600
+    },
+    {
+      "epoch": 4.21,
+      "learning_rate": 2.471734995773457e-05,
+      "loss": 0.8215,
+      "step": 14700
+    },
+    {
+      "epoch": 4.24,
+      "learning_rate": 2.4321111580726964e-05,
+      "loss": 0.8214,
+      "step": 14800
+    },
+    {
+      "epoch": 4.27,
+      "learning_rate": 2.3924873203719357e-05,
+      "loss": 0.811,
+      "step": 14900
+    },
+    {
+      "epoch": 4.3,
+      "learning_rate": 2.352863482671175e-05,
+      "loss": 0.8075,
+      "step": 15000
+    },
+    {
+      "epoch": 4.3,
+      "eval_loss": Infinity,
+      "eval_runtime": 973.424,
+      "eval_samples_per_second": 16.458,
+      "eval_steps_per_second": 1.029,
+      "eval_wer": 0.1923276841233885,
+      "step": 15000
+    },
+    {
+      "epoch": 4.33,
+      "learning_rate": 2.313239644970414e-05,
+      "loss": 0.8085,
+      "step": 15100
+    },
+    {
+      "epoch": 4.36,
+      "learning_rate": 2.273615807269653e-05,
+      "loss": 0.8149,
+      "step": 15200
+    },
+    {
+      "epoch": 4.39,
+      "learning_rate": 2.2339919695688927e-05,
+      "loss": 0.8103,
+      "step": 15300
+    },
+    {
+      "epoch": 4.42,
+      "learning_rate": 2.194368131868132e-05,
+      "loss": 0.8053,
+      "step": 15400
+    },
+    {
+      "epoch": 4.44,
+      "learning_rate": 2.154744294167371e-05,
+      "loss": 0.8038,
+      "step": 15500
+    },
+    {
+      "epoch": 4.47,
+      "learning_rate": 2.11512045646661e-05,
+      "loss": 0.8062,
+      "step": 15600
+    },
+    {
+      "epoch": 4.5,
+      "learning_rate": 2.075892857142857e-05,
+      "loss": 0.8074,
+      "step": 15700
+    },
+    {
+      "epoch": 4.53,
+      "learning_rate": 2.0362690194420963e-05,
+      "loss": 0.8082,
+      "step": 15800
+    },
+    {
+      "epoch": 4.56,
+      "learning_rate": 1.9966451817413352e-05,
+      "loss": 0.8093,
+      "step": 15900
+    },
+    {
+      "epoch": 4.59,
+      "learning_rate": 1.9570213440405745e-05,
+      "loss": 0.8069,
+      "step": 16000
+    },
+    {
+      "epoch": 4.59,
+      "eval_loss": Infinity,
+      "eval_runtime": 966.1996,
+      "eval_samples_per_second": 16.581,
+      "eval_steps_per_second": 1.037,
+      "eval_wer": 0.18768168488060938,
+      "step": 16000
+    },
+    {
+      "epoch": 4.62,
+      "learning_rate": 1.917397506339814e-05,
+      "loss": 0.8039,
+      "step": 16100
+    },
+    {
+      "epoch": 4.64,
+      "learning_rate": 1.8777736686390533e-05,
+      "loss": 0.8069,
+      "step": 16200
+    },
+    {
+      "epoch": 4.67,
+      "learning_rate": 1.8381498309382922e-05,
+      "loss": 0.8108,
+      "step": 16300
+    },
+    {
+      "epoch": 4.7,
+      "learning_rate": 1.7985259932375315e-05,
+      "loss": 0.7968,
+      "step": 16400
+    },
+    {
+      "epoch": 4.73,
+      "learning_rate": 1.7589021555367707e-05,
+      "loss": 0.8013,
+      "step": 16500
+    },
+    {
+      "epoch": 4.76,
+      "learning_rate": 1.71927831783601e-05,
+      "loss": 0.8059,
+      "step": 16600
+    },
+    {
+      "epoch": 4.79,
+      "learning_rate": 1.6796544801352492e-05,
+      "loss": 0.8076,
+      "step": 16700
+    },
+    {
+      "epoch": 4.82,
+      "learning_rate": 1.6400306424344885e-05,
+      "loss": 0.808,
+      "step": 16800
+    },
+    {
+      "epoch": 4.85,
+      "learning_rate": 1.6004068047337277e-05,
+      "loss": 0.8053,
+      "step": 16900
+    },
+    {
+      "epoch": 4.87,
+      "learning_rate": 1.560782967032967e-05,
+      "loss": 0.8064,
+      "step": 17000
+    },
+    {
+      "epoch": 4.87,
+      "eval_loss": Infinity,
+      "eval_runtime": 964.7985,
+      "eval_samples_per_second": 16.606,
+      "eval_steps_per_second": 1.039,
+      "eval_wer": 0.19554266426238345,
+      "step": 17000
+    },
+    {
+      "epoch": 4.9,
+      "learning_rate": 1.521159129332206e-05,
+      "loss": 0.8192,
+      "step": 17100
+    },
+    {
+      "epoch": 4.93,
+      "learning_rate": 1.4815352916314453e-05,
+      "loss": 0.8165,
+      "step": 17200
+    },
+    {
+      "epoch": 4.96,
+      "learning_rate": 1.4419114539306845e-05,
+      "loss": 0.8164,
+      "step": 17300
+    },
+    {
+      "epoch": 4.99,
+      "learning_rate": 1.4022876162299238e-05,
+      "loss": 0.8161,
+      "step": 17400
+    },
+    {
+      "epoch": 5.02,
+      "learning_rate": 1.362663778529163e-05,
+      "loss": 0.8215,
+      "step": 17500
+    },
+    {
+      "epoch": 5.05,
+      "learning_rate": 1.3230399408284023e-05,
+      "loss": 0.8304,
+      "step": 17600
+    },
+    {
+      "epoch": 5.07,
+      "learning_rate": 1.2834161031276414e-05,
+      "loss": 0.8087,
+      "step": 17700
+    },
+    {
+      "epoch": 5.1,
+      "learning_rate": 1.2437922654268808e-05,
+      "loss": 0.8133,
+      "step": 17800
+    },
+    {
+      "epoch": 5.13,
+      "learning_rate": 1.2041684277261199e-05,
+      "loss": 0.7987,
+      "step": 17900
+    },
+    {
+      "epoch": 5.16,
+      "learning_rate": 1.1645445900253593e-05,
+      "loss": 0.801,
+      "step": 18000
+    },
+    {
+      "epoch": 5.16,
+      "eval_loss": Infinity,
+      "eval_runtime": 968.3005,
+      "eval_samples_per_second": 16.545,
+      "eval_steps_per_second": 1.035,
+      "eval_wer": 0.18908061835426385,
+      "step": 18000
+    },
+    {
+      "epoch": 5.19,
+      "learning_rate": 1.1249207523245984e-05,
+      "loss": 0.8075,
+      "step": 18100
+    },
+    {
+      "epoch": 5.22,
+      "learning_rate": 1.0852969146238374e-05,
+      "loss": 0.8069,
+      "step": 18200
+    },
+    {
+      "epoch": 5.25,
+      "learning_rate": 1.0456730769230769e-05,
+      "loss": 0.8095,
+      "step": 18300
+    },
+    {
+      "epoch": 5.28,
+      "learning_rate": 1.006049239222316e-05,
+      "loss": 0.8069,
+      "step": 18400
+    },
+    {
+      "epoch": 5.3,
+      "learning_rate": 9.664254015215553e-06,
+      "loss": 0.8127,
+      "step": 18500
+    },
+    {
+      "epoch": 5.33,
+      "learning_rate": 9.268015638207944e-06,
+      "loss": 0.8132,
+      "step": 18600
+    },
+    {
+      "epoch": 5.36,
+      "learning_rate": 8.871777261200337e-06,
+      "loss": 0.8086,
+      "step": 18700
+    },
+    {
+      "epoch": 5.39,
+      "learning_rate": 8.47553888419273e-06,
+      "loss": 0.8018,
+      "step": 18800
+    },
+    {
+      "epoch": 5.42,
+      "learning_rate": 8.079300507185122e-06,
+      "loss": 0.8102,
+      "step": 18900
+    },
+    {
+      "epoch": 5.45,
+      "learning_rate": 7.683062130177514e-06,
+      "loss": 0.8022,
+      "step": 19000
+    },
+    {
+      "epoch": 5.45,
+      "eval_loss": Infinity,
+      "eval_runtime": 968.9288,
+      "eval_samples_per_second": 16.535,
+      "eval_steps_per_second": 1.034,
+      "eval_wer": 0.18952981717607953,
+      "step": 19000
+    },
+    {
+      "epoch": 5.48,
+      "learning_rate": 7.286823753169907e-06,
+      "loss": 0.7979,
+      "step": 19100
+    },
+    {
+      "epoch": 5.5,
+      "learning_rate": 6.890585376162298e-06,
+      "loss": 0.797,
+      "step": 19200
+    },
+    {
+      "epoch": 5.53,
+      "learning_rate": 6.494346999154691e-06,
+      "loss": 0.7969,
+      "step": 19300
+    },
+    {
+      "epoch": 5.56,
+      "learning_rate": 6.098108622147083e-06,
+      "loss": 0.8034,
+      "step": 19400
+    },
+    {
+      "epoch": 5.59,
+      "learning_rate": 5.701870245139475e-06,
+      "loss": 0.8025,
+      "step": 19500
+    },
+    {
+      "epoch": 5.62,
+      "learning_rate": 5.305631868131867e-06,
+      "loss": 0.7938,
+      "step": 19600
+    },
+    {
+      "epoch": 5.65,
+      "learning_rate": 4.90939349112426e-06,
+      "loss": 0.7942,
+      "step": 19700
+    },
+    {
+      "epoch": 5.68,
+      "learning_rate": 4.513155114116652e-06,
+      "loss": 0.7855,
+      "step": 19800
+    },
+    {
+      "epoch": 5.71,
+      "learning_rate": 4.116916737109044e-06,
+      "loss": 0.7798,
+      "step": 19900
+    },
+    {
+      "epoch": 5.73,
+      "learning_rate": 3.724640743871513e-06,
+      "loss": 0.792,
+      "step": 20000
+    },
+    {
+      "epoch": 5.73,
+      "eval_loss": Infinity,
+      "eval_runtime": 967.729,
+      "eval_samples_per_second": 16.555,
+      "eval_steps_per_second": 1.035,
+      "eval_wer": 0.1853586852592198,
+      "step": 20000
+    },
+    {
+      "epoch": 5.76,
+      "learning_rate": 3.3284023668639047e-06,
+      "loss": 0.786,
+      "step": 20100
+    },
+    {
+      "epoch": 5.79,
+      "learning_rate": 2.932163989856297e-06,
+      "loss": 0.7907,
+      "step": 20200
+    },
+    {
+      "epoch": 5.82,
+      "learning_rate": 2.5359256128486896e-06,
+      "loss": 0.7837,
+      "step": 20300
+    },
+    {
+      "epoch": 5.85,
+      "learning_rate": 2.1396872358410817e-06,
+      "loss": 0.7769,
+      "step": 20400
+    },
+    {
+      "epoch": 5.88,
+      "learning_rate": 1.743448858833474e-06,
+      "loss": 0.7821,
+      "step": 20500
+    },
+    {
+      "epoch": 5.91,
+      "learning_rate": 1.3511728655959423e-06,
+      "loss": 0.7837,
+      "step": 20600
+    },
+    {
+      "epoch": 5.93,
+      "learning_rate": 9.549344885883346e-07,
+      "loss": 0.7856,
+      "step": 20700
+    },
+    {
+      "epoch": 5.96,
+      "learning_rate": 5.586961115807269e-07,
+      "loss": 0.7895,
+      "step": 20800
+    },
+    {
+      "epoch": 5.99,
+      "learning_rate": 1.6245773457311917e-07,
+      "loss": 0.7739,
+      "step": 20900
+    },
+    {
+      "epoch": 6.0,
+      "step": 20928,
+      "total_flos": 1.2767684631907514e+21,
+      "train_loss": 0.30566064197718185,
+      "train_runtime": 111829.698,
+      "train_samples_per_second": 23.955,
+      "train_steps_per_second": 0.187
     }
   ],
-  "max_steps": 13952,
-  "num_train_epochs": 4,
-  "total_flos": 8.511794559686228e+20,
+  "max_steps": 20928,
+  "num_train_epochs": 6,
+  "total_flos": 1.2767684631907514e+21,
   "trial_name": null,
   "trial_params": null
 }

wandb/run-20220206_201634-uhiy9e2t/files/output.log

@@ -11369,3 +11369,514 @@ Upload file wandb/run-20220206_201634-uhiy9e2t/run-uhiy9e2t.wandb:  98%|██
    89ae304..5c2523c  main -> main
 Upload file wandb/run-20220206_201634-uhiy9e2t/run-uhiy9e2t.wandb: 100%|████████████████████████████████████████████████████████████████████| 117M/117M [00:08<00:00, 15.1MB/s]
 Dropping the following result as it does not have all the necessary fields:
+{}
+
+
+
+
+
+
+
+
+
+
+
+
+Upload file wandb/run-20220206_201634-uhiy9e2t/run-uhiy9e2t.wandb:  63%|█████████████████████████████████████████▉                         | 73.4M/117M [00:24<00:05, 7.97MB/s]
+02/08/2022 03:25:43 - WARNING - huggingface_hub.repository - To https://huggingface.co/Plim/xls-r-1b-cv_8-fr
+Upload file wandb/run-20220206_201634-uhiy9e2t/run-uhiy9e2t.wandb:  89%|████████████████████████████████████████████████████████████▌       | 104M/117M [00:26<00:01, 12.1MB/s]To https://huggingface.co/Plim/xls-r-1b-cv_8-fr
+   5c2523c..39a2094  main -> main
+Upload file wandb/run-20220206_201634-uhiy9e2t/run-uhiy9e2t.wandb: 100%|████████████████████████████████████████████████████████████████████| 117M/117M [00:27<00:00, 4.51MB/s]
+The following columns in the evaluation set  don't have a corresponding argument in `Wav2Vec2ForCTC.forward` and have been ignored: input_length. If input_length are not expected by `Wav2Vec2ForCTC.forward`,  you can safely ignore this message.
+***** Running Evaluation *****
+  Num examples = 16021
+  Batch size = 16
+***** train metrics *****
+  epoch                    =               6.0
+  train_loss               =            0.3057
+  train_runtime            = 1 day, 7:03:49.69
+  train_samples            =            446485
+  train_samples_per_second =            23.955
+  train_steps_per_second   =             0.187
+02/08/2022 03:25:47 - INFO - __main__ - *** Evaluate ***
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 1002/1002 [16:09<00:00,  1.03it/s]
+Saving model checkpoint to ./
+Configuration saved in ./config.json
+***** eval metrics *****
+  epoch                   =        6.0
+  eval_loss               =        inf
+  eval_runtime            = 0:16:10.12
+  eval_samples            =      16021
+  eval_samples_per_second =     16.514
+  eval_steps_per_second   =      1.033
+  eval_wer                =     0.2937
+Model weights saved in ./pytorch_model.bin

wandb/run-20220206_201634-uhiy9e2t/run-uhiy9e2t.wandb

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:160673feed9ea3ddc328247490fe7d2950cd358fb40b4b827edc8123c80ab0af
-size 123076744
+oid sha256:408ae6e3f28dda1ea915dfd924f64e8ed1220e8622555b73f8a99c8961884603
+size 123251777