End of training

all_results.json

@@ -0,0 +1,14 @@
+{
+    "epoch": 80.0,
+    "eval_loss": 0.3218217194080353,
+    "eval_runtime": 61.3941,
+    "eval_samples": 1681,
+    "eval_samples_per_second": 27.38,
+    "eval_steps_per_second": 3.437,
+    "eval_wer": 0.3094967839576239,
+    "train_loss": 1.1972084400946634,
+    "train_runtime": 14228.904,
+    "train_samples": 3626,
+    "train_samples_per_second": 20.387,
+    "train_steps_per_second": 0.32
+}

eval_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 80.0,
+    "eval_loss": 0.3218217194080353,
+    "eval_runtime": 61.3941,
+    "eval_samples": 1681,
+    "eval_samples_per_second": 27.38,
+    "eval_steps_per_second": 3.437,
+    "eval_wer": 0.3094967839576239
+}

train_results.json

@@ -0,0 +1,8 @@
+{
+    "epoch": 80.0,
+    "train_loss": 1.1972084400946634,
+    "train_runtime": 14228.904,
+    "train_samples": 3626,
+    "train_samples_per_second": 20.387,
+    "train_steps_per_second": 0.32
+}

trainer_state.json

@@ -0,0 +1,160 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 80.0,
+  "global_step": 4560,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 8.77,
+      "learning_rate": 4.8822115384615383e-05,
+      "loss": 6.3683,
+      "step": 500
+    },
+    {
+      "epoch": 8.77,
+      "eval_loss": 3.128021001815796,
+      "eval_runtime": 61.7966,
+      "eval_samples_per_second": 27.202,
+      "eval_steps_per_second": 3.414,
+      "eval_wer": 1.0,
+      "step": 500
+    },
+    {
+      "epoch": 17.54,
+      "learning_rate": 4.28125e-05,
+      "loss": 1.9915,
+      "step": 1000
+    },
+    {
+      "epoch": 17.54,
+      "eval_loss": 0.6600078344345093,
+      "eval_runtime": 61.9497,
+      "eval_samples_per_second": 27.135,
+      "eval_steps_per_second": 3.406,
+      "eval_wer": 0.6444381384790011,
+      "step": 1000
+    },
+    {
+      "epoch": 26.32,
+      "learning_rate": 3.680288461538462e-05,
+      "loss": 0.6565,
+      "step": 1500
+    },
+    {
+      "epoch": 26.32,
+      "eval_loss": 0.4207763671875,
+      "eval_runtime": 61.6392,
+      "eval_samples_per_second": 27.272,
+      "eval_steps_per_second": 3.423,
+      "eval_wer": 0.4486379114642452,
+      "step": 1500
+    },
+    {
+      "epoch": 35.09,
+      "learning_rate": 3.079326923076923e-05,
+      "loss": 0.4484,
+      "step": 2000
+    },
+    {
+      "epoch": 35.09,
+      "eval_loss": 0.38846877217292786,
+      "eval_runtime": 60.5282,
+      "eval_samples_per_second": 27.772,
+      "eval_steps_per_second": 3.486,
+      "eval_wer": 0.4005864547862278,
+      "step": 2000
+    },
+    {
+      "epoch": 43.86,
+      "learning_rate": 2.478365384615385e-05,
+      "loss": 0.3573,
+      "step": 2500
+    },
+    {
+      "epoch": 43.86,
+      "eval_loss": 0.3548009693622589,
+      "eval_runtime": 60.4916,
+      "eval_samples_per_second": 27.789,
+      "eval_steps_per_second": 3.488,
+      "eval_wer": 0.36256148316307224,
+      "step": 2500
+    },
+    {
+      "epoch": 52.63,
+      "learning_rate": 1.8774038461538462e-05,
+      "loss": 0.3063,
+      "step": 3000
+    },
+    {
+      "epoch": 52.63,
+      "eval_loss": 0.3374779522418976,
+      "eval_runtime": 60.6448,
+      "eval_samples_per_second": 27.719,
+      "eval_steps_per_second": 3.479,
+      "eval_wer": 0.34298146046159667,
+      "step": 3000
+    },
+    {
+      "epoch": 61.4,
+      "learning_rate": 1.2764423076923077e-05,
+      "loss": 0.2751,
+      "step": 3500
+    },
+    {
+      "epoch": 61.4,
+      "eval_loss": 0.3359163701534271,
+      "eval_runtime": 61.2942,
+      "eval_samples_per_second": 27.425,
+      "eval_steps_per_second": 3.442,
+      "eval_wer": 0.32406356413166854,
+      "step": 3500
+    },
+    {
+      "epoch": 70.18,
+      "learning_rate": 6.754807692307692e-06,
+      "loss": 0.2511,
+      "step": 4000
+    },
+    {
+      "epoch": 70.18,
+      "eval_loss": 0.32216018438339233,
+      "eval_runtime": 60.7546,
+      "eval_samples_per_second": 27.669,
+      "eval_steps_per_second": 3.473,
+      "eval_wer": 0.31082103670071887,
+      "step": 4000
+    },
+    {
+      "epoch": 78.95,
+      "learning_rate": 7.451923076923078e-07,
+      "loss": 0.2361,
+      "step": 4500
+    },
+    {
+      "epoch": 78.95,
+      "eval_loss": 0.3205258548259735,
+      "eval_runtime": 61.518,
+      "eval_samples_per_second": 27.325,
+      "eval_steps_per_second": 3.43,
+      "eval_wer": 0.3083617101778282,
+      "step": 4500
+    },
+    {
+      "epoch": 80.0,
+      "step": 4560,
+      "total_flos": 3.6657223992885486e+19,
+      "train_loss": 1.1972084400946634,
+      "train_runtime": 14228.904,
+      "train_samples_per_second": 20.387,
+      "train_steps_per_second": 0.32
+    }
+  ],
+  "max_steps": 4560,
+  "num_train_epochs": 80,
+  "total_flos": 3.6657223992885486e+19,
+  "trial_name": null,
+  "trial_params": null
+}

wandb/run-20220204_170031-gzsqxey3/files/output.log

@@ -4875,3 +4875,61 @@ Upload file pytorch_model.bin:  98%|██████████████
    d144cd4..4fe64b9  main -> main
 Upload file pytorch_model.bin: 100%|█████████████████████████████████| 1.18G/1.18G [00:41<00:00, 30.5MB/s]
 02/04/2022 20:59:39 - WARNING - huggingface_hub.repository - To https://huggingface.co/ayameRushia/wav2vec2-large-xls-r-300m-el
+   d144cd4..4fe64b9  main -> main
+Dropping the following result as it does not have all the necessary fields:
+{'dataset': {'name': 'common_voice', 'type': 'common_voice', 'args': 'el'}}
+To https://huggingface.co/ayameRushia/wav2vec2-large-xls-r-300m-el
+   4fe64b9..2cd2df2  main -> main
+02/04/2022 20:59:46 - WARNING - huggingface_hub.repository - To https://huggingface.co/ayameRushia/wav2vec2-large-xls-r-300m-el
+   4fe64b9..2cd2df2  main -> main
+The following columns in the evaluation set  don't have a corresponding argument in `Wav2Vec2ForCTC.forward` and have been ignored: input_length.
+***** Running Evaluation *****
+  Num examples = 1681
+  Batch size = 8
+  1%|▋                                                                    | 2/211 [00:00<00:24,  8.66it/s]
+***** train metrics *****
+  epoch                    =       80.0
+  train_loss               =     1.1972
+  train_runtime            = 3:57:08.90
+  train_samples            =       3626
+  train_samples_per_second =     20.387
+  train_steps_per_second   =       0.32
+  5%|███▏                                                                | 10/211 [00:02<00:57,  3.50it/s]
+  8%|█████▍                                                              | 17/211 [00:04<00:53,  3.63it/s]
+ 12%|████████                                                            | 25/211 [00:06<00:59,  3.15it/s]
+ 15%|█████████▉                                                          | 31/211 [00:08<00:55,  3.24it/s]
+ 18%|████████████▏                                                       | 38/211 [00:10<00:52,  3.31it/s]
+ 21%|██████████████▏                                                     | 44/211 [00:12<00:59,  2.82it/s]
+ 24%|████████████████                                                    | 50/211 [00:14<00:50,  3.21it/s]
+ 27%|██████████████████▎                                                 | 57/211 [00:17<00:43,  3.57it/s]
+ 30%|████████████████████▋                                               | 64/211 [00:19<00:44,  3.31it/s]
+ 34%|███████████████████████▏                                            | 72/211 [00:21<00:40,  3.43it/s]
+ 37%|█████████████████████████▏                                          | 78/211 [00:23<00:38,  3.49it/s]
+ 39%|██████████████████████████▋                                         | 83/211 [00:24<00:40,  3.19it/s]
+ 43%|█████████████████████████████▎                                      | 91/211 [00:26<00:27,  4.37it/s]
+ 46%|███████████████████████████████▌                                    | 98/211 [00:28<00:30,  3.66it/s]
+ 50%|█████████████████████████████████▎                                 | 105/211 [00:30<00:30,  3.48it/s]
+ 54%|███████████████████████████████████▉                               | 113/211 [00:32<00:26,  3.68it/s]
+ 57%|██████████████████████████████████████                             | 120/211 [00:34<00:24,  3.67it/s]
+ 57%|██████████████████████████████████████                             | 120/211 [00:34<00:24,  3.67it/s]
+ 57%|██████████████████████████████████████                             | 120/211 [00:34<00:24,  3.67it/s]
+ 57%|██████████████████████████████████████                             | 120/211 [00:34<00:24,  3.67it/s]
+ 57%|██████████████████████████████████████                             | 120/211 [00:34<00:24,  3.67it/s]
+ 57%|██████████████████████████████████████                             | 120/211 [00:34<00:24,  3.67it/s]
+ 57%|██████████████████████████████████████                             | 120/211 [00:34<00:24,  3.67it/s]
+ 57%|██████████████████████████████████████                             | 120/211 [00:34<00:24,  3.67it/s]
+ 57%|██████████████████████████████████████                             | 120/211 [00:34<00:24,  3.67it/s]
+ 57%|██████████████████████████████████████                             | 120/211 [00:34<00:24,  3.67it/s]
+ 57%|██████████████████████████████████████                             | 120/211 [00:34<00:24,  3.67it/s]
+ 57%|██████████████████████████████████████                             | 120/211 [00:34<00:24,  3.67it/s]
+ 57%|██████████████████████████████████████                             | 120/211 [00:34<00:24,  3.67it/s]
+ 57%|██████████████████████████████████████                             | 120/211 [00:34<00:24,  3.67it/s]
+***** eval metrics *****
+  epoch                   =       80.0
+  eval_loss               =     0.3218
+  eval_runtime            = 0:01:01.39
+  eval_samples            =       1681
+  eval_samples_per_second =      27.38
+  eval_steps_per_second   =      3.437
+  eval_wer                =     0.3095
+ 57%|██████████████████████████████████████                             | 120/211 [00:34<00:24,  3.67it/s]

wandb/run-20220204_170031-gzsqxey3/logs/debug-internal.log

@@ -8496,3 +8496,55 @@
 2022-02-04 20:59:40,934 DEBUG   HandlerThread:622748 [handler.py:handle_request():130] handle_request: stop_status
 2022-02-04 20:59:40,934 DEBUG   SenderThread:622748 [sender.py:send_request():248] send_request: stop_status
 2022-02-04 20:59:41,262 INFO    Thread-8  :622748 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/wav2vec2-large-xls-r-300m-el/wandb/run-20220204_170031-gzsqxey3/files/output.log
+2022-02-04 20:59:45,264 INFO    Thread-8  :622748 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/wav2vec2-large-xls-r-300m-el/wandb/run-20220204_170031-gzsqxey3/files/output.log
+2022-02-04 20:59:49,266 INFO    Thread-8  :622748 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/wav2vec2-large-xls-r-300m-el/wandb/run-20220204_170031-gzsqxey3/files/output.log
+2022-02-04 20:59:51,267 INFO    Thread-8  :622748 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/wav2vec2-large-xls-r-300m-el/wandb/run-20220204_170031-gzsqxey3/files/output.log
+2022-02-04 20:59:53,268 INFO    Thread-8  :622748 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/wav2vec2-large-xls-r-300m-el/wandb/run-20220204_170031-gzsqxey3/files/output.log
+2022-02-04 20:59:55,270 INFO    Thread-8  :622748 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/wav2vec2-large-xls-r-300m-el/wandb/run-20220204_170031-gzsqxey3/files/output.log
+2022-02-04 20:59:56,094 DEBUG   HandlerThread:622748 [handler.py:handle_request():130] handle_request: stop_status
+2022-02-04 20:59:56,095 DEBUG   SenderThread:622748 [sender.py:send_request():248] send_request: stop_status
+2022-02-04 20:59:57,271 INFO    Thread-8  :622748 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/wav2vec2-large-xls-r-300m-el/wandb/run-20220204_170031-gzsqxey3/files/output.log
+2022-02-04 20:59:59,272 INFO    Thread-8  :622748 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/wav2vec2-large-xls-r-300m-el/wandb/run-20220204_170031-gzsqxey3/files/output.log
+2022-02-04 21:00:01,273 INFO    Thread-8  :622748 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/wav2vec2-large-xls-r-300m-el/wandb/run-20220204_170031-gzsqxey3/files/output.log
+2022-02-04 21:00:03,275 INFO    Thread-8  :622748 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/wav2vec2-large-xls-r-300m-el/wandb/run-20220204_170031-gzsqxey3/files/output.log
+2022-02-04 21:00:03,751 DEBUG   SenderThread:622748 [sender.py:send():234] send: stats
+2022-02-04 21:00:05,276 INFO    Thread-8  :622748 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/wav2vec2-large-xls-r-300m-el/wandb/run-20220204_170031-gzsqxey3/files/output.log
+2022-02-04 21:00:07,277 INFO    Thread-8  :622748 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/wav2vec2-large-xls-r-300m-el/wandb/run-20220204_170031-gzsqxey3/files/output.log
+2022-02-04 21:00:09,279 INFO    Thread-8  :622748 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/wav2vec2-large-xls-r-300m-el/wandb/run-20220204_170031-gzsqxey3/files/output.log
+2022-02-04 21:00:11,281 INFO    Thread-8  :622748 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/wav2vec2-large-xls-r-300m-el/wandb/run-20220204_170031-gzsqxey3/files/output.log
+2022-02-04 21:00:11,363 DEBUG   HandlerThread:622748 [handler.py:handle_request():130] handle_request: stop_status
+2022-02-04 21:00:11,364 DEBUG   SenderThread:622748 [sender.py:send_request():248] send_request: stop_status
+2022-02-04 21:00:13,282 INFO    Thread-8  :622748 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/wav2vec2-large-xls-r-300m-el/wandb/run-20220204_170031-gzsqxey3/files/output.log
+2022-02-04 21:00:15,284 INFO    Thread-8  :622748 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/wav2vec2-large-xls-r-300m-el/wandb/run-20220204_170031-gzsqxey3/files/output.log
+2022-02-04 21:00:17,285 INFO    Thread-8  :622748 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/wav2vec2-large-xls-r-300m-el/wandb/run-20220204_170031-gzsqxey3/files/output.log
+2022-02-04 21:00:19,286 INFO    Thread-8  :622748 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/wav2vec2-large-xls-r-300m-el/wandb/run-20220204_170031-gzsqxey3/files/output.log
+2022-02-04 21:00:21,288 INFO    Thread-8  :622748 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/wav2vec2-large-xls-r-300m-el/wandb/run-20220204_170031-gzsqxey3/files/output.log
+2022-02-04 21:00:23,290 INFO    Thread-8  :622748 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/wav2vec2-large-xls-r-300m-el/wandb/run-20220204_170031-gzsqxey3/files/output.log
+2022-02-04 21:00:25,292 INFO    Thread-8  :622748 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/wav2vec2-large-xls-r-300m-el/wandb/run-20220204_170031-gzsqxey3/files/output.log
+2022-02-04 21:00:26,607 DEBUG   HandlerThread:622748 [handler.py:handle_request():130] handle_request: stop_status
+2022-02-04 21:00:26,607 DEBUG   SenderThread:622748 [sender.py:send_request():248] send_request: stop_status
+2022-02-04 21:00:27,294 INFO    Thread-8  :622748 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/wav2vec2-large-xls-r-300m-el/wandb/run-20220204_170031-gzsqxey3/files/output.log
+2022-02-04 21:00:29,296 INFO    Thread-8  :622748 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/wav2vec2-large-xls-r-300m-el/wandb/run-20220204_170031-gzsqxey3/files/output.log
+2022-02-04 21:00:31,297 INFO    Thread-8  :622748 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/wav2vec2-large-xls-r-300m-el/wandb/run-20220204_170031-gzsqxey3/files/output.log
+2022-02-04 21:00:33,299 INFO    Thread-8  :622748 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/wav2vec2-large-xls-r-300m-el/wandb/run-20220204_170031-gzsqxey3/files/output.log
+2022-02-04 21:00:33,961 DEBUG   SenderThread:622748 [sender.py:send():234] send: stats
+2022-02-04 21:00:35,301 INFO    Thread-8  :622748 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/wav2vec2-large-xls-r-300m-el/wandb/run-20220204_170031-gzsqxey3/files/output.log
+2022-02-04 21:00:38,303 INFO    Thread-8  :622748 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/wav2vec2-large-xls-r-300m-el/wandb/run-20220204_170031-gzsqxey3/files/output.log
+2022-02-04 21:00:40,304 INFO    Thread-8  :622748 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/wav2vec2-large-xls-r-300m-el/wandb/run-20220204_170031-gzsqxey3/files/output.log
+2022-02-04 21:00:41,856 DEBUG   HandlerThread:622748 [handler.py:handle_request():130] handle_request: stop_status
+2022-02-04 21:00:41,856 DEBUG   SenderThread:622748 [sender.py:send_request():248] send_request: stop_status
+2022-02-04 21:00:42,306 INFO    Thread-8  :622748 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/wav2vec2-large-xls-r-300m-el/wandb/run-20220204_170031-gzsqxey3/files/output.log
+2022-02-04 21:00:44,307 INFO    Thread-8  :622748 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/wav2vec2-large-xls-r-300m-el/wandb/run-20220204_170031-gzsqxey3/files/output.log
+2022-02-04 21:00:46,308 INFO    Thread-8  :622748 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/wav2vec2-large-xls-r-300m-el/wandb/run-20220204_170031-gzsqxey3/files/output.log
+2022-02-04 21:00:48,309 INFO    Thread-8  :622748 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/wav2vec2-large-xls-r-300m-el/wandb/run-20220204_170031-gzsqxey3/files/output.log
+2022-02-04 21:00:50,311 INFO    Thread-8  :622748 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/wav2vec2-large-xls-r-300m-el/wandb/run-20220204_170031-gzsqxey3/files/output.log
+2022-02-04 21:00:50,741 DEBUG   SenderThread:622748 [sender.py:send():234] send: history
+2022-02-04 21:00:50,780 DEBUG   SenderThread:622748 [sender.py:send():234] send: summary
+2022-02-04 21:00:50,823 INFO    SenderThread:622748 [sender.py:_save_file():939] saving file wandb-summary.json with policy end
+2022-02-04 21:00:51,312 INFO    Thread-8  :622748 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/wav2vec2-large-xls-r-300m-el/wandb/run-20220204_170031-gzsqxey3/files/wandb-summary.json
+2022-02-04 21:00:51,312 INFO    Thread-8  :622748 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/wav2vec2-large-xls-r-300m-el/wandb/run-20220204_170031-gzsqxey3/files/output.log
+2022-02-04 21:00:52,313 INFO    Thread-8  :622748 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/wav2vec2-large-xls-r-300m-el/wandb/run-20220204_170031-gzsqxey3/files/output.log
+2022-02-04 21:00:54,315 INFO    Thread-8  :622748 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/wav2vec2-large-xls-r-300m-el/wandb/run-20220204_170031-gzsqxey3/files/output.log
+2022-02-04 21:00:57,096 DEBUG   HandlerThread:622748 [handler.py:handle_request():130] handle_request: stop_status
+2022-02-04 21:00:57,097 DEBUG   SenderThread:622748 [sender.py:send_request():248] send_request: stop_status
+2022-02-04 21:01:04,178 DEBUG   SenderThread:622748 [sender.py:send():234] send: stats