End of training

all_results.json

@@ -0,0 +1,14 @@
+{
+    "epoch": 40.0,
+    "eval_loss": 0.5502147674560547,
+    "eval_runtime": 99.1578,
+    "eval_samples": 1882,
+    "eval_samples_per_second": 18.98,
+    "eval_steps_per_second": 2.38,
+    "eval_wer": 0.4042111019961717,
+    "train_loss": 1.0643680714425587,
+    "train_runtime": 10312.3076,
+    "train_samples": 4007,
+    "train_samples_per_second": 15.543,
+    "train_steps_per_second": 0.244
+}

eval_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 40.0,
+    "eval_loss": 0.5502147674560547,
+    "eval_runtime": 99.1578,
+    "eval_samples": 1882,
+    "eval_samples_per_second": 18.98,
+    "eval_steps_per_second": 2.38,
+    "eval_wer": 0.4042111019961717
+}

train_results.json

@@ -0,0 +1,8 @@
+{
+    "epoch": 40.0,
+    "train_loss": 1.0643680714425587,
+    "train_runtime": 10312.3076,
+    "train_samples": 4007,
+    "train_samples_per_second": 15.543,
+    "train_steps_per_second": 0.244
+}

trainer_state.json

@@ -0,0 +1,109 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 40.0,
+  "global_step": 2520,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 6.35,
+      "eval_loss": 0.9379647374153137,
+      "eval_runtime": 95.9228,
+      "eval_samples_per_second": 19.62,
+      "eval_steps_per_second": 2.46,
+      "eval_wer": 0.7901558654634947,
+      "step": 400
+    },
+    {
+      "epoch": 7.94,
+      "learning_rate": 0.00029939999999999996,
+      "loss": 3.2674,
+      "step": 500
+    },
+    {
+      "epoch": 12.7,
+      "eval_loss": 0.5793638825416565,
+      "eval_runtime": 96.8489,
+      "eval_samples_per_second": 19.432,
+      "eval_steps_per_second": 2.437,
+      "eval_wer": 0.530872299699207,
+      "step": 800
+    },
+    {
+      "epoch": 15.87,
+      "learning_rate": 0.00022589108910891088,
+      "loss": 0.7531,
+      "step": 1000
+    },
+    {
+      "epoch": 19.05,
+      "eval_loss": 0.5748894810676575,
+      "eval_runtime": 94.3353,
+      "eval_samples_per_second": 19.95,
+      "eval_steps_per_second": 2.502,
+      "eval_wer": 0.4814875581077386,
+      "step": 1200
+    },
+    {
+      "epoch": 23.81,
+      "learning_rate": 0.0001516336633663366,
+      "loss": 0.5382,
+      "step": 1500
+    },
+    {
+      "epoch": 25.4,
+      "eval_loss": 0.5529599785804749,
+      "eval_runtime": 100.0797,
+      "eval_samples_per_second": 18.805,
+      "eval_steps_per_second": 2.358,
+      "eval_wer": 0.44473612250478534,
+      "step": 1600
+    },
+    {
+      "epoch": 31.75,
+      "learning_rate": 7.737623762376237e-05,
+      "loss": 0.4293,
+      "step": 2000
+    },
+    {
+      "epoch": 31.75,
+      "eval_loss": 0.5709025263786316,
+      "eval_runtime": 93.9383,
+      "eval_samples_per_second": 20.034,
+      "eval_steps_per_second": 2.512,
+      "eval_wer": 0.42373530216024063,
+      "step": 2000
+    },
+    {
+      "epoch": 38.1,
+      "eval_loss": 0.5476023554801941,
+      "eval_runtime": 95.0441,
+      "eval_samples_per_second": 19.801,
+      "eval_steps_per_second": 2.483,
+      "eval_wer": 0.4059064807219032,
+      "step": 2400
+    },
+    {
+      "epoch": 39.68,
+      "learning_rate": 3.1188118811881187e-06,
+      "loss": 0.3635,
+      "step": 2500
+    },
+    {
+      "epoch": 40.0,
+      "step": 2520,
+      "total_flos": 2.64515330270157e+19,
+      "train_loss": 1.0643680714425587,
+      "train_runtime": 10312.3076,
+      "train_samples_per_second": 15.543,
+      "train_steps_per_second": 0.244
+    }
+  ],
+  "max_steps": 2520,
+  "num_train_epochs": 40,
+  "total_flos": 2.64515330270157e+19,
+  "trial_name": null,
+  "trial_params": null
+}

wandb/run-20220204_121013-1s9mhb6h/files/output.log

@@ -2866,3 +2866,79 @@ Upload file pytorch_model.bin: 100%|██████████████
    8eaa216..f311688  main -> main
 Upload file pytorch_model.bin: 100%|███████████████████████████████████████████████████████| 1.18G/1.18G [00:41<00:00, 30.4MB/s]
 Dropping the following result as it does not have all the necessary fields:
+{'dataset': {'name': 'common_voice', 'type': 'common_voice', 'args': 'mn'}}
+02/04/2022 15:04:07 - WARNING - huggingface_hub.repository - To https://huggingface.co/ayameRushia/wav2vec2-large-xls-r-300m-mn
+   f311688..db0751c  main -> main
+To https://huggingface.co/ayameRushia/wav2vec2-large-xls-r-300m-mn
+   f311688..db0751c  main -> main
+***** train metrics *****
+  epoch                    =       40.0
+  train_loss               =     1.0644
+  train_runtime            = 2:51:52.30
+  train_samples            =       4007
+  train_samples_per_second =     15.543
+  train_steps_per_second   =      0.244
+02/04/2022 15:04:10 - INFO - __main__ - *** Evaluate ***
+The following columns in the evaluation set  don't have a corresponding argument in `Wav2Vec2ForCTC.forward` and have been ignored: input_length.
+***** Running Evaluation *****
+  Num examples = 1882
+  Batch size = 8
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+100%|█████████████████████████████████████████████████████████████████████████████████████████| 236/236 [01:37<00:00,  3.10it/s]
+***** eval metrics *****
+  epoch                   =       40.0
+  eval_loss               =     0.5502
+  eval_runtime            = 0:01:39.15
+  eval_samples            =       1882
+  eval_samples_per_second =      18.98
+  eval_steps_per_second   =       2.38
+100%|█████████████████████████████████████████████████████████████████████████████████████████| 236/236 [01:38<00:00,  2.39it/s]
+Saving model checkpoint to ./
+Configuration saved in ./config.json
+Model weights saved in ./pytorch_model.bin

wandb/run-20220204_121013-1s9mhb6h/logs/debug-internal.log

@@ -5833,3 +5833,78 @@
 2022-02-04 15:04:03,894 DEBUG   HandlerThread:508894 [handler.py:handle_request():130] handle_request: stop_status
 2022-02-04 15:04:03,895 DEBUG   SenderThread:508894 [sender.py:send_request():248] send_request: stop_status
 2022-02-04 15:04:03,974 INFO    Thread-8  :508894 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/wav2vec2-large-xls-r-300m-mn/wandb/run-20220204_121013-1s9mhb6h/files/output.log
+2022-02-04 15:04:09,977 INFO    Thread-8  :508894 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/wav2vec2-large-xls-r-300m-mn/wandb/run-20220204_121013-1s9mhb6h/files/output.log
+2022-02-04 15:04:11,978 INFO    Thread-8  :508894 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/wav2vec2-large-xls-r-300m-mn/wandb/run-20220204_121013-1s9mhb6h/files/output.log
+2022-02-04 15:04:13,979 INFO    Thread-8  :508894 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/wav2vec2-large-xls-r-300m-mn/wandb/run-20220204_121013-1s9mhb6h/files/output.log
+2022-02-04 15:04:15,656 DEBUG   SenderThread:508894 [sender.py:send():234] send: stats
+2022-02-04 15:04:15,981 INFO    Thread-8  :508894 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/wav2vec2-large-xls-r-300m-mn/wandb/run-20220204_121013-1s9mhb6h/files/output.log
+2022-02-04 15:04:17,982 INFO    Thread-8  :508894 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/wav2vec2-large-xls-r-300m-mn/wandb/run-20220204_121013-1s9mhb6h/files/output.log
+2022-02-04 15:04:19,228 DEBUG   HandlerThread:508894 [handler.py:handle_request():130] handle_request: stop_status
+2022-02-04 15:04:19,229 DEBUG   SenderThread:508894 [sender.py:send_request():248] send_request: stop_status
+2022-02-04 15:04:19,984 INFO    Thread-8  :508894 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/wav2vec2-large-xls-r-300m-mn/wandb/run-20220204_121013-1s9mhb6h/files/output.log
+2022-02-04 15:04:21,985 INFO    Thread-8  :508894 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/wav2vec2-large-xls-r-300m-mn/wandb/run-20220204_121013-1s9mhb6h/files/output.log
+2022-02-04 15:04:23,986 INFO    Thread-8  :508894 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/wav2vec2-large-xls-r-300m-mn/wandb/run-20220204_121013-1s9mhb6h/files/output.log
+2022-02-04 15:04:26,988 INFO    Thread-8  :508894 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/wav2vec2-large-xls-r-300m-mn/wandb/run-20220204_121013-1s9mhb6h/files/output.log
+2022-02-04 15:04:28,989 INFO    Thread-8  :508894 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/wav2vec2-large-xls-r-300m-mn/wandb/run-20220204_121013-1s9mhb6h/files/output.log
+2022-02-04 15:04:30,990 INFO    Thread-8  :508894 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/wav2vec2-large-xls-r-300m-mn/wandb/run-20220204_121013-1s9mhb6h/files/output.log
+2022-02-04 15:04:32,992 INFO    Thread-8  :508894 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/wav2vec2-large-xls-r-300m-mn/wandb/run-20220204_121013-1s9mhb6h/files/output.log
+2022-02-04 15:04:34,614 DEBUG   HandlerThread:508894 [handler.py:handle_request():130] handle_request: stop_status
+2022-02-04 15:04:34,615 DEBUG   SenderThread:508894 [sender.py:send_request():248] send_request: stop_status
+2022-02-04 15:04:34,993 INFO    Thread-8  :508894 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/wav2vec2-large-xls-r-300m-mn/wandb/run-20220204_121013-1s9mhb6h/files/output.log
+2022-02-04 15:04:36,995 INFO    Thread-8  :508894 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/wav2vec2-large-xls-r-300m-mn/wandb/run-20220204_121013-1s9mhb6h/files/output.log
+2022-02-04 15:04:38,996 INFO    Thread-8  :508894 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/wav2vec2-large-xls-r-300m-mn/wandb/run-20220204_121013-1s9mhb6h/files/output.log
+2022-02-04 15:04:40,997 INFO    Thread-8  :508894 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/wav2vec2-large-xls-r-300m-mn/wandb/run-20220204_121013-1s9mhb6h/files/output.log
+2022-02-04 15:04:42,998 INFO    Thread-8  :508894 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/wav2vec2-large-xls-r-300m-mn/wandb/run-20220204_121013-1s9mhb6h/files/output.log
+2022-02-04 15:04:44,999 INFO    Thread-8  :508894 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/wav2vec2-large-xls-r-300m-mn/wandb/run-20220204_121013-1s9mhb6h/files/output.log
+2022-02-04 15:04:45,901 DEBUG   SenderThread:508894 [sender.py:send():234] send: stats
+2022-02-04 15:04:47,008 INFO    Thread-8  :508894 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/wav2vec2-large-xls-r-300m-mn/wandb/run-20220204_121013-1s9mhb6h/files/output.log
+2022-02-04 15:04:49,009 INFO    Thread-8  :508894 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/wav2vec2-large-xls-r-300m-mn/wandb/run-20220204_121013-1s9mhb6h/files/output.log
+2022-02-04 15:04:49,838 DEBUG   HandlerThread:508894 [handler.py:handle_request():130] handle_request: stop_status
+2022-02-04 15:04:49,839 DEBUG   SenderThread:508894 [sender.py:send_request():248] send_request: stop_status
+2022-02-04 15:04:51,013 INFO    Thread-8  :508894 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/wav2vec2-large-xls-r-300m-mn/wandb/run-20220204_121013-1s9mhb6h/files/output.log
+2022-02-04 15:04:53,014 INFO    Thread-8  :508894 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/wav2vec2-large-xls-r-300m-mn/wandb/run-20220204_121013-1s9mhb6h/files/output.log
+2022-02-04 15:04:56,016 INFO    Thread-8  :508894 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/wav2vec2-large-xls-r-300m-mn/wandb/run-20220204_121013-1s9mhb6h/files/output.log
+2022-02-04 15:04:58,017 INFO    Thread-8  :508894 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/wav2vec2-large-xls-r-300m-mn/wandb/run-20220204_121013-1s9mhb6h/files/output.log
+2022-02-04 15:05:00,018 INFO    Thread-8  :508894 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/wav2vec2-large-xls-r-300m-mn/wandb/run-20220204_121013-1s9mhb6h/files/output.log
+2022-02-04 15:05:02,019 INFO    Thread-8  :508894 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/wav2vec2-large-xls-r-300m-mn/wandb/run-20220204_121013-1s9mhb6h/files/output.log
+2022-02-04 15:05:04,020 INFO    Thread-8  :508894 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/wav2vec2-large-xls-r-300m-mn/wandb/run-20220204_121013-1s9mhb6h/files/output.log
+2022-02-04 15:05:05,117 DEBUG   HandlerThread:508894 [handler.py:handle_request():130] handle_request: stop_status
+2022-02-04 15:05:05,118 DEBUG   SenderThread:508894 [sender.py:send_request():248] send_request: stop_status
+2022-02-04 15:05:06,022 INFO    Thread-8  :508894 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/wav2vec2-large-xls-r-300m-mn/wandb/run-20220204_121013-1s9mhb6h/files/output.log
+2022-02-04 15:05:08,023 INFO    Thread-8  :508894 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/wav2vec2-large-xls-r-300m-mn/wandb/run-20220204_121013-1s9mhb6h/files/output.log
+2022-02-04 15:05:10,024 INFO    Thread-8  :508894 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/wav2vec2-large-xls-r-300m-mn/wandb/run-20220204_121013-1s9mhb6h/files/output.log
+2022-02-04 15:05:12,025 INFO    Thread-8  :508894 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/wav2vec2-large-xls-r-300m-mn/wandb/run-20220204_121013-1s9mhb6h/files/output.log
+2022-02-04 15:05:14,026 INFO    Thread-8  :508894 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/wav2vec2-large-xls-r-300m-mn/wandb/run-20220204_121013-1s9mhb6h/files/output.log
+2022-02-04 15:05:16,027 INFO    Thread-8  :508894 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/wav2vec2-large-xls-r-300m-mn/wandb/run-20220204_121013-1s9mhb6h/files/output.log
+2022-02-04 15:05:16,114 DEBUG   SenderThread:508894 [sender.py:send():234] send: stats
+2022-02-04 15:05:18,029 INFO    Thread-8  :508894 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/wav2vec2-large-xls-r-300m-mn/wandb/run-20220204_121013-1s9mhb6h/files/output.log
+2022-02-04 15:05:20,030 INFO    Thread-8  :508894 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/wav2vec2-large-xls-r-300m-mn/wandb/run-20220204_121013-1s9mhb6h/files/output.log
+2022-02-04 15:05:20,382 DEBUG   HandlerThread:508894 [handler.py:handle_request():130] handle_request: stop_status
+2022-02-04 15:05:20,382 DEBUG   SenderThread:508894 [sender.py:send_request():248] send_request: stop_status
+2022-02-04 15:05:22,031 INFO    Thread-8  :508894 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/wav2vec2-large-xls-r-300m-mn/wandb/run-20220204_121013-1s9mhb6h/files/output.log
+2022-02-04 15:05:24,032 INFO    Thread-8  :508894 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/wav2vec2-large-xls-r-300m-mn/wandb/run-20220204_121013-1s9mhb6h/files/output.log
+2022-02-04 15:05:27,034 INFO    Thread-8  :508894 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/wav2vec2-large-xls-r-300m-mn/wandb/run-20220204_121013-1s9mhb6h/files/output.log
+2022-02-04 15:05:29,035 INFO    Thread-8  :508894 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/wav2vec2-large-xls-r-300m-mn/wandb/run-20220204_121013-1s9mhb6h/files/output.log
+2022-02-04 15:05:31,036 INFO    Thread-8  :508894 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/wav2vec2-large-xls-r-300m-mn/wandb/run-20220204_121013-1s9mhb6h/files/output.log
+2022-02-04 15:05:33,038 INFO    Thread-8  :508894 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/wav2vec2-large-xls-r-300m-mn/wandb/run-20220204_121013-1s9mhb6h/files/output.log
+2022-02-04 15:05:35,039 INFO    Thread-8  :508894 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/wav2vec2-large-xls-r-300m-mn/wandb/run-20220204_121013-1s9mhb6h/files/output.log
+2022-02-04 15:05:35,648 DEBUG   HandlerThread:508894 [handler.py:handle_request():130] handle_request: stop_status
+2022-02-04 15:05:35,649 DEBUG   SenderThread:508894 [sender.py:send_request():248] send_request: stop_status
+2022-02-04 15:05:37,040 INFO    Thread-8  :508894 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/wav2vec2-large-xls-r-300m-mn/wandb/run-20220204_121013-1s9mhb6h/files/output.log
+2022-02-04 15:05:39,042 INFO    Thread-8  :508894 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/wav2vec2-large-xls-r-300m-mn/wandb/run-20220204_121013-1s9mhb6h/files/output.log
+2022-02-04 15:05:41,042 INFO    Thread-8  :508894 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/wav2vec2-large-xls-r-300m-mn/wandb/run-20220204_121013-1s9mhb6h/files/output.log
+2022-02-04 15:05:43,044 INFO    Thread-8  :508894 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/wav2vec2-large-xls-r-300m-mn/wandb/run-20220204_121013-1s9mhb6h/files/output.log
+2022-02-04 15:05:45,045 INFO    Thread-8  :508894 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/wav2vec2-large-xls-r-300m-mn/wandb/run-20220204_121013-1s9mhb6h/files/output.log
+2022-02-04 15:05:46,349 DEBUG   SenderThread:508894 [sender.py:send():234] send: stats
+2022-02-04 15:05:47,046 INFO    Thread-8  :508894 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/wav2vec2-large-xls-r-300m-mn/wandb/run-20220204_121013-1s9mhb6h/files/output.log
+2022-02-04 15:05:49,048 INFO    Thread-8  :508894 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/wav2vec2-large-xls-r-300m-mn/wandb/run-20220204_121013-1s9mhb6h/files/output.log
+2022-02-04 15:05:49,718 DEBUG   SenderThread:508894 [sender.py:send():234] send: history
+2022-02-04 15:05:49,752 DEBUG   SenderThread:508894 [sender.py:send():234] send: summary
+2022-02-04 15:05:49,809 INFO    SenderThread:508894 [sender.py:_save_file():939] saving file wandb-summary.json with policy end
+2022-02-04 15:05:50,048 INFO    Thread-8  :508894 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/wav2vec2-large-xls-r-300m-mn/wandb/run-20220204_121013-1s9mhb6h/files/wandb-summary.json
+2022-02-04 15:05:50,820 DEBUG   HandlerThread:508894 [handler.py:handle_request():130] handle_request: stop_status
+2022-02-04 15:05:50,821 DEBUG   SenderThread:508894 [sender.py:send_request():248] send_request: stop_status
+2022-02-04 15:05:51,049 INFO    Thread-8  :508894 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/wav2vec2-large-xls-r-300m-mn/wandb/run-20220204_121013-1s9mhb6h/files/output.log
+2022-02-04 15:05:53,050 INFO    Thread-8  :508894 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/wav2vec2-large-xls-r-300m-mn/wandb/run-20220204_121013-1s9mhb6h/files/output.log
+2022-02-04 15:06:05,986 DEBUG   HandlerThread:508894 [handler.py:handle_request():130] handle_request: stop_status
+2022-02-04 15:06:05,986 DEBUG   SenderThread:508894 [sender.py:send_request():248] send_request: stop_status