Training in progress, step 2000

.gitattributes

@@ -26,3 +26,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zstandard filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
 wandb/run-20220129_131141-h6nhqm30/run-h6nhqm30.wandb filter=lfs diff=lfs merge=lfs -text
+wandb/run-20220129_215451-1vipdbow/run-1vipdbow.wandb filter=lfs diff=lfs merge=lfs -text

pytorch_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2c62db10c9bd2b82f65dad64fa2f0c1c6d5b6e343d156b0b844fedb94f0b92c8
+oid sha256:49c9b3549be0994812f899b8789d7940b7291c45a024a4a3637aaf87584f355d
 size 1262817457

wandb/run-20220129_215451-1vipdbow/files/output.log

@@ -1781,3 +1781,597 @@ Configuration saved in ./checkpoint-1500/config.json
 {'eval_loss': 0.881136417388916, 'eval_wer': 0.7127472384241911, 'eval_runtime': 297.1806, 'eval_samples_per_second': 19.49, 'eval_steps_per_second': 0.306, 'epoch': 0.22}
 Model weights saved in ./checkpoint-1500/pytorch_model.bin
 Configuration saved in ./checkpoint-1500/preprocessor_config.json
+Configuration saved in ./preprocessor_config.json
+Deleting older checkpoint [checkpoint-500] due to args.save_total_limit
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 12%|██████████████████                                                                                                                                          | 1599/13822 [1:43:15<5:47:32,  1.71s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 12%|███████████████████▏                                                                                                                                        | 1698/13822 [1:48:29<6:15:51,  1.86s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 13%|████████████████████▎                                                                                                                                       | 1800/13822 [1:53:46<5:31:16,  1.65s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 14%|█████████████████████▍                                                                                                                                      | 1900/13822 [1:59:00<5:35:53,  1.69s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 14%|██████████████████████▌                                                                                                                                     | 2000/13822 [2:04:15<5:37:37,  1.71s/it]The following columns in the evaluation set  don't have a corresponding argument in `Wav2Vec2ForCTC.forward` and have been ignored: input_length.
+***** Running Evaluation *****
+  Num examples = 5792
+  Batch size = 64
+{'loss': 1.3488, 'learning_rate': 9.597467943515663e-05, 'epoch': 0.29}
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 91/91 [04:08<00:00,  2.39s/it]
+
+Configuration saved in ./checkpoint-2000/config.json
+Model weights saved in ./checkpoint-2000/pytorch_model.bin
+Configuration saved in ./checkpoint-2000/preprocessor_config.json
+Configuration saved in ./preprocessor_config.json
+01/30/2022 00:05:21 - WARNING - huggingface_hub.repository - Adding files tracked by Git LFS: ['wandb/run-20220129_215451-1vipdbow/run-1vipdbow.wandb']. This may take a bit of time if the files are large.