Training in progress, step 500

.ipynb_checkpoints/run-checkpoint.sh

@@ -4,12 +4,12 @@ python run_speech_recognition_ctc.py \
 	--dataset_config_name="sv-SE" \
 	--output_dir="./" \
 	--overwrite_output_dir \
-	--num_train_epochs="5" \
+	--num_train_epochs="50" \
 	--per_device_train_batch_size="8" \
 	--per_device_eval_batch_size="8" \
 	--gradient_accumulation_steps="4" \
 	--learning_rate="7.5e-5" \
-	--warmup_steps="20" \
+	--warmup_steps="2000" \
 	--length_column_name="input_length" \
 	--evaluation_strategy="steps" \
 	--text_column_name="sentence" \

checkpoint-500/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:dab21b247ec51e4dee7afa84fedd77b13635e30b256cb9caa89e1dfc094123c9
+oid sha256:2c14be45652078c4081eae22591a3468388b3df1aa0b51a3891ac56aa39e8fb4
 size 2490362385

checkpoint-500/pytorch_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:401051ca4bb075cafe2856a328c8f6a5e3e87fa0c13df1de35ee6e7873004e73
+oid sha256:ff113c93cbec054547814830ddbe3b7e8b56d08002f35055cf5facbab2743f16
 size 1262075377

checkpoint-500/scaler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0c8bceadebe118ff459b01a775a8e9b38a6b8302c162d022f78d3646163e6486
+oid sha256:36e3cff1dc7d80e97893463000a3b7c05ebd26294d7bb5df853e56508bf1c337
 size 559

checkpoint-500/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3757aee7e707dcd124ce47058f7058fcb392f4c693c82774d9b2b5dcccf35b49
+oid sha256:8ad1a782987dd67d49f235620bab30ace2edd4b98948c7bdb2f642185a1f585b
 size 623

checkpoint-500/trainer_state.json

@@ -9,46 +9,46 @@
   "log_history": [
     {
       "epoch": 0.29,
-      "learning_rate": 7.151470588235293e-05,
-      "loss": 5.1135,
+      "learning_rate": 3.7125e-06,
+      "loss": 12.2713,
       "step": 100
     },
     {
       "epoch": 0.58,
-      "learning_rate": 6.710294117647058e-05,
-      "loss": 3.0957,
+      "learning_rate": 7.4625e-06,
+      "loss": 6.2026,
       "step": 200
     },
     {
       "epoch": 0.87,
-      "learning_rate": 6.269117647058824e-05,
-      "loss": 3.0078,
+      "learning_rate": 1.1212499999999998e-05,
+      "loss": 4.1374,
       "step": 300
     },
     {
       "epoch": 1.16,
-      "learning_rate": 5.827941176470588e-05,
-      "loss": 2.9785,
+      "learning_rate": 1.49625e-05,
+      "loss": 3.6755,
       "step": 400
     },
     {
       "epoch": 1.45,
-      "learning_rate": 5.3867647058823525e-05,
-      "loss": 2.9069,
+      "learning_rate": 1.8712499999999997e-05,
+      "loss": 3.3332,
       "step": 500
     },
     {
       "epoch": 1.45,
-      "eval_loss": 2.9046826362609863,
-      "eval_runtime": 128.7688,
-      "eval_samples_per_second": 35.878,
-      "eval_steps_per_second": 4.489,
+      "eval_loss": 3.292029857635498,
+      "eval_runtime": 137.2974,
+      "eval_samples_per_second": 33.65,
+      "eval_steps_per_second": 4.21,
       "eval_wer": 1.0,
       "step": 500
     }
   ],
-  "max_steps": 1720,
-  "num_train_epochs": 5,
+  "max_steps": 17200,
+  "num_train_epochs": 50,
   "total_flos": 1.4827194756605722e+18,
   "trial_name": null,
   "trial_params": null

checkpoint-500/training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1208f22bb7e06e1e9a51692db0520fbddfc3640941d51dfe45ba3188ada2ecbf
+oid sha256:5a168f126d02648193e16ba893c3e1ef9f2c2de91803928caf5e25532aff1325
 size 2991

pytorch_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f22fe96372581760966d6d3482ae1d1974fb3aaebb6f1e041246fb9b3244f3ef
+oid sha256:ff113c93cbec054547814830ddbe3b7e8b56d08002f35055cf5facbab2743f16
 size 1262075377

run.sh

@@ -4,12 +4,12 @@ python run_speech_recognition_ctc.py \
 	--dataset_config_name="sv-SE" \
 	--output_dir="./" \
 	--overwrite_output_dir \
-	--num_train_epochs="5" \
+	--num_train_epochs="50" \
 	--per_device_train_batch_size="8" \
 	--per_device_eval_batch_size="8" \
 	--gradient_accumulation_steps="4" \
 	--learning_rate="7.5e-5" \
-	--warmup_steps="20" \
+	--warmup_steps="2000" \
 	--length_column_name="input_length" \
 	--evaluation_strategy="steps" \
 	--text_column_name="sentence" \

special_tokens_map.json

@@ -1 +1 @@
-{"bos_token": "<s>", "eos_token": "</s>", "unk_token": "[UNK]", "pad_token": "[PAD]", "additional_special_tokens": [{"content": "<s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, {"content": "</s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}]}
+{"bos_token": "<s>", "eos_token": "</s>", "unk_token": "[UNK]", "pad_token": "[PAD]", "additional_special_tokens": [{"content": "<s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, {"content": "</s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, {"content": "<s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, {"content": "</s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}]}

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1208f22bb7e06e1e9a51692db0520fbddfc3640941d51dfe45ba3188ada2ecbf
+oid sha256:5a168f126d02648193e16ba893c3e1ef9f2c2de91803928caf5e25532aff1325
 size 2991