Training in progress, step 1000

pytorch_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e5447c8090dadc8995e3c0111e08b30c73a5a9c06c4b2511e4a5bcb050133486
+oid sha256:7102a6b0b37c636335ba700287e7fcad791b55cc59af204105d8d3dfb248cf52
 size 6173655480

run.sh

@@ -6,17 +6,17 @@ python run_speech_recognition_seq2seq_streaming.py \
 	--train_split_name="train+validation" \
 	--eval_split_name="test" \
 	--model_index_name="Whisper Large Vietnamese" \
-	--max_steps="1000" \
+	--max_steps="5000" \
 	--output_dir="./" \
 	--per_device_train_batch_size="32" \
 	--per_device_eval_batch_size="16" \
 	--logging_steps="25" \
-	--learning_rate="1e-5" \
+	--learning_rate="1e-6" \
 	--warmup_steps="100" \
 	--evaluation_strategy="steps" \
-	--eval_steps="250" \
+	--eval_steps="1000" \
 	--save_strategy="steps" \
-	--save_steps="250" \
+	--save_steps="1000" \
 	--generation_max_length="225" \
 	--length_column_name="input_length" \
 	--max_duration_in_seconds="30" \

run_speech_recognition_seq2seq_streaming.py

@@ -501,10 +501,11 @@ def main():
     def is_audio_in_length_range(length):
         return min_input_length < length < max_input_length
 
-    vectorized_datasets["train"] = vectorized_datasets["train"].filter(
-        is_audio_in_length_range,
-        input_columns=["input_length"],
-    )
+    if training_args.do_train:
+        vectorized_datasets["train"] = vectorized_datasets["train"].filter(
+            is_audio_in_length_range,
+            input_columns=["input_length"],
+        )
 
     # 8. Load Metric
     metric = evaluate.load("wer")
@@ -522,6 +523,9 @@ def main():
         if do_normalize_eval:
             pred_str = [normalizer(pred) for pred in pred_str]
             label_str = [normalizer(label) for label in label_str]
+            # filtering step to only evaluate the samples that correspond to non-zero references:
+            pred_str = [pred_str[i] for i in range(len(pred_str)) if len(label_str[i]) > 0]
+            label_str = [label_str[i] for i in range(len(label_str)) if len(label_str[i]) > 0]
 
         wer = 100 * metric.compute(predictions=pred_str, references=label_str)
 

runs/Dec12_16-17-24_spartan-gpgpu115.hpc.unimelb.edu.au/1670822323.3846912/events.out.tfevents.1670822323.spartan-gpgpu115.hpc.unimelb.edu.au.248583.1

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6ce0047001e53c6fe1d14167b5b5fb1fa5f87196f1e7de29cd173591852bc6cc
+size 5730

runs/Dec12_16-17-24_spartan-gpgpu115.hpc.unimelb.edu.au/events.out.tfevents.1670822323.spartan-gpgpu115.hpc.unimelb.edu.au.248583.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9dea4a566557912ace98f6d6d7b35a7f9a5f72a52f0a425e7e32c22790d66d7f
+size 4203

runs/Dec12_18-44-54_spartan-gpgpu115.hpc.unimelb.edu.au/1670831139.1402473/events.out.tfevents.1670831139.spartan-gpgpu115.hpc.unimelb.edu.au.146080.1

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c7e19cba18916450c4882276ba80316dc46b320da86e129cb8c6e7cad1ac7d16
+size 5730

runs/Dec12_18-44-54_spartan-gpgpu115.hpc.unimelb.edu.au/events.out.tfevents.1670831139.spartan-gpgpu115.hpc.unimelb.edu.au.146080.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a470c12c1f15c6ba258717c9b1462892974cc1419b9296ef55fd07467047b3e0
+size 10786

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a94b1449f30d120ad9017982c5fa06a4324241f23aab28f07da46bce5ae0d57d
+oid sha256:46a814996c8c7be78345a57772149f9ab0950a112cff44d30a75b7b82eb3b63e
 size 3515