debug decoding

run_speech_recognition_seq2seq_streaming.py

@@ -432,36 +432,6 @@ def load_maybe_streaming_dataset(
         return dataset
 
 
-def load_common_voice_like_dataset(
-    dataset_name,
-    config,
-    split,
-    audio_column_name=None,
-    sampling_rate=None,
-    streaming=True,
-    use_auth_token=False
-):
-
-    """
-    Utility function to load the Common Voice dataset.
-    """
-    dataset = load_dataset(
-        dataset_name,
-        config,
-        split=split,
-        streaming=streaming,
-        use_auth_token=use_auth_token,
-    )
-    if audio_column_name is not None and sampling_rate is not None:
-        dataset = dataset.cast_column(
-            audio_column_name, datasets.features.Audio(sampling_rate=sampling_rate)
-        )
-    return dataset
-
-
-# def load_nst_nbailab(config, split, )
-
-
 def main():
     # 1. Parse input arguments
     # See all possible arguments in src/transformers/training_args.py
@@ -476,8 +446,6 @@ def main():
         model_args, data_args, training_args = parser.parse_json_file(json_file=os.path.abspath(sys.argv[1]))
     else:
         model_args, data_args, training_args = parser.parse_args_into_dataclasses()
-    training_args.do_train = True
-    training_args.do_eval = True
 
     # Sending telemetry. Tracking the example usage helps us better allocate resources to maintain them. The
     # information sent is the one passed as arguments along with your Python/PyTorch versions.
@@ -541,6 +509,9 @@ def main():
     logger.info("*** Load dataset ***")
     raw_datasets = IterableDatasetDict() if data_args.streaming else DatasetDict()
 
+    if len(data_args.language_eval.split(",")) > 1:
+        raise ValueError("Implementation does not support multiple language evaluation.")
+
     if training_args.do_train:
         raw_datasets["train"] = load_maybe_streaming_dataset(
             data_args.dataset_train_name,
@@ -807,10 +778,31 @@ def main():
         trainer.save_state()
         logger.info("*** State saved ***")
 
+    # Run a test prediction to check outputs
+    predictions = trainer.predict(
+        test_dataset=vectorized_datasets["test"].shuffle(seed=training_args.seed).select(range(5)),
+        metric_key_prefix="test",
+        max_length=training_args.generation_max_length,
+        num_beams=training_args.generation_num_beams,
+    )
+    logger.info("*** Test prediction done ***")
+    predictions = processor.batch_decode(predictions.predictions)
+    labels = processor.batch_decode(predictions.label_ids)
+    pred_labels = [f"Prediction: {pred}\nLabel: {label}\n" for pred, label in zip(predictions, labels)]
+    logger.info("Before setting language and task")
+    logger.info(f"{pred_labels}")
+    trainer.data_collator.processor.tokenizer.set_prefix_tokens(language=data_args.language_eval, task=data_args.task)
+    predictions = processor.batch_decode(predictions.predictions)
+    labels = processor.batch_decode(predictions.label_ids)
+    pred_labels = [f"Prediction: {pred}\nLabel: {label}\n" for pred, label in zip(predictions, labels)]
+    logger.info("After setting language and task")
+    logger.info(f"{pred_labels}")
+
     # 13. Evaluation
     results = {}
     if training_args.do_eval:
         logger.info("*** Evaluate ***")
+        
         metrics = trainer.evaluate(
             metric_key_prefix="eval",
             max_length=training_args.generation_max_length,

test_run_nordic.sh

@@ -4,9 +4,9 @@ python $1run_speech_recognition_seq2seq_streaming.py \
 	--dataset_train_config_name="sv-SE,da,nn-NO,nst,no-distant,16K_mp3_nynorsk,sv_se,da_dk,nb_no" \
 	--language_train="sv,da,no,sv,no,no,sv,da,no" \
 	--train_split_name="train+validation,train+validation,train+validation,train,train+test,train+validation,train+validation,train+validation,train+validation" \
-	--dataset_eval_name="mozilla-foundation/common_voice_11_0,mozilla-foundation/common_voice_11_0,mozilla-foundation/common_voice_11_0" \
-	--dataset_eval_config_name="sv-SE,da,nn-NO" \
-	--language_eval="sv,da,no" \
+	--dataset_eval_name="mozilla-foundation/common_voice_11_0" \
+	--dataset_eval_config_name="sv-SE" \
+	--language_eval="sv" \
 	--eval_split_name="test" \
 	--model_index_name="Whisper Tiny Nordic" \
 	--max_train_samples="64" \