use multiple datasets

run_speech_recognition_seq2seq_streaming.py

@@ -394,12 +394,16 @@ def load_maybe_streaming_dataset(
                 else:
                     dataset = load_dataset(dataset_name, split=split_name, streaming=streaming, **kwargs)              
                 raw_datasets_features = list(dataset.features.keys())
+                logger.info(f"Dataset {dataset_name} - Features: {raw_datasets_features}")
                 if text_col_name_ref not in raw_datasets_features:
                     if len(text_column_names) == 1:
-                        raise ValueError("Column name not found in dataset.")
+                        raise ValueError("None of the text column names provided found in dataset."
+                                         f"Text columns: {text_column_names}"
+                                         f"Dataset columns: {raw_datasets_features}")
                     flag = False
                     for text_column_name in text_column_names:
-                        if text_column_name in raw_datasets_features: 
+                        if text_column_name in raw_datasets_features:
+                            logger.info(f"Renaming text column {text_column_name} to {text_col_name_ref}")
                             dataset = dataset.rename_column(text_column_name, text_col_name_ref)
                             flag = True
                             break
@@ -408,9 +412,15 @@ def load_maybe_streaming_dataset(
                                          f"Text columns: {text_column_names}"
                                          f"Dataset columns: {raw_datasets_features}")
                 if audio_column_name is not None and sampling_rate is not None:
-                    dataset = dataset.cast_column(
-                        audio_column_name, datasets.features.Audio(sampling_rate=sampling_rate)
-                    )
+                    ds_sr = int(dataset.features[audio_column_name].sampling_rate)
+                    if ds_sr != sampling_rate:
+                        dataset = dataset.cast_column(
+                            audio_column_name, datasets.features.Audio(sampling_rate=sampling_rate)
+                        )
+                raw_datasets_features = list(dataset.features.keys())
+                raw_datasets_features.remove(audio_column_name)
+                raw_datasets_features.remove(text_col_name_ref)
+                dataset = dataset.remove_columns(column_names=raw_datasets_features)
                 dataset_splits.append(dataset)
 
         # interleave multiple splits to form one dataset
@@ -422,6 +432,36 @@ def load_maybe_streaming_dataset(
         return dataset
 
 
+def load_common_voice_like_dataset(
+    dataset_name,
+    config,
+    split,
+    audio_column_name=None,
+    sampling_rate=None,
+    streaming=True,
+    use_auth_token=False
+):
+
+    """
+    Utility function to load the Common Voice dataset.
+    """
+    dataset = load_dataset(
+        dataset_name,
+        config,
+        split=split,
+        streaming=streaming,
+        use_auth_token=use_auth_token,
+    )
+    if audio_column_name is not None and sampling_rate is not None:
+        dataset = dataset.cast_column(
+            audio_column_name, datasets.features.Audio(sampling_rate=sampling_rate)
+        )
+    return dataset
+
+
+# def load_nst_nbailab(config, split, )
+
+
 def main():
     # 1. Parse input arguments
     # See all possible arguments in src/transformers/training_args.py
@@ -510,7 +550,7 @@ def main():
             streaming=data_args.streaming,
             text_column_name=data_args.text_column_name,
             audio_column_name=data_args.audio_column_name,
-            sampling_rate=feature_extractor.sampling_rate,
+            sampling_rate=int(feature_extractor.sampling_rate),
             # language=data_args.language_train
         )
 
@@ -523,7 +563,7 @@ def main():
             streaming=data_args.streaming,
             text_column_name=data_args.text_column_name,
             audio_column_name=data_args.audio_column_name,
-            sampling_rate=feature_extractor.sampling_rate,
+            sampling_rate=int(feature_extractor.sampling_rate),
             # language=data_args.language_eval
         )
 

test_run_nordic.sh

@@ -1,9 +1,9 @@
 python $1run_speech_recognition_seq2seq_streaming.py \
 	--model_name_or_path="openai/whisper-tiny" \
-	--dataset_train_name="mozilla-foundation/common_voice_11_0,mozilla-foundation/common_voice_11_0,mozilla-foundation/common_voice_11_0,babelbox/babelbox_voice,NbAiLab/NST,arpelarpe/nota,NbAiLab/NPSC,google/fleurs,google/fleurs,google/fleurs" \
-	--dataset_train_config_name="sv-SE,da,nn-NO,nst,no-distant,,16K_mp3_nynorsk,sv_se,da_dk,nb_no" \
-	--language_train="swedish,danish,norwegian,swedish,norwegian,danish,norwegian,swedish,danish,norwegian" \
-	--train_split_name="train+validation,train+validation,train+validation,train,train+test,train,train+validation,train+validation,train+validation,train+validation" \
+	--dataset_train_name="mozilla-foundation/common_voice_11_0,mozilla-foundation/common_voice_11_0,mozilla-foundation/common_voice_11_0,babelbox/babelbox_voice,NbAiLab/NST,NbAiLab/NPSC,google/fleurs,google/fleurs,google/fleurs" \
+	--dataset_train_config_name="sv-SE,da,nn-NO,nst,no-distant,16K_mp3_nynorsk,sv_se,da_dk,nb_no" \
+	--language_train="swedish,danish,norwegian,swedish,norwegian,norwegian,swedish,danish,norwegian" \
+	--train_split_name="train+validation,train+validation,train+validation,train,train+test,train+validation,train+validation,train+validation,train+validation" \
 	--dataset_eval_name="mozilla-foundation/common_voice_11_0,mozilla-foundation/common_voice_11_0,mozilla-foundation/common_voice_11_0" \
 	--dataset_eval_config_name="sv-SE,da,nn-NO" \
 	--language_eval="swedish,danish,norwegian" \