first commit

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+*.arpa filter=lfs diff=lfs merge=lfs -text

EnglishCV/common_voice_prepare.py

@@ -0,0 +1,410 @@
+"""
+Data preparation.
+Download: https://voice.mozilla.org/en/datasets
+Author
+------
+Titouan Parcollet
+Luca Della Libera 2022
+Pooneh Mousavi 2022
+"""
+
+from dataclasses import dataclass
+import os
+import csv
+import re
+import logging
+import torchaudio
+from tqdm import tqdm
+import unicodedata
+import functools
+torchaudio.set_audio_backend("soundfile")
+from speechbrain.utils.parallel import parallel_map
+from speechbrain.dataio.dataio import read_audio_info
+
+logger = logging.getLogger(__name__)
+
+
+def prepare_common_voice(
+    data_folder,
+    save_folder,
+    train_tsv_file=None,
+    dev_tsv_file=None,
+    test_tsv_file=None,
+    accented_letters=False,
+    language="en",
+    skip_prep=False,
+):
+    """
+    Prepares the csv files for the Mozilla Common Voice dataset.
+    Download: https://voice.mozilla.org/en/datasets
+    Arguments
+    ---------
+    data_folder : str
+        Path to the folder where the original Common Voice dataset is stored.
+        This path should include the lang: /datasets/CommonVoice/<language>/
+    save_folder : str
+        The directory where to store the csv files.
+    train_tsv_file : str, optional
+        Path to the Train Common Voice .tsv file (cs)
+    dev_tsv_file : str, optional
+        Path to the Dev Common Voice .tsv file (cs)
+    test_tsv_file : str, optional
+        Path to the Test Common Voice .tsv file (cs)
+    accented_letters : bool, optional
+        Defines if accented letters will be kept as individual letters or
+        transformed to the closest non-accented letters.
+    language: str
+        Specify the language for text normalization.
+    skip_prep: bool
+        If True, skip data preparation.
+    Example
+    -------
+    >>> from recipes.CommonVoice.common_voice_prepare import prepare_common_voice
+    >>> data_folder = '/datasets/CommonVoice/en'
+    >>> save_folder = 'exp/CommonVoice_exp'
+    >>> train_tsv_file = '/datasets/CommonVoice/en/train.tsv'
+    >>> dev_tsv_file = '/datasets/CommonVoice/en/dev.tsv'
+    >>> test_tsv_file = '/datasets/CommonVoice/en/test.tsv'
+    >>> accented_letters = False
+    >>> duration_threshold = 10
+    >>> prepare_common_voice( \
+                 data_folder, \
+                 save_folder, \
+                 train_tsv_file, \
+                 dev_tsv_file, \
+                 test_tsv_file, \
+                 accented_letters, \
+                 language="en" \
+                 )
+    """
+
+    if skip_prep:
+        return
+
+    # If not specified point toward standard location w.r.t CommonVoice tree
+    if train_tsv_file is None:
+        train_tsv_file = data_folder + "/train.tsv"
+    else:
+        train_tsv_file = train_tsv_file
+
+    if dev_tsv_file is None:
+        dev_tsv_file = data_folder + "/dev.tsv"
+    else:
+        dev_tsv_file = dev_tsv_file
+
+    if test_tsv_file is None:
+        test_tsv_file = data_folder + "/test.tsv"
+    else:
+        test_tsv_file = test_tsv_file
+
+    # Setting the save folder
+    if not os.path.exists(save_folder):
+        os.makedirs(save_folder)
+
+    # Setting ouput files
+    save_csv_train = save_folder + "/train.csv"
+    save_csv_dev = save_folder + "/dev.csv"
+    save_csv_test = save_folder + "/test.csv"
+
+    # If csv already exists, we skip the data preparation
+    if skip(save_csv_train, save_csv_dev, save_csv_test):
+
+        msg = "%s already exists, skipping data preparation!" % (save_csv_train)
+        logger.info(msg)
+
+        msg = "%s already exists, skipping data preparation!" % (save_csv_dev)
+        logger.info(msg)
+
+        msg = "%s already exists, skipping data preparation!" % (save_csv_test)
+        logger.info(msg)
+
+        return
+
+    # Additional checks to make sure the data folder contains Common Voice
+    check_commonvoice_folders(data_folder)
+    # Creating csv files for {train, dev, test} data
+    file_pairs = zip(
+        [train_tsv_file, dev_tsv_file, test_tsv_file],
+        [save_csv_train, save_csv_dev, save_csv_test],
+    )
+    for tsv_file, save_csv in file_pairs:
+        create_csv(
+            tsv_file, save_csv, data_folder, accented_letters, language,
+        )
+
+
+def skip(save_csv_train, save_csv_dev, save_csv_test):
+    """
+    Detects if the Common Voice data preparation has been already done.
+    If the preparation has been done, we can skip it.
+    Returns
+    -------
+    bool
+        if True, the preparation phase can be skipped.
+        if False, it must be done.
+    """
+
+    # Checking folders and save options
+    skip = False
+
+    if (
+        os.path.isfile(save_csv_train)
+        and os.path.isfile(save_csv_dev)
+        and os.path.isfile(save_csv_test)
+    ):
+        skip = True
+
+    return skip
+
+
+@dataclass
+class CVRow:
+    snt_id: str
+    duration: float
+    mp3_path: str
+    spk_id: str
+    words: str
+
+
+def process_line(line, data_folder, language, accented_letters):
+    # Path is at indice 1 in Common Voice tsv files. And .mp3 files
+    # are located in datasets/lang/clips/
+    mp3_path = data_folder + "/clips/" + line.split("\t")[1]
+    file_name = mp3_path.split(".")[-2].split("/")[-1]
+    spk_id = line.split("\t")[0]
+    snt_id = file_name
+
+    # Setting torchaudio backend to sox-io (needed to read mp3 files)
+    """
+    if torchaudio.get_audio_backend() != "sox_io":
+        logger.warning("This recipe needs the sox-io backend of torchaudio")
+        logger.warning("The torchaudio backend is changed to sox_io")
+        torchaudio.set_audio_backend("sox_io")
+    """
+    # Reading the signal (to retrieve duration in seconds)
+    if os.path.isfile(mp3_path):
+        info = read_audio_info(mp3_path)
+    else:
+        msg = "\tError loading: %s" % (str(len(file_name)))
+        logger.info(msg)
+        return None
+
+    duration = info.num_frames / info.sample_rate
+
+    # Getting transcript
+    words = line.split("\t")[2]
+
+    # Unicode Normalization
+    words = unicode_normalisation(words)
+
+    # !! Language specific cleaning !!
+    words = language_specific_preprocess(language, words)
+
+    # Remove accents if specified
+    if not accented_letters:
+        words = strip_accents(words)
+        words = words.replace("'", " ")
+        words = words.replace("’", " ")
+
+    # Remove multiple spaces
+    words = re.sub(" +", " ", words)
+
+    # Remove spaces at the beginning and the end of the sentence
+    words = words.lstrip().rstrip()
+
+    # Getting chars
+    chars = words.replace(" ", "_")
+    chars = " ".join([char for char in chars][:])
+
+    # Remove too short sentences (or empty):
+    if language in ["ja", "ch"]:
+        if len(chars) < 3:
+            return None
+    else:
+        if len(words.split(" ")) < 3:
+            return None
+
+    # Composition of the csv_line
+    return CVRow(snt_id, duration, mp3_path, spk_id, words)
+
+
+def create_csv(
+    orig_tsv_file, csv_file, data_folder, accented_letters=False, language="en"
+):
+    """
+    Creates the csv file given a list of wav files.
+    Arguments
+    ---------
+    orig_tsv_file : str
+        Path to the Common Voice tsv file (standard file).
+    data_folder : str
+        Path of the CommonVoice dataset.
+    accented_letters : bool, optional
+        Defines if accented letters will be kept as individual letters or
+        transformed to the closest non-accented letters.
+    Returns
+    -------
+    None
+    """
+
+    # Check if the given files exists
+    if not os.path.isfile(orig_tsv_file):
+        msg = "\t%s doesn't exist, verify your dataset!" % (orig_tsv_file)
+        logger.info(msg)
+        raise FileNotFoundError(msg)
+
+    # We load and skip the header
+    loaded_csv = open(orig_tsv_file, "r").readlines()[1:]
+    nb_samples = len(loaded_csv)
+
+    msg = "Preparing CSV files for %s samples ..." % (str(nb_samples))
+    logger.info(msg)
+
+    # Adding some Prints
+    msg = "Creating csv lists in %s ..." % (csv_file)
+    logger.info(msg)
+
+    # Process and write lines
+    total_duration = 0.0
+
+    line_processor = functools.partial(
+        process_line,
+        data_folder=data_folder,
+        language=language,
+        accented_letters=accented_letters,
+    )
+
+    # Stream into a .tmp file, and rename it to the real path at the end.
+    csv_file_tmp = csv_file + ".tmp"
+
+    with open(csv_file_tmp, mode="w", encoding="utf-8") as csv_f:
+        csv_writer = csv.writer(
+            csv_f, delimiter=",", quotechar='"', quoting=csv.QUOTE_MINIMAL
+        )
+
+        csv_writer.writerow(["ID", "duration", "wav", "spk_id", "wrd"])
+        for line in tqdm(loaded_csv) : 
+
+            row = line_processor(line)
+            if row is not None  : 
+                total_duration += row.duration
+                csv_writer.writerow(
+                    [
+                        row.snt_id,
+                        str(row.duration),
+                        row.mp3_path,
+                        row.spk_id,
+                        row.words,
+                    ]
+                )
+
+    os.replace(csv_file_tmp, csv_file)
+
+    # Final prints
+    msg = "%s successfully created!" % (csv_file)
+    logger.info(msg)
+    msg = "Number of samples: %s " % (str(len(loaded_csv)))
+    logger.info(msg)
+    msg = "Total duration: %s Hours" % (str(round(total_duration / 3600, 2)))
+    logger.info(msg)
+
+
+def language_specific_preprocess(language, words):
+    # !! Language specific cleaning !!
+    # Important: feel free to specify the text normalization
+    # corresponding to your alphabet.
+
+    if language in ["en", "fr", "it", "rw"]:
+        words = re.sub(
+            "[^’'A-Za-z0-9À-ÖØ-öø-ÿЀ-ӿéæœâçèàûî]+", " ", words
+        ).upper()
+
+    if language == "de":
+        # this replacement helps preserve the case of ß
+        # (and helps retain solitary occurrences of SS)
+        # since python's upper() converts ß to SS.
+        words = words.replace("ß", "0000ß0000")
+        words = re.sub("[^’'A-Za-z0-9öÖäÄüÜß]+", " ", words).upper()
+        words = words.replace("'", " ")
+        words = words.replace("’", " ")
+        words = words.replace(
+            "0000SS0000", "ß"
+        )  # replace 0000SS0000 back to ß as its initial presence in the corpus
+
+    if language == "fr":
+        # Replace J'y D'hui etc by J_ D_hui
+        words = words.replace("'", " ")
+        words = words.replace("’", " ")
+
+    elif language == "ar":
+        HAMZA = "\u0621"
+        ALEF_MADDA = "\u0622"
+        ALEF_HAMZA_ABOVE = "\u0623"
+        letters = (
+            "ابتةثجحخدذرزژشسصضطظعغفقكلمنهويىءآأؤإئ"
+            + HAMZA
+            + ALEF_MADDA
+            + ALEF_HAMZA_ABOVE
+        )
+        words = re.sub("[^" + letters + " ]+", "", words).upper()
+    elif language == "fa":
+        HAMZA = "\u0621"
+        ALEF_MADDA = "\u0622"
+        ALEF_HAMZA_ABOVE = "\u0623"
+        letters = (
+            "ابپتةثجحخچدذرزژسشصضطظعغفقگکلمنهویىءآأؤإئ"
+            + HAMZA
+            + ALEF_MADDA
+            + ALEF_HAMZA_ABOVE
+        )
+        words = re.sub("[^" + letters + " ]+", "", words).upper()
+    elif language == "ga-IE":
+        # Irish lower() is complicated, but upper() is nondeterministic, so use lowercase
+        def pfxuc(a):
+            return len(a) >= 2 and a[0] in "tn" and a[1] in "AEIOUÁÉÍÓÚ"
+
+        def galc(w):
+            return w.lower() if not pfxuc(w) else w[0] + "-" + w[1:].lower()
+
+        words = re.sub("[^-A-Za-z'ÁÉÍÓÚáéíóú]+", " ", words)
+        words = " ".join(map(galc, words.split(" ")))
+    elif language == "es":
+        # Fix the following error in dataset large:
+        # KeyError: 'The item En noviembre lanzaron Queen Elizabeth , coproducida por Foreign Noi$e . requires replacements which were not supplied.'
+        words = words.replace("$", "s")
+    return words
+
+
+def check_commonvoice_folders(data_folder):
+    """
+    Check if the data folder actually contains the Common Voice dataset.
+    If not, raises an error.
+    Returns
+    -------
+    None
+    Raises
+    ------
+    FileNotFoundError
+        If data folder doesn't contain Common Voice dataset.
+    """
+    files_str = "/clips"
+    # Checking clips
+    if not os.path.exists(data_folder + files_str):
+        err_msg = (
+            "the folder %s does not exist (it is expected in "
+            "the Common Voice dataset)" % (data_folder + files_str)
+        )
+        raise FileNotFoundError(err_msg)
+
+
+def unicode_normalisation(text):
+    return str(text)
+
+
+def strip_accents(text):
+    text = (
+        unicodedata.normalize("NFD", text)
+        .encode("ascii", "ignore")
+        .decode("utf-8")
+    )
+    return str(text)

EnglishCV/results/final_cs/hyperparams.yaml

@@ -0,0 +1,144 @@
+# Generated 2023-09-08 from:
+# /gpfsssd/scratch/rech/nou/uzn19yk/switched_data/stac.yaml
+# yamllint disable
+# Generated 2023-08-03 from:
+# /home/salah/new_tunisian_model/hparams/train_tunisian_withwavlm.yaml
+# yamllint disable
+# ################################
+# Model: wav2vec2 + DNN + CTC
+# Augmentation: SpecAugment
+# Authors: Titouan Parcollet 2021
+# ################################
+
+seed: 1994
+__set_seed: !!python/object/apply:torch.manual_seed [1234]
+output_folder: results/non_semi_final_stac
+wer_file: results/non_semi_final_stac/wer.txt
+save_folder: results/non_semi_final_stac/save
+train_log: results/non_semi_final_stac/train_log.txt
+
+
+
+# Data files
+data_folder: junk  # e.g, /localscratch/cv-corpus-5.1-2020-06-22/fr
+train_tsv_file: junk/train.tsv                # Standard CommonVoice .tsv files
+dev_tsv_file: junk/dev.tsv                # Standard CommonVoice .tsv files
+test_tsv_file: junk/test.tsv                # Standard CommonVoice .tsv files
+accented_letters: true
+
+csv_folder: /gpfsscratch/rech/nou/uzn19yk/switched_data/extended_clean/
+train_csv: /gpfsscratch/rech/nou/uzn19yk/switched_data/extended_clean//train.csv
+valid_csv: /gpfsscratch/rech/nou/uzn19yk/switched_data/extended_clean//dev.csv
+test_csv:
+- all_tests/cs_test.csv
+- all_tests/stac_test.csv
+
+# We remove utterance slonger than 10s in the train/dev/test sets as
+# longer sentences certainly correspond to "open microphones".
+avoid_if_longer_than: 13.0
+avoid_if_shorter_than: 0.5
+
+# Training parameters
+number_of_epochs: 20
+lr: 0.0002
+lr_weights: 0.01
+sorting: ascending
+auto_mix_prec: false
+sample_rate: 16000
+language_modelling: true
+ngram_lm_path: 
+  /gpfsstore/rech/nou/uzn19yk/switched_code_tunisian/train/tunisian_asr/arpas/pluslanguages_everything.arpa
+
+# With data_parallel batch_size is split into N jobs
+# With DDP batch_size is multiplied by N jobs
+# Must be 3 per GPU to fit 32GB of VRAM
+batch_size: 3
+test_batch_size: 4
+
+# Dataloader options
+dataloader_options:
+  batch_size: 3
+  num_workers: 6
+
+test_dataloader_options:
+  batch_size: 4
+  num_workers: 6
+
+# Model parameters
+activation: !name:torch.nn.Sigmoid
+dnn_layers: 1
+dnn_neurons: 768
+freeze_encoder: true
+
+# Outputs
+output_neurons: 76  # BPE size, index(blank/eos/bos) = 0
+
+# Functions and classes
+#
+epoch_counter: &id006 !new:speechbrain.utils.epoch_loop.EpochCounter
+  limit: 20
+
+encoder_dim: 3217
+enc: &id001 !new:speechbrain.nnet.RNN.LSTM
+  input_shape: [null, null, 3217]
+  num_layers: 2
+  bidirectional: true
+  dropout: 0.2
+  hidden_size: 1024
+
+ctc_lin: &id002 !new:speechbrain.nnet.linear.Linear
+
+  input_size: 2048
+  n_neurons: 76
+
+log_softmax: !new:speechbrain.nnet.activations.Softmax
+  apply_log: true
+
+ctc_cost: !name:speechbrain.nnet.losses.ctc_loss
+  blank_index: 0
+
+modules:
+  enc: *id001
+  ctc_lin: *id002
+model: &id003 !new:torch.nn.ModuleList
+- [*id001, *id002]
+model_opt_class: !name:torch.optim.Adam
+  lr: 0.0002
+
+weights_opt_class: !name:torch.optim.Adam
+  lr: 0.01
+
+lr_annealing_model: &id004 !new:speechbrain.nnet.schedulers.NewBobScheduler
+  initial_value: 0.0002
+  improvement_threshold: 0.0025
+  annealing_factor: 0.8
+  patient: 0
+
+lr_annealing_weights: &id005 !new:speechbrain.nnet.schedulers.NewBobScheduler
+  initial_value: 0.01
+  improvement_threshold: 0.0025
+  annealing_factor: 0.9
+  patient: 0
+
+label_encoder: &id007 !new:speechbrain.dataio.encoder.CTCTextEncoder
+
+
+checkpointer: !new:speechbrain.utils.checkpoints.Checkpointer
+  checkpoints_dir: results/non_semi_final_stac/save
+  recoverables:
+    model: *id003
+    scheduler_model: *id004
+    scheduler_encoder: *id005
+    counter: *id006
+    tokenizer: *id007
+blank_index: 0
+unk_index: 1
+
+
+train_logger: !new:speechbrain.utils.train_logger.FileTrainLogger
+  save_file: results/non_semi_final_stac/train_log.txt
+
+error_rate_computer: !name:speechbrain.utils.metric_stats.ErrorRateStats
+
+cer_computer: !name:speechbrain.utils.metric_stats.ErrorRateStats
+  split_tokens: true

EnglishCV/results/final_cs/save/CKPT+2023-09-08+01-40-18+00/CKPT.yaml

@@ -0,0 +1,4 @@
+# yamllint disable
+WER: 51.292116454039906
+end-of-epoch: true
+unixtime: 1694130018.9642384

EnglishCV/results/final_cs/save/CKPT+2023-09-08+01-40-18+00/brain.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e5c026fe6fa51700406bd476e131950c797b0b3bacb3daae0854e85689bb4cf9
+size 50

EnglishCV/results/final_cs/save/CKPT+2023-09-08+01-40-18+00/counter.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f5ca38f748a1d6eaf726b8a42fb575c3c71f1864a8143301782de13da2d9202b
+size 2

EnglishCV/results/final_cs/save/CKPT+2023-09-08+01-40-18+00/dataloader-TRAIN.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d7e1edcac43af8cea1439d222314af06354ae31da6a3d90b8cc6bcebc5c8e397
+size 4

EnglishCV/results/final_cs/save/CKPT+2023-09-08+01-40-18+00/model.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:da683a8efa5709a06af9b258452c243da841780a0a7942c196c472a3e21e5010
+size 240389017

EnglishCV/results/final_cs/save/CKPT+2023-09-08+01-40-18+00/modelopt.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:416feb314443cf839f4425fc382e555dec90e3dea26fa52b75e4ac1b702c5078
+size 480787579

EnglishCV/results/final_cs/save/CKPT+2023-09-08+01-40-18+00/scheduler_encoder.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2e2efd50f0cf28a080e2625fdd8a1852c669841537cdc0a57fce60bc6c1eec11
+size 515

EnglishCV/results/final_cs/save/CKPT+2023-09-08+01-40-18+00/scheduler_model.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cec54cc9236fa7aa965b397675d24299b973675cc0c6345de038fc70e51629ab
+size 703

EnglishCV/results/final_cs/save/CKPT+2023-09-08+01-40-18+00/tokenizer.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:21080a140faeb4f39fad188aaf081914ec782be9c4320d6415e8822709e18017
+size 39

EnglishCV/results/final_cs/save/label_encoder.txt

@@ -0,0 +1,80 @@
+'و' => 74
+'ي' => 1
+'ن' => 2
+' ' => 3
+'م' => 4
+'ش' => 5
+'ل' => 6
+'س' => 7
+'ت' => 8
+'ا' => 9
+'د' => 10
+'ر' => 11
+'ى' => 12
+'ب' => 13
+'ح' => 14
+'ط' => 15
+'ع' => 16
+'ك' => 17
+'ف' => 18
+'ق' => 19
+'ذ' => 20
+'ث' => 21
+'ج' => 22
+'ة' => 23
+'غ' => 24
+'o' => 25
+'k' => 26
+'b' => 27
+'n' => 28
+'خ' => 29
+'ه' => 30
+'v' => 31
+'i' => 32
+'l' => 33
+'à' => 34
+'ص' => 35
+'ض' => 36
+'a' => 37
+'u' => 38
+'t' => 39
+'m' => 40
+'q' => 41
+'e' => 42
+'d' => 43
+'c' => 44
+'p' => 45
+'r' => 46
+'أ' => 47
+'إ' => 48
+'s' => 49
+'j' => 50
+'ز' => 51
+'ء' => 52
+'h' => 53
+'f' => 54
+'آ' => 55
+'ئ' => 56
+'ؤ' => 57
+'ظ' => 58
+'y' => 59
+'é' => 60
+"'" => 61
+'z' => 62
+'x' => 63
+'w' => 64
+'g' => 65
+'è' => 66
+'û' => 67
+'ç' => 68
+'ê' => 69
+'ô' => 70
+'ù' => 71
+'î' => 72
+'â' => 73
+'<blank>' => 0
+1 => 75
+================
+'starting_index' => 0
+'unk_label' => 1
+'blank_label' => '<blank>'

EnglishCV/results/final_cs/train_mixer.py

@@ -0,0 +1,756 @@
+# -*- coding: utf-8 -*-
+
+import os
+import sys
+import torch
+import logging
+import speechbrain as sb
+from speechbrain.utils.distributed import run_on_main
+from hyperpyyaml import load_hyperpyyaml
+from pathlib import Path
+import torchaudio.transforms as T
+from cv_train import ASRCV
+import torchaudio
+import numpy as np
+import kenlm
+from pyctcdecode import build_ctcdecoder
+import re
+
+# Commented out IPython magic to ensure Python compatibility.
+# %cd /content/drive/MyDrive/tunisian_corpora/tunisian_without_wavlm
+#hparams_file, run_opts, overrides = sb.parse_arguments(["/gpfsstore/rech/nou/uzn19yk/switched_code_tunisian/train/tunisian_asr/hparams/train_semi.yaml"])
+hparams_file, run_opts, overrides = sb.parse_arguments(["semi_supervised_test_tunisian.yaml"])
+
+# If distributed_launch=True then
+# create ddp_group with the right communication protocol
+sb.utils.distributed.ddp_init_group(run_opts)
+
+with open(hparams_file) as fin:
+    hparams = load_hyperpyyaml(fin, overrides)
+
+# Create experiment directory
+sb.create_experiment_directory(
+    experiment_directory=hparams["output_folder"],
+    hyperparams_to_save=hparams_file,
+    overrides=overrides,
+)
+# Dataset prep (parsing Librispeech)
+
+def dataio_prepare(hparams):
+    """This function prepares the datasets to be used in the brain class.
+    It also defines the data processing pipeline through user-defined functions."""
+
+    # 1. Define datasets
+    data_folder = hparams["data_folder"]
+
+    train_data = sb.dataio.dataset.DynamicItemDataset.from_csv(
+        csv_path=hparams["train_csv"], replacements={"data_root": data_folder},
+    )
+
+    if hparams["sorting"] == "ascending":
+        # we sort training data to speed up training and get better results.
+        train_data = train_data.filtered_sorted(
+            sort_key="duration",
+            key_max_value={"duration": hparams["avoid_if_longer_than"]},
+        )
+        # when sorting do not shuffle in dataloader ! otherwise is pointless
+        hparams["dataloader_options"]["shuffle"] = False
+
+    elif hparams["sorting"] == "descending":
+        train_data = train_data.filtered_sorted(
+            sort_key="duration",
+            reverse=True,
+            key_max_value={"duration": hparams["avoid_if_longer_than"]},
+        )
+        # when sorting do not shuffle in dataloader ! otherwise is pointless
+        hparams["dataloader_options"]["shuffle"] = False
+
+    elif hparams["sorting"] == "random":
+        pass
+
+    else:
+        raise NotImplementedError(
+            "sorting must be random, ascending or descending"
+        )
+
+    valid_data = sb.dataio.dataset.DynamicItemDataset.from_csv(
+        csv_path=hparams["valid_csv"], replacements={"data_root": data_folder},
+    )
+    # We also sort the validation data so it is faster to validate
+    valid_data = valid_data.filtered_sorted(sort_key="duration")
+    test_datasets = {}
+    for csv_file in hparams["test_csv"]:
+        name = Path(csv_file).stem
+        test_datasets[name] = sb.dataio.dataset.DynamicItemDataset.from_csv(
+            csv_path=csv_file, replacements={"data_root": data_folder}
+        )
+        test_datasets[name] = test_datasets[name].filtered_sorted(
+            sort_key="duration"
+        )
+
+    datasets = [train_data, valid_data] + [i for k, i in test_datasets.items()]
+
+
+    # 2. Define audio pipeline:
+    @sb.utils.data_pipeline.takes("wav")
+    @sb.utils.data_pipeline.provides("sig")
+    def audio_pipeline(wav):
+        info = torchaudio.info(wav)
+        sig = sb.dataio.dataio.read_audio(wav)
+        if len(sig.shape)>1 :
+            sig = torch.mean(sig, dim=1)
+        resampled = torchaudio.transforms.Resample(
+            info.sample_rate, hparams["sample_rate"],
+        )(sig)
+        return resampled
+
+    sb.dataio.dataset.add_dynamic_item(datasets, audio_pipeline)
+    label_encoder = sb.dataio.encoder.CTCTextEncoder()
+
+    # 3. Define text pipeline:
+    @sb.utils.data_pipeline.takes("wrd")
+    @sb.utils.data_pipeline.provides(
+        "wrd", "char_list", "tokens_list", "tokens"
+    )
+    def text_pipeline(wrd):
+        yield wrd
+        char_list = list(wrd)
+        yield char_list
+        tokens_list = label_encoder.encode_sequence(char_list)
+        yield tokens_list
+        tokens = torch.LongTensor(tokens_list)
+        yield tokens
+
+    sb.dataio.dataset.add_dynamic_item(datasets, text_pipeline)
+    lab_enc_file = os.path.join(hparams["save_folder"], "label_encoder.txt")
+    special_labels = {
+        "blank_label": hparams["blank_index"],
+        "unk_label": hparams["unk_index"]
+    }
+    label_encoder.load_or_create(
+        path=lab_enc_file,
+        from_didatasets=[train_data],
+        output_key="char_list",
+        special_labels=special_labels,
+        sequence_input=True,
+    )
+
+    # 4. Set output:
+    sb.dataio.dataset.set_output_keys(
+        datasets, ["id", "sig", "wrd", "char_list", "tokens"],
+    )
+    return train_data, valid_data,test_datasets, label_encoder
+
+class ASR(sb.core.Brain):
+    def compute_forward(self, batch, stage):
+        """Forward computations from the waveform batches to the output probabilities."""
+
+        batch = batch.to(self.device)
+        wavs, wav_lens = batch.sig
+        wavs, wav_lens = wavs.to(self.device), wav_lens.to(self.device)
+
+        if stage == sb.Stage.TRAIN:
+            if hasattr(self.hparams, "augmentation"):
+                wavs = self.hparams.augmentation(wavs, wav_lens)
+
+        # Forward pass
+        feats = self.modules.wav2vec2(wavs, wav_lens)
+        x = self.modules.enc(feats)
+        logits = self.modules.ctc_lin(x)
+        p_ctc = self.hparams.log_softmax(logits)
+
+        return p_ctc, wav_lens
+
+    def custom_encode(self,wavs,wav_lens) :
+        wavs = wavs.to(self.device)
+        if(wav_lens is not None): wav_lens.to(self.device)
+
+        feats = self.modules.wav2vec2(wavs, wav_lens)
+        x = self.modules.enc(feats)
+        logits = self.modules.ctc_lin(x)
+        p_ctc = self.hparams.log_softmax(logits)
+
+        return feats,p_ctc
+
+
+
+    def compute_objectives(self, predictions, batch, stage):
+        """Computes the loss (CTC) given predictions and targets."""
+
+        p_ctc, wav_lens = predictions
+
+        ids = batch.id
+        tokens, tokens_lens = batch.tokens
+
+        loss = self.hparams.ctc_cost(p_ctc, tokens, wav_lens, tokens_lens)
+
+        if stage != sb.Stage.TRAIN:
+            predicted_tokens = sb.decoders.ctc_greedy_decode(
+                p_ctc, wav_lens, blank_id=self.hparams.blank_index
+            )
+            # Decode token terms to words
+            if self.hparams.use_language_modelling:
+                predicted_words = []
+                for logs in p_ctc:
+                    text = decoder.decode(logs.detach().cpu().numpy())
+                    predicted_words.append(text.split(" "))
+            else:
+                predicted_words = [
+                    "".join(self.tokenizer.decode_ndim(utt_seq)).split(" ")
+                    for utt_seq in predicted_tokens
+                ]
+            # Convert indices to words
+            target_words = [wrd.split(" ") for wrd in batch.wrd]
+
+            self.wer_metric.append(ids, predicted_words, target_words)
+            self.cer_metric.append(ids, predicted_words, target_words)
+
+        return loss
+
+    def fit_batch(self, batch):
+        """Train the parameters given a single batch in input"""
+        should_step = self.step % self.grad_accumulation_factor == 0
+        # Managing automatic mixed precision
+        # TOFIX: CTC fine-tuning currently is unstable
+        # This is certainly due to CTC being done in fp16 instead of fp32
+        if self.auto_mix_prec:
+            with torch.cuda.amp.autocast():
+                with self.no_sync():
+                    outputs = self.compute_forward(batch, sb.Stage.TRAIN)
+                loss = self.compute_objectives(outputs, batch, sb.Stage.TRAIN)
+            with self.no_sync(not should_step):
+                self.scaler.scale(
+                    loss / self.grad_accumulation_factor
+                ).backward()
+            if should_step:
+
+                if not self.hparams.wav2vec2.freeze:
+                    self.scaler.unscale_(self.wav2vec_optimizer)
+                self.scaler.unscale_(self.model_optimizer)
+                if self.check_gradients(loss):
+                    if not self.hparams.wav2vec2.freeze:
+                        if self.optimizer_step >= self.hparams.warmup_steps:
+                            self.scaler.step(self.wav2vec_optimizer)
+                    self.scaler.step(self.model_optimizer)
+                self.scaler.update()
+                self.zero_grad()
+                self.optimizer_step += 1
+        else:
+            # This is mandatory because HF models have a weird behavior with DDP
+            # on the forward pass
+            with self.no_sync():
+                outputs = self.compute_forward(batch, sb.Stage.TRAIN)
+
+            loss = self.compute_objectives(outputs, batch, sb.Stage.TRAIN)
+
+            with self.no_sync(not should_step):
+                (loss / self.grad_accumulation_factor).backward()
+            if should_step:
+                if self.check_gradients(loss):
+                    if not self.hparams.wav2vec2.freeze:
+                        if self.optimizer_step >= self.hparams.warmup_steps:
+                            self.wav2vec_optimizer.step()
+                    self.model_optimizer.step()
+                self.zero_grad()
+                self.optimizer_step += 1
+
+        self.on_fit_batch_end(batch, outputs, loss, should_step)
+        return loss.detach().cpu()
+
+    def evaluate_batch(self, batch, stage):
+        """Computations needed for validation/test batches"""
+        predictions = self.compute_forward(batch, stage=stage)
+        with torch.no_grad():
+            loss = self.compute_objectives(predictions, batch, stage=stage)
+        return loss.detach()
+
+    def on_stage_start(self, stage, epoch):
+        """Gets called at the beginning of each epoch"""
+        if stage != sb.Stage.TRAIN:
+            self.cer_metric = self.hparams.cer_computer()
+            self.wer_metric = self.hparams.error_rate_computer()
+
+    def on_stage_end(self, stage, stage_loss, epoch):
+        """Gets called at the end of an epoch."""
+        # Compute/store important stats
+        stage_stats = {"loss": stage_loss}
+        if stage == sb.Stage.TRAIN:
+            self.train_stats = stage_stats
+        else:
+            stage_stats["CER"] = self.cer_metric.summarize("error_rate")
+            stage_stats["WER"] = self.wer_metric.summarize("error_rate")
+
+        # Perform end-of-iteration things, like annealing, logging, etc.
+        if stage == sb.Stage.VALID:
+            old_lr_model, new_lr_model = self.hparams.lr_annealing_model(
+                stage_stats["loss"]
+            )
+            old_lr_wav2vec, new_lr_wav2vec = self.hparams.lr_annealing_wav2vec(
+                stage_stats["loss"]
+            )
+            sb.nnet.schedulers.update_learning_rate(
+                self.model_optimizer, new_lr_model
+            )
+            if not self.hparams.wav2vec2.freeze:
+                sb.nnet.schedulers.update_learning_rate(
+                    self.wav2vec_optimizer, new_lr_wav2vec
+                )
+            self.hparams.train_logger.log_stats(
+                stats_meta={
+                    "epoch": epoch,
+                    "lr_model": old_lr_model,
+                    "lr_wav2vec": old_lr_wav2vec,
+                },
+                train_stats=self.train_stats,
+                valid_stats=stage_stats,
+            )
+            self.checkpointer.save_and_keep_only(
+                meta={"WER": stage_stats["WER"]}, min_keys=["WER"],
+            )
+        elif stage == sb.Stage.TEST:
+            self.hparams.train_logger.log_stats(
+                stats_meta={"Epoch loaded": self.hparams.epoch_counter.current},
+                test_stats=stage_stats,
+            )
+            with open(self.hparams.wer_file, "w") as w:
+                self.wer_metric.write_stats(w)
+
+    def init_optimizers(self):
+        "Initializes the wav2vec2 optimizer and model optimizer"
+
+        # If the wav2vec encoder is unfrozen, we create the optimizer
+        if not self.hparams.wav2vec2.freeze:
+            self.wav2vec_optimizer = self.hparams.wav2vec_opt_class(
+                self.modules.wav2vec2.parameters()
+            )
+            if self.checkpointer is not None:
+                self.checkpointer.add_recoverable(
+                    "wav2vec_opt", self.wav2vec_optimizer
+                )
+
+        self.model_optimizer = self.hparams.model_opt_class(
+            self.hparams.model.parameters()
+        )
+
+        if self.checkpointer is not None:
+            self.checkpointer.add_recoverable("modelopt", self.model_optimizer)
+
+    def zero_grad(self, set_to_none=False):
+        if not self.hparams.wav2vec2.freeze:
+            self.wav2vec_optimizer.zero_grad(set_to_none)
+        self.model_optimizer.zero_grad(set_to_none)
+
+
+"""
+label_encoder = sb.dataio.encoder.CTCTextEncoder()
+
+train_data, valid_data, test_datasets, label_encoder = dataio_prepare(
+        hparams
+    )
+
+
+# We dynamicaly add the tokenizer to our brain class.
+# NB: This tokenizer corresponds to the one used for the LM!!
+"""
+from speechbrain.pretrained import EncoderASR,EncoderDecoderASR
+french_asr_model = EncoderASR.from_hparams(source="speechbrain/asr-wav2vec2-commonvoice-fr", savedir="pretrained_models/asr-wav2vec2-commonvoice-fr").cuda()
+#french_asr_model = "r"
+
+cvhparams_file, cvrun_opts, cvoverrides = sb.parse_arguments(["en_cv.yaml"])
+with open(cvhparams_file) as cvfin:
+    cvhparams = load_hyperpyyaml(cvfin, cvoverrides)
+english_asr_model = ASRCV(
+        modules=cvhparams["modules"],
+        hparams=cvhparams,
+        run_opts=cvrun_opts,
+        checkpointer=cvhparams["checkpointer"],
+    )
+english_asr_model.checkpointer.recover_if_possible()
+asr_brain = ASR(
+    modules=hparams["modules"],
+    hparams=hparams,
+    run_opts=run_opts,
+    checkpointer=hparams["checkpointer"],
+)
+asr_brain.checkpointer.recover_if_possible()
+asr_brain.modules.eval()
+english_asr_model.modules.eval()
+french_asr_model.mods.eval()
+"""
+asr_brain.tokenizer = label_encoder
+
+# Testing
+real = True
+if real :
+    for k in test_datasets.keys():  # keys are test_clean, test_other etc
+        asr_brain.hparams.wer_file = os.path.join(
+            hparams["output_folder"], "wer_{}.txt".format(k)
+        )
+        asr_brain.evaluate(
+            test_datasets[k], test_loader_kwargs=hparams["dataloader_options"]
+        )
+"""
+
+"""
+from torch.nn.utils.rnn import pad_sequence
+def load_paths(wavs_path):
+    waveforms = []
+    for path in wavs_path :
+        waveform, _ = torchaudio.load(path)
+        waveforms.append(waveform.squeeze(0))
+    # normalize array length to the bigger arrays by pading with 0's
+    padded_arrays = pad_sequence(waveforms, batch_first=True)
+    return torch.tensor(padded_arrays)
+
+waveform = load_paths(["/content/drive/MyDrive/tunisian_corpora/tunisian_without_wavlm/samples/Salah10.wav","/content/drive/MyDrive/tunisian_corpora/tunisian_without_wavlm/samples/Salah10.wav"])
+embeddings, posteriogram = asr_brain.custom_encode(waveform,None)
+print(embeddings.shape)
+print(posteriogram.shape)
+"""
+
+from speechbrain.pretrained import EncoderASR,EncoderDecoderASR
+import torchaudio
+import  speechbrain as sb
+import torch
+from torch.nn.utils.rnn import pad_sequence
+import torch
+import speechbrain as sb
+import numpy as np
+import torch.optim as optim
+import torch.nn as nn
+
+# Commented out IPython magic to ensure Python compatibility.
+# %ls
+
+#UTILS FUNCTIOJNS
+def get_size_dimensions(arr):
+    size_dimensions = []
+    while isinstance(arr, list):
+        size_dimensions.append(len(arr))
+        arr = arr[0]
+    return size_dimensions
+
+def scale_array(batch,n):
+    scaled_batch = []
+
+    for array in batch:
+        if(n < len(array)): raise ValueError("Cannot scale Array down")
+
+        repeat = round(n/len(array))+1
+        scaled_length_array= []
+
+        for i in array:
+            for j in range(repeat) :
+                if(len(scaled_length_array) == n): break
+                scaled_length_array.append(i)
+
+        scaled_batch.append(scaled_length_array)
+
+    return torch.tensor(scaled_batch)
+
+
+def load_paths(wavs_path):
+    waveforms = []
+    for path in wavs_path :
+        waveform, _ = torchaudio.load(path)
+        waveforms.append(waveform.squeeze(0))
+    # normalize array length to the bigger arrays by pading with 0's
+    padded_arrays = pad_sequence(waveforms, batch_first=True)
+    return torch.tensor(padded_arrays)
+
+
+
+def word_to_vec(input_string):
+    mapping= {'a': 1, 'b': 2, 'c': 3, 'd': 4, 'e': 5, 'f': 6, 'g': 7, 'h': 8, 'i': 9, 'j': 10, 'k': 11, 'l': 12, 'm': 13, 'n': 14, 'o': 15, 'p': 16, 'q': 17, 'r': 18, 's': 19, 't': 20, 'u': 21, 'v': 22, 'w': 23, 'x': 24, 'y': 25, 'z': 26, 'ا': 27, 'ب': 28, 'ت': 29, 'ث': 30, 'ج': 31, 'ح': 32, 'خ': 33, 'د': 34, 'ذ': 35, 'ر': 36, 'ز': 37, 'س': 38, 'ش': 39, 'ص': 40, 'ض': 41, 'ط': 42, 'ظ': 43, 'ع': 44, 'غ': 45, 'ف': 46, 'ق': 47, 'ك': 48, 'ل': 49, 'م': 50, 'ن': 51, 'ه': 52, 'و': 53, 'ي': 54,' ':55}
+
+    numbers = [mapping[word] for word in input_string if word in mapping]
+    return numbers
+
+device = 'cuda'
+verbose = 0
+#FLOW LEVEL FUNCTIONS
+def merge_strategy(embeddings1, embeddings2, embeddings3,post1, post2,post3):
+
+
+    post1 = post1.to(device)
+    post2 = post2.to(device)
+    post3 = post3.to(device)
+    embeddings1 = embeddings1.to(device)
+    embeddings2 = embeddings2.to(device)
+    embeddings3 = embeddings3.to(device)
+
+    posteriograms_merged = torch.cat((post1,post2,post3),dim=2)
+    embeddings_merged = torch.cat((embeddings1,embeddings2,embeddings3),dim=2)
+
+    if(verbose !=0):
+      print('MERGED POST ',posteriograms_merged.shape)
+      print('MERGED emb ',embeddings_merged.shape)
+
+    return torch.cat((posteriograms_merged,embeddings_merged),dim=2).to(device)
+
+def decode(model,wavs,wav_lens):
+
+    with torch.no_grad():
+        wav_lens = wav_lens.to(model.device)
+        encoder_out = model.encode_batch(wavs, wav_lens)
+        predictions = model.decoding_function(encoder_out, wav_lens)
+        return predictions
+
+def middle_layer(batch, lens):
+
+    tn_embeddings, tn_posteriogram = asr_brain.custom_encode(batch,None)
+
+    fr_embeddings = french_asr_model.mods.encoder.wav2vec2(batch)
+    fr_posteriogram =french_asr_model.encode_batch(batch,lens)
+    en_embeddings = english_asr_model.modules.wav2vec2(batch, lens)
+    x = english_asr_model.modules.enc(en_embeddings)
+    en_posteriogram = english_asr_model.modules.ctc_lin(x)
+    #scores, en_posteriogram = english_asr_model.mods.decoder(en_embeddings ,lens)
+    if(verbose !=0):
+      print('[EMBEDDINGS] FR:',fr_embeddings.shape, "EN:",en_embeddings.shape, "TN:", tn_embeddings.shape)
+      print('[POSTERIOGRAM] FR:',fr_posteriogram.shape, "EN:",en_posteriogram.shape,"TN:",tn_posteriogram.shape)
+
+
+    bilangual_sample = merge_strategy(fr_embeddings,en_embeddings,tn_embeddings,fr_posteriogram,en_posteriogram,tn_posteriogram)
+    return bilangual_sample
+
+class Mixer(sb.core.Brain):
+
+    def compute_forward(self, batch, stage):
+        """Forward computations from the waveform batches to the output probabilities."""
+        wavs, wav_lens = batch.sig
+        wavs, wav_lens = wavs.to(self.device), wav_lens.to(self.device)
+
+        if stage == sb.Stage.TRAIN:
+            if hasattr(self.hparams, "augmentation"):
+                wavs = self.hparams.augmentation(wavs, wav_lens)
+
+        multi_langual_feats = middle_layer(wavs, wav_lens)
+        multi_langual_feats= multi_langual_feats.to(device)
+        feats, _ = self.modules.enc(multi_langual_feats)
+        logits = self.modules.ctc_lin(feats)
+        p_ctc = self.hparams.log_softmax(logits)
+        
+        if stage!= sb.Stage.TRAIN:
+            p_tokens = sb.decoders.ctc_greedy_decode(
+                p_ctc, wav_lens, blank_id=self.hparams.blank_index
+            )
+        else : 
+            p_tokens = None
+        return p_ctc, wav_lens, p_tokens
+
+    def compute_objectives(self, predictions, batch, stage):
+        """Computes the loss (CTC) given predictions and targets."""
+
+        p_ctc, wav_lens , predicted_tokens= predictions
+
+        ids = batch.id
+        tokens, tokens_lens = batch.tokens
+
+        loss = self.hparams.ctc_cost(p_ctc, tokens, wav_lens, tokens_lens)
+
+
+        if stage == sb.Stage.VALID:
+            predicted_words = [
+                "".join(self.tokenizer.decode_ndim(utt_seq)).split(" ")
+                for utt_seq in predicted_tokens
+            ]
+            target_words = [wrd.split(" ") for wrd in batch.wrd]
+            self.wer_metric.append(ids, predicted_words, target_words)
+            self.cer_metric.append(ids, predicted_words, target_words)
+        if stage ==sb.Stage.TEST : 
+            if self.hparams.language_modelling:
+                predicted_words = []
+                for logs in p_ctc:
+                    text = decoder.decode(logs.detach().cpu().numpy())
+                    predicted_words.append(text.split(" "))
+            else : 
+                predicted_words = [
+                    "".join(self.tokenizer.decode_ndim(utt_seq)).split(" ")
+                    for utt_seq in predicted_tokens
+                ]
+
+            target_words = [wrd.split(" ") for wrd in batch.wrd]
+            self.wer_metric.append(ids, predicted_words, target_words)
+            self.cer_metric.append(ids, predicted_words, target_words)
+
+        return loss
+
+    def fit_batch(self, batch):
+        """Train the parameters given a single batch in input"""
+        should_step = self.step % self.grad_accumulation_factor == 0
+        # Managing automatic mixed precision
+        # TOFIX: CTC fine-tuning currently is unstable
+        # This is certainly due to CTC being done in fp16 instead of fp32
+        if self.auto_mix_prec:
+            with torch.cuda.amp.autocast():
+                with self.no_sync():
+                    outputs = self.compute_forward(batch, sb.Stage.TRAIN)
+                loss = self.compute_objectives(outputs, batch, sb.Stage.TRAIN)
+            with self.no_sync(not should_step):
+                self.scaler.scale(
+                    loss / self.grad_accumulation_factor
+                ).backward()
+            if should_step:
+
+
+                self.scaler.unscale_(self.model_optimizer)
+                if self.check_gradients(loss):
+                    self.scaler.step(self.model_optimizer)
+                self.scaler.update()
+                self.zero_grad()
+                self.optimizer_step += 1
+        else:
+            # This is mandatory because HF models have a weird behavior with DDP
+            # on the forward pass
+            with self.no_sync():
+                outputs = self.compute_forward(batch, sb.Stage.TRAIN)
+
+            loss = self.compute_objectives(outputs, batch, sb.Stage.TRAIN)
+
+            with self.no_sync(not should_step):
+                (loss / self.grad_accumulation_factor).backward()
+            if should_step:
+                if self.check_gradients(loss):
+                    self.model_optimizer.step()
+                self.zero_grad()
+                self.optimizer_step += 1
+
+        self.on_fit_batch_end(batch, outputs, loss, should_step)
+        return loss.detach().cpu()
+
+    def evaluate_batch(self, batch, stage):
+        """Computations needed for validation/test batches"""
+        predictions = self.compute_forward(batch, stage=stage)
+        with torch.no_grad():
+            loss = self.compute_objectives(predictions, batch, stage=stage)
+        return loss.detach()
+
+    def on_stage_start(self, stage, epoch):
+        """Gets called at the beginning of each epoch"""
+        if stage != sb.Stage.TRAIN:
+            self.cer_metric = self.hparams.cer_computer()
+            self.wer_metric = self.hparams.error_rate_computer()
+
+    def on_stage_end(self, stage, stage_loss, epoch):
+        """Gets called at the end of an epoch."""
+        # Compute/store important stats
+        stage_stats = {"loss": stage_loss}
+        if stage == sb.Stage.TRAIN:
+            self.train_stats = stage_stats
+        else:
+            stage_stats["CER"] = self.cer_metric.summarize("error_rate")
+            stage_stats["WER"] = self.wer_metric.summarize("error_rate")
+
+        # Perform end-of-iteration things, like annealing, logging, etc.
+        if stage == sb.Stage.VALID:
+            old_lr_model, new_lr_model = self.hparams.lr_annealing_model(
+                stage_stats["loss"]
+            )
+            sb.nnet.schedulers.update_learning_rate(
+                self.model_optimizer, new_lr_model
+            )
+            self.hparams.train_logger.log_stats(
+                stats_meta={
+                    "epoch": epoch,
+                    "lr_model": old_lr_model,
+                },
+                train_stats=self.train_stats,
+                valid_stats=stage_stats,
+            )
+            self.checkpointer.save_and_keep_only(
+                meta={"WER": stage_stats["WER"]}, min_keys=["WER"],
+            )
+        elif stage == sb.Stage.TEST:
+            self.hparams.train_logger.log_stats(
+                stats_meta={"Epoch loaded": self.hparams.epoch_counter.current},
+                test_stats=stage_stats,
+            )
+            with open(self.hparams.wer_file, "w") as w:
+                self.wer_metric.write_stats(w)
+
+    def init_optimizers(self):
+
+        self.model_optimizer = self.hparams.model_opt_class(
+            self.hparams.model.parameters()
+        )
+
+        if self.checkpointer is not None:
+            self.checkpointer.add_recoverable("modelopt", self.model_optimizer)
+
+    def zero_grad(self, set_to_none=False):
+
+        self.model_optimizer.zero_grad(set_to_none)
+
+
+hparams_file, run_opts, overrides = sb.parse_arguments(sys.argv[1:])
+
+# If distributed_launch=True then
+# create ddp_group with the right communication protocol
+sb.utils.distributed.ddp_init_group(run_opts)
+
+with open(hparams_file) as fin:
+    hparams = load_hyperpyyaml(fin, overrides)
+
+# Create experiment directory
+sb.create_experiment_directory(
+    experiment_directory=hparams["output_folder"],
+    hyperparams_to_save=hparams_file,
+    overrides=overrides,
+)
+def read_labels_file(labels_file):
+    with open(labels_file, "r",encoding="utf-8") as lf:
+        lines = lf.read().splitlines()
+        division = "==="
+        numbers = {}
+        for line in lines :
+            if division in line :
+                break
+            string, number = line.split("=>")
+            number = int(number)
+            string = string[1:-2]
+            numbers[number] = string
+        return [numbers[x] for x in range(len(numbers))]
+train_data, valid_data, test_datasets, label_encoder = dataio_prepare(
+        hparams
+    )
+
+
+labels = read_labels_file(os.path.join(hparams["save_folder"], "label_encoder.txt"))
+labels = [""] + labels[1:-1] + ["1"] 
+if hparams["language_modelling"]:
+    decoder = build_ctcdecoder(
+        labels,
+        kenlm_model_path=hparams["ngram_lm_path"],  # either .arpa or .bin file
+        alpha=0.5,  # tuned on a val set
+        beta=1,  # tuned on a val set
+    )
+
+
+
+
+mixer = Mixer(
+    modules=hparams["modules"],
+    hparams=hparams,
+    run_opts=run_opts,
+    checkpointer=hparams["checkpointer"],
+)
+mixer.tokenizer = label_encoder
+
+
+mixer.fit(
+    mixer.hparams.epoch_counter,
+    train_data,
+    valid_data,
+    train_loader_kwargs=hparams["dataloader_options"],
+    valid_loader_kwargs=hparams["test_dataloader_options"],
+)
+print(test_datasets.keys())
+for k in test_datasets.keys():  # keys are test_clean, test_other etc
+    mixer.hparams.wer_file = os.path.join(
+        hparams["output_folder"], "wer_{}.txt".format(k)
+    )
+    mixer.evaluate(
+        test_datasets[k], test_loader_kwargs=hparams["test_dataloader_options"]
+        )
+

EnglishCV/results/wav2vec2_ctc_en/1234/hyperparams.yaml

@@ -0,0 +1,190 @@
+# Generated 2023-09-06 from:
+# /gpfsdswork/projects/rech/nou/uzn19yk/final_forke/speechbrain-3/recipes/CommonVoice/ASR/CTC/hparams/train_en_with_wav2vec.yaml
+# yamllint disable
+# ################################
+# Model: wav2vec2 + DNN + CTC
+# Augmentation: SpecAugment
+# Authors: Titouan Parcollet 2021
+# ################################
+
+# Seed needs to be set at top of yaml, before objects with parameters are made
+seed: 1234
+__set_seed: !!python/object/apply:torch.manual_seed [1234]
+output_folder: results/wav2vec2_ctc_en/1234
+wer_file: results/wav2vec2_ctc_en/1234/wer.txt
+save_folder: results/wav2vec2_ctc_en/1234/save
+train_log: results/wav2vec2_ctc_en/1234/train_log.txt
+
+# URL for the biggest Fairseq english wav2vec2 model.
+wav2vec2_hub: facebook/wav2vec2-large-lv60
+wav2vec2_folder: results/wav2vec2_ctc_en/1234/save/wav2vec2_checkpoint
+
+# Data files
+data_folder: 
+  /gpfsscratch/rech/nou/uzn19yk/download/cv-corpus-12.0-2022-12-07/en/cv-corpus-12.0-2022-12-07/en             # e.g, /localscratch/cv-corpus-5.1-2020-06-22/fr
+train_tsv_file: 
+  /gpfsscratch/rech/nou/uzn19yk/download/cv-corpus-12.0-2022-12-07/en/cv-corpus-12.0-2022-12-07/en/train.tsv # Standard CommonVoice .tsv files
+dev_tsv_file: 
+  /gpfsscratch/rech/nou/uzn19yk/download/cv-corpus-12.0-2022-12-07/en/cv-corpus-12.0-2022-12-07/en/dev.tsv # Standard CommonVoice .tsv files
+test_tsv_file: 
+  /gpfsscratch/rech/nou/uzn19yk/download/cv-corpus-12.0-2022-12-07/en/cv-corpus-12.0-2022-12-07/en/test.tsv # Standard CommonVoice .tsv files
+accented_letters: false
+language: en # use 'it' for Italian, 'rw' for Kinyarwanda, 'en' for english
+train_csv: results/wav2vec2_ctc_en/1234/save/train.csv
+valid_csv: results/wav2vec2_ctc_en/1234/save/dev.csv
+test_csv: results/wav2vec2_ctc_en/1234/save/test.csv
+skip_prep: false # Skip data preparation
+
+# We remove utterance slonger than 10s in the train/dev/test sets as
+# longer sentences certainly correspond to "open microphones".
+avoid_if_longer_than: 10.0
+
+# Training parameters
+number_of_epochs: 10
+lr: 1.0
+lr_wav2vec: 0.0001
+sorting: ascending
+auto_mix_prec: false
+sample_rate: 16000
+ckpt_interval_minutes: 30 # save checkpoint every N min
+
+# With data_parallel batch_size is split into N jobs
+# With DDP batch_size is multiplied by N jobs
+# Must be 8 per GPU to fit 32GB of VRAM
+batch_size: 8
+test_batch_size: 4
+
+dataloader_options:
+  batch_size: 8
+  num_workers: 6
+test_dataloader_options:
+  batch_size: 4
+  num_workers: 6
+
+# BPE parameters
+token_type: char  # ["unigram", "bpe", "char"]
+character_coverage: 1.0
+
+# Model parameters
+# activation: !name:torch.nn.LeakyReLU
+wav2vec_output_dim: 1024
+dnn_neurons: 1024
+freeze_wav2vec: false
+freeze_feature_extractor: true
+dropout: 0.15
+warmup_steps: 500
+
+# Outputs
+output_neurons: 29  # BPE size, index(blank/eos/bos) = 0
+
+# Decoding parameters
+# Be sure that the bos and eos index match with the BPEs ones
+blank_index: 0
+bos_index: 1
+eos_index: 2
+
+#
+# Functions and classes
+#
+epoch_counter: &id007 !new:speechbrain.utils.epoch_loop.EpochCounter
+
+  limit: 10
+
+augmentation: !new:speechbrain.lobes.augment.TimeDomainSpecAugment
+  sample_rate: 16000
+  speeds: [95, 100, 105]
+
+enc: &id002 !new:speechbrain.nnet.containers.Sequential
+  input_shape: [null, null, 1024]
+  linear1: !name:speechbrain.nnet.linear.Linear
+    n_neurons: 1024
+    bias: true
+  bn1: !name:speechbrain.nnet.normalization.BatchNorm1d
+  activation: !new:torch.nn.LeakyReLU
+  drop: !new:torch.nn.Dropout
+    p: 0.15
+  linear2: !name:speechbrain.nnet.linear.Linear
+    n_neurons: 1024
+    bias: true
+  bn2: !name:speechbrain.nnet.normalization.BatchNorm1d
+  activation2: !new:torch.nn.LeakyReLU
+  drop2: !new:torch.nn.Dropout
+    p: 0.15
+  linear3: !name:speechbrain.nnet.linear.Linear
+    n_neurons: 1024
+    bias: true
+  bn3: !name:speechbrain.nnet.normalization.BatchNorm1d
+  activation3: !new:torch.nn.LeakyReLU
+
+wav2vec2: &id001 !new:speechbrain.lobes.models.huggingface_wav2vec.HuggingFaceWav2Vec2
+  source: /gpfsscratch/rech/nou/uzn19yk/wav2vec2-large-lv60/
+  output_norm: true
+  freeze: false
+  freeze_feature_extractor: true
+  save_path: results/wav2vec2_ctc_en/1234/save/wav2vec2_checkpoint
+
+#####
+# Uncomment this block if you prefer to use a Fairseq pretrained model instead
+# of a HuggingFace one. Here, we provide an URL that is obtained from the
+# Fairseq github for the multilingual XLSR.
+#
+#wav2vec2_url: https://dl.fbaipublicfiles.com/fairseq/wav2vec/xlsr_53_56k.pt
+#wav2vec2: !new:speechbrain.lobes.models.fairseq_wav2vec.FairseqWav2Vec2
+#    pretrained_path: !ref <wav2vec2_url>
+#    output_norm: True
+#    freeze: False
+#    save_path: !ref <save_folder>/wav2vec2_checkpoint/model.pt
+#####
+
+ctc_lin: &id003 !new:speechbrain.nnet.linear.Linear
+
+  input_size: 1024
+  n_neurons: 29
+
+log_softmax: !new:speechbrain.nnet.activations.Softmax
+  apply_log: true
+
+ctc_cost: !name:speechbrain.nnet.losses.ctc_loss
+  blank_index: 0
+
+modules:
+  wav2vec2: *id001
+  enc: *id002
+  ctc_lin: *id003
+model: &id004 !new:torch.nn.ModuleList
+- [*id002, *id003]
+model_opt_class: !name:torch.optim.Adadelta
+  lr: 1.0
+  rho: 0.95
+  eps: 1.e-8
+
+wav2vec_opt_class: !name:torch.optim.Adam
+  lr: 0.0001
+
+lr_annealing_model: &id005 !new:speechbrain.nnet.schedulers.NewBobScheduler
+  initial_value: 1.0
+  improvement_threshold: 0.0025
+  annealing_factor: 0.8
+  patient: 0
+
+lr_annealing_wav2vec: &id006 !new:speechbrain.nnet.schedulers.NewBobScheduler
+  initial_value: 0.0001
+  improvement_threshold: 0.0025
+  annealing_factor: 0.9
+  patient: 0
+
+checkpointer: !new:speechbrain.utils.checkpoints.Checkpointer
+  checkpoints_dir: results/wav2vec2_ctc_en/1234/save
+  recoverables:
+    wav2vec2: *id001
+    model: *id004
+    scheduler_model: *id005
+    scheduler_wav2vec: *id006
+    counter: *id007
+train_logger: !new:speechbrain.utils.train_logger.FileTrainLogger
+  save_file: results/wav2vec2_ctc_en/1234/train_log.txt
+
+error_rate_computer: !name:speechbrain.utils.metric_stats.ErrorRateStats
+
+cer_computer: !name:speechbrain.utils.metric_stats.ErrorRateStats
+  split_tokens: true

EnglishCV/results/wav2vec2_ctc_en/1234/save/29_char.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ee4214a3ebba9461ca02ca61220a2338412bbf9ef5a5982f2bc40740c4ab91a8
+size 238011

EnglishCV/results/wav2vec2_ctc_en/1234/save/29_char.vocab

@@ -0,0 +1,28 @@
+<unk>	0
+▁	-1.786
+E	-2.27261
+A	-2.6326
+T	-2.64317
+I	-2.76341
+S	-2.81519
+O	-2.8189
+N	-2.83568
+R	-2.87568
+H	-3.22802
+L	-3.30075
+D	-3.43047
+C	-3.58554
+U	-3.84445
+M	-3.84732
+F	-4.07023
+P	-4.09107
+G	-4.16259
+W	-4.25412
+Y	-4.30147
+B	-4.36224
+V	-4.71267
+K	-5.1744
+X	-6.46672
+J	-6.5246
+Z	-6.95828
+Q	-7.12388

EnglishCV/results/wav2vec2_ctc_en/1234/save/CKPT+2023-09-06+22-56-31+00/CKPT.yaml

@@ -0,0 +1,4 @@
+# yamllint disable
+WER: 18.234978071545488
+end-of-epoch: true
+unixtime: 1694033791.9455216

EnglishCV/results/wav2vec2_ctc_en/1234/save/CKPT+2023-09-06+22-56-31+00/brain.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:06617abf655f8550362b963062fc2a57bd819826ab70e63701676ea09d23618d
+size 51

EnglishCV/results/wav2vec2_ctc_en/1234/save/CKPT+2023-09-06+22-56-31+00/counter.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d4735e3a265e16eee03f59718b9b5d03019c07d8b6c51f90da3a666eec13ab35
+size 1

EnglishCV/results/wav2vec2_ctc_en/1234/save/CKPT+2023-09-06+22-56-31+00/dataloader-TRAIN.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f21c20a479fcc07663ec4255ad1c85466afb791f514f8f3baa174bd56edca2d4
+size 6

EnglishCV/results/wav2vec2_ctc_en/1234/save/CKPT+2023-09-06+22-56-31+00/model.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:422a1d7a30720e846d2cb79ff510832fe96c1495f559f08fb37bdd118269ea7b
+size 12769326

EnglishCV/results/wav2vec2_ctc_en/1234/save/CKPT+2023-09-06+22-56-31+00/modelopt.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:65cda77e4403deb7c8cee3052ac687bfc3bf6e68264dcb0e297e8f88bccf0d66
+size 25485359

EnglishCV/results/wav2vec2_ctc_en/1234/save/CKPT+2023-09-06+22-56-31+00/scheduler_model.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c9c36e38dd81971c68387a9f921cf0d61adad21f5b3f6420b6f3015b0f9d20df
+size 511

EnglishCV/results/wav2vec2_ctc_en/1234/save/CKPT+2023-09-06+22-56-31+00/scheduler_wav2vec.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0293788921aad16c6e904d7ec0b7dba2dd4778fa3b7f1bfa04276b3965599999
+size 515

EnglishCV/results/wav2vec2_ctc_en/1234/save/CKPT+2023-09-06+22-56-31+00/wav2vec2.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0f7073aa70c88927f11cff4f2ba63a026c8ff6c119837391d84013feb229ad3e
+size 1261924189

EnglishCV/results/wav2vec2_ctc_en/1234/save/CKPT+2023-09-06+22-56-31+00/wav2vec_opt.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:42691a96ebaba3dd3baf7e2521763db7f79b37a6bde9b0ea9d1adc2cac5bdf5e
+size 2490156402

EnglishCV/results/wav2vec2_ctc_en/1234/train_with_wav2vec.py

@@ -0,0 +1,388 @@
+#!/usr/bin/env python3
+import sys
+import torch
+import logging
+import speechbrain as sb
+import torchaudio
+from hyperpyyaml import load_hyperpyyaml
+from speechbrain.tokenizers.SentencePiece import SentencePiece
+from speechbrain.utils.data_utils import undo_padding
+from speechbrain.utils.distributed import run_on_main
+
+"""Recipe for training a sequence-to-sequence ASR system with CommonVoice.
+The system employs a wav2vec2 encoder and a CTC decoder.
+Decoding is performed with greedy decoding (will be extended to beam search).
+
+To run this recipe, do the following:
+> python train_with_wav2vec2.py hparams/train_with_wav2vec2.yaml
+
+With the default hyperparameters, the system employs a pretrained wav2vec2 encoder.
+The wav2vec2 model is pretrained following the model given in the hprams file.
+It may be dependent on the language.
+
+The neural network is trained with CTC on sub-word units estimated with
+Byte Pairwise Encoding (BPE).
+
+The experiment file is flexible enough to support a large variety of
+different systems. By properly changing the parameter files, you can try
+different encoders, decoders, tokens (e.g, characters instead of BPE),
+training languages (all CommonVoice languages), and many
+other possible variations.
+
+Authors
+ * Titouan Parcollet 2021
+"""
+
+logger = logging.getLogger(__name__)
+
+
+# Define training procedure
+class ASR(sb.core.Brain):
+    def compute_forward(self, batch, stage):
+        """Forward computations from the waveform batches to the output probabilities."""
+
+        batch = batch.to(self.device)
+        wavs, wav_lens = batch.sig
+        tokens_bos, _ = batch.tokens_bos
+        wavs, wav_lens = wavs.to(self.device), wav_lens.to(self.device)
+
+        if stage == sb.Stage.TRAIN:
+            if hasattr(self.hparams, "augmentation"):
+                wavs = self.hparams.augmentation(wavs, wav_lens)
+
+        # Forward pass
+        feats = self.modules.wav2vec2(wavs, wav_lens)
+        x = self.modules.enc(feats)
+        logits = self.modules.ctc_lin(x)
+        p_ctc = self.hparams.log_softmax(logits)
+
+        return p_ctc, wav_lens
+
+    def compute_objectives(self, predictions, batch, stage):
+        """Computes the loss (CTC) given predictions and targets."""
+
+        p_ctc, wav_lens = predictions
+
+        ids = batch.id
+        tokens_eos, tokens_eos_lens = batch.tokens_eos
+        tokens, tokens_lens = batch.tokens
+
+        loss = self.hparams.ctc_cost(p_ctc, tokens, wav_lens, tokens_lens)
+
+        if stage != sb.Stage.TRAIN:
+            # Decode token terms to words
+            sequence = sb.decoders.ctc_greedy_decode(
+                p_ctc, wav_lens, blank_id=self.hparams.blank_index
+            )
+
+            predicted_words = self.tokenizer(sequence, task="decode_from_list")
+
+            # Convert indices to words
+            target_words = undo_padding(tokens, tokens_lens)
+            target_words = self.tokenizer(target_words, task="decode_from_list")
+
+            self.wer_metric.append(ids, predicted_words, target_words)
+            self.cer_metric.append(ids, predicted_words, target_words)
+
+        return loss
+
+    def fit_batch(self, batch):
+        """Train the parameters given a single batch in input"""
+        should_step = self.step % self.grad_accumulation_factor == 0
+        # Managing automatic mixed precision
+        # TOFIX: CTC fine-tuning currently is unstable
+        # This is certainly due to CTC being done in fp16 instead of fp32
+        if self.auto_mix_prec:
+            with torch.cuda.amp.autocast():
+                with self.no_sync():
+                    outputs = self.compute_forward(batch, sb.Stage.TRAIN)
+                loss = self.compute_objectives(outputs, batch, sb.Stage.TRAIN)
+            with self.no_sync(not should_step):
+                self.scaler.scale(
+                    loss / self.grad_accumulation_factor
+                ).backward()
+            if should_step:
+
+                if not self.hparams.wav2vec2.freeze:
+                    self.scaler.unscale_(self.wav2vec_optimizer)
+                self.scaler.unscale_(self.model_optimizer)
+                if self.check_gradients(loss):
+                    if not self.hparams.wav2vec2.freeze:
+                        if self.optimizer_step >= self.hparams.warmup_steps:
+                            self.scaler.step(self.wav2vec_optimizer)
+                    self.scaler.step(self.model_optimizer)
+                self.scaler.update()
+                self.zero_grad()
+                self.optimizer_step += 1
+        else:
+            # This is mandatory because HF models have a weird behavior with DDP
+            # on the forward pass
+            with self.no_sync():
+                outputs = self.compute_forward(batch, sb.Stage.TRAIN)
+
+            loss = self.compute_objectives(outputs, batch, sb.Stage.TRAIN)
+
+            with self.no_sync(not should_step):
+                (loss / self.grad_accumulation_factor).backward()
+            if should_step:
+                if self.check_gradients(loss):
+                    if not self.hparams.wav2vec2.freeze:
+                        if self.optimizer_step >= self.hparams.warmup_steps:
+                            self.wav2vec_optimizer.step()
+                    self.model_optimizer.step()
+                self.zero_grad()
+                self.optimizer_step += 1
+
+        self.on_fit_batch_end(batch, outputs, loss, should_step)
+        return loss.detach().cpu()
+
+    def evaluate_batch(self, batch, stage):
+        """Computations needed for validation/test batches"""
+        predictions = self.compute_forward(batch, stage=stage)
+        with torch.no_grad():
+            loss = self.compute_objectives(predictions, batch, stage=stage)
+        return loss.detach()
+
+    def on_stage_start(self, stage, epoch):
+        """Gets called at the beginning of each epoch"""
+        if stage != sb.Stage.TRAIN:
+            self.cer_metric = self.hparams.cer_computer()
+            self.wer_metric = self.hparams.error_rate_computer()
+
+    def on_stage_end(self, stage, stage_loss, epoch):
+        """Gets called at the end of an epoch."""
+        # Compute/store important stats
+        stage_stats = {"loss": stage_loss}
+        if stage == sb.Stage.TRAIN:
+            self.train_stats = stage_stats
+        else:
+            stage_stats["CER"] = self.cer_metric.summarize("error_rate")
+            stage_stats["WER"] = self.wer_metric.summarize("error_rate")
+
+        # Perform end-of-iteration things, like annealing, logging, etc.
+        if stage == sb.Stage.VALID:
+            old_lr_model, new_lr_model = self.hparams.lr_annealing_model(
+                stage_stats["loss"]
+            )
+            old_lr_wav2vec, new_lr_wav2vec = self.hparams.lr_annealing_wav2vec(
+                stage_stats["loss"]
+            )
+            sb.nnet.schedulers.update_learning_rate(
+                self.model_optimizer, new_lr_model
+            )
+            if not self.hparams.wav2vec2.freeze:
+                sb.nnet.schedulers.update_learning_rate(
+                    self.wav2vec_optimizer, new_lr_wav2vec
+                )
+            self.hparams.train_logger.log_stats(
+                stats_meta={
+                    "epoch": epoch,
+                    "lr_model": old_lr_model,
+                    "lr_wav2vec": old_lr_wav2vec,
+                },
+                train_stats=self.train_stats,
+                valid_stats=stage_stats,
+            )
+            self.checkpointer.save_and_keep_only(
+                meta={"WER": stage_stats["WER"]}, min_keys=["WER"],
+            )
+        elif stage == sb.Stage.TEST:
+            self.hparams.train_logger.log_stats(
+                stats_meta={"Epoch loaded": self.hparams.epoch_counter.current},
+                test_stats=stage_stats,
+            )
+            with open(self.hparams.wer_file, "w") as w:
+                self.wer_metric.write_stats(w)
+
+    def init_optimizers(self):
+        "Initializes the wav2vec2 optimizer and model optimizer"
+
+        # If the wav2vec encoder is unfrozen, we create the optimizer
+        if not self.hparams.wav2vec2.freeze:
+            self.wav2vec_optimizer = self.hparams.wav2vec_opt_class(
+                self.modules.wav2vec2.parameters()
+            )
+            if self.checkpointer is not None:
+                self.checkpointer.add_recoverable(
+                    "wav2vec_opt", self.wav2vec_optimizer
+                )
+
+        self.model_optimizer = self.hparams.model_opt_class(
+            self.hparams.model.parameters()
+        )
+
+        if self.checkpointer is not None:
+            self.checkpointer.add_recoverable("modelopt", self.model_optimizer)
+
+    def zero_grad(self, set_to_none=False):
+        if not self.hparams.wav2vec2.freeze:
+            self.wav2vec_optimizer.zero_grad(set_to_none)
+        self.model_optimizer.zero_grad(set_to_none)
+
+
+# Define custom data procedure
+def dataio_prepare(hparams, tokenizer):
+    """This function prepares the datasets to be used in the brain class.
+    It also defines the data processing pipeline through user-defined functions."""
+
+    # 1. Define datasets
+    data_folder = hparams["data_folder"]
+
+    train_data = sb.dataio.dataset.DynamicItemDataset.from_csv(
+        csv_path=hparams["train_csv"], replacements={"data_root": data_folder},
+    )
+
+    if hparams["sorting"] == "ascending":
+        # we sort training data to speed up training and get better results.
+        train_data = train_data.filtered_sorted(
+            sort_key="duration",
+            key_max_value={"duration": hparams["avoid_if_longer_than"]},
+        )
+        # when sorting do not shuffle in dataloader ! otherwise is pointless
+        hparams["dataloader_options"]["shuffle"] = False
+
+    elif hparams["sorting"] == "descending":
+        train_data = train_data.filtered_sorted(
+            sort_key="duration",
+            reverse=True,
+            key_max_value={"duration": hparams["avoid_if_longer_than"]},
+        )
+        # when sorting do not shuffle in dataloader ! otherwise is pointless
+        hparams["dataloader_options"]["shuffle"] = False
+
+    elif hparams["sorting"] == "random":
+        pass
+
+    else:
+        raise NotImplementedError(
+            "sorting must be random, ascending or descending"
+        )
+
+    valid_data = sb.dataio.dataset.DynamicItemDataset.from_csv(
+        csv_path=hparams["valid_csv"], replacements={"data_root": data_folder},
+    )
+    # We also sort the validation data so it is faster to validate
+    valid_data = valid_data.filtered_sorted(sort_key="duration")
+
+    test_data = sb.dataio.dataset.DynamicItemDataset.from_csv(
+        csv_path=hparams["test_csv"], replacements={"data_root": data_folder},
+    )
+
+    # We also sort the validation data so it is faster to validate
+    test_data = test_data.filtered_sorted(sort_key="duration")
+
+    datasets = [train_data, valid_data, test_data]
+
+    # 2. Define audio pipeline:
+    @sb.utils.data_pipeline.takes("wav")
+    @sb.utils.data_pipeline.provides("sig")
+    def audio_pipeline(wav):
+        info = torchaudio.info(wav)
+        sig = sb.dataio.dataio.read_audio(wav)
+        resampled = torchaudio.transforms.Resample(
+            info.sample_rate, hparams["sample_rate"],
+        )(sig)
+        return resampled
+
+    sb.dataio.dataset.add_dynamic_item(datasets, audio_pipeline)
+
+    # 3. Define text pipeline:
+    @sb.utils.data_pipeline.takes("wrd")
+    @sb.utils.data_pipeline.provides(
+        "tokens_list", "tokens_bos", "tokens_eos", "tokens"
+    )
+    def text_pipeline(wrd):
+        tokens_list = tokenizer.sp.encode_as_ids(wrd)
+        yield tokens_list
+        tokens_bos = torch.LongTensor([hparams["bos_index"]] + (tokens_list))
+        yield tokens_bos
+        tokens_eos = torch.LongTensor(tokens_list + [hparams["eos_index"]])
+        yield tokens_eos
+        tokens = torch.LongTensor(tokens_list)
+        yield tokens
+
+    sb.dataio.dataset.add_dynamic_item(datasets, text_pipeline)
+
+    # 4. Set output:
+    sb.dataio.dataset.set_output_keys(
+        datasets, ["id", "sig", "tokens_bos", "tokens_eos", "tokens"],
+    )
+    return train_data, valid_data, test_data
+
+
+if __name__ == "__main__":
+
+    # Load hyperparameters file with command-line overrides
+    hparams_file, run_opts, overrides = sb.parse_arguments(sys.argv[1:])
+    with open(hparams_file) as fin:
+        hparams = load_hyperpyyaml(fin, overrides)
+
+    # If --distributed_launch then
+    # create ddp_group with the right communication protocol
+    sb.utils.distributed.ddp_init_group(run_opts)
+
+    # Dataset preparation (parsing CommonVoice)
+    from common_voice_prepare import prepare_common_voice  # noqa
+
+    # Create experiment directory
+    sb.create_experiment_directory(
+        experiment_directory=hparams["output_folder"],
+        hyperparams_to_save=hparams_file,
+        overrides=overrides,
+    )
+
+    # Due to DDP, we do the preparation ONLY on the main python process
+    run_on_main(
+        prepare_common_voice,
+        kwargs={
+            "data_folder": hparams["data_folder"],
+            "save_folder": hparams["save_folder"],
+            "train_tsv_file": hparams["train_tsv_file"],
+            "dev_tsv_file": hparams["dev_tsv_file"],
+            "test_tsv_file": hparams["test_tsv_file"],
+            "accented_letters": hparams["accented_letters"],
+            "language": hparams["language"],
+            "skip_prep": hparams["skip_prep"],
+        },
+    )
+
+    # Defining tokenizer and loading it
+    tokenizer = SentencePiece(
+        model_dir=hparams["save_folder"],
+        vocab_size=hparams["output_neurons"],
+        annotation_train=hparams["train_csv"],
+        annotation_read="wrd",
+        model_type=hparams["token_type"],
+        character_coverage=hparams["character_coverage"],
+    )
+
+    # Create the datasets objects as well as tokenization and encoding :-D
+    train_data, valid_data, test_data = dataio_prepare(hparams, tokenizer)
+
+    # Trainer initialization
+    asr_brain = ASR(
+        modules=hparams["modules"],
+        hparams=hparams,
+        run_opts=run_opts,
+        checkpointer=hparams["checkpointer"],
+    )
+
+    # Adding objects to trainer.
+    asr_brain.tokenizer = tokenizer
+
+    # Training
+    asr_brain.fit(
+        asr_brain.hparams.epoch_counter,
+        train_data,
+        valid_data,
+        train_loader_kwargs=hparams["dataloader_options"],
+        valid_loader_kwargs=hparams["test_dataloader_options"],
+    )
+
+    # Test
+    asr_brain.hparams.wer_file = hparams["output_folder"] + "/wer_test.txt"
+    asr_brain.evaluate(
+        test_data,
+        min_key="WER",
+        test_loader_kwargs=hparams["test_dataloader_options"],
+    )

EnglishCV/train_en_with_wav2vec.yaml

@@ -0,0 +1,184 @@
+# ################################
+# Model: wav2vec2 + DNN + CTC
+# Augmentation: SpecAugment
+# Authors: Titouan Parcollet 2021
+# ################################
+
+# Seed needs to be set at top of yaml, before objects with parameters are made
+seed: 1234
+__set_seed: !!python/object/apply:torch.manual_seed [!ref <seed>]
+output_folder: !ref results/wav2vec2_ctc_en/<seed>
+wer_file: !ref <output_folder>/wer.txt
+save_folder: !ref <output_folder>/save
+train_log: !ref <output_folder>/train_log.txt
+
+# URL for the biggest Fairseq english wav2vec2 model.
+wav2vec2_hub: facebook/wav2vec2-large-lv60
+wav2vec2_folder: !ref <save_folder>/wav2vec2_checkpoint
+
+# Data files
+data_folder: /gpfsscratch/rech/nou/uzn19yk/download/cv-corpus-12.0-2022-12-07/en/cv-corpus-12.0-2022-12-07/en  # e.g, /localscratch/cv-corpus-5.1-2020-06-22/fr
+train_tsv_file: !ref <data_folder>/train.tsv  # Standard CommonVoice .tsv files
+dev_tsv_file: !ref <data_folder>/dev.tsv  # Standard CommonVoice .tsv files
+test_tsv_file: !ref <data_folder>/test.tsv  # Standard CommonVoice .tsv files
+accented_letters: False
+language: en # use 'it' for Italian, 'rw' for Kinyarwanda, 'en' for english
+train_csv: !ref <save_folder>/train.csv
+valid_csv: !ref <save_folder>/dev.csv
+test_csv: !ref <save_folder>/test.csv
+skip_prep: False # Skip data preparation
+
+# We remove utterance slonger than 10s in the train/dev/test sets as
+# longer sentences certainly correspond to "open microphones".
+avoid_if_longer_than: 10.0
+
+# Training parameters
+number_of_epochs: 10
+lr: 1.0
+lr_wav2vec: 0.0001
+sorting: ascending
+auto_mix_prec: False
+sample_rate: 16000
+ckpt_interval_minutes: 30 # save checkpoint every N min
+
+# With data_parallel batch_size is split into N jobs
+# With DDP batch_size is multiplied by N jobs
+# Must be 8 per GPU to fit 32GB of VRAM
+batch_size: 8
+test_batch_size: 4
+
+dataloader_options:
+    batch_size: !ref <batch_size>
+    num_workers: 6
+test_dataloader_options:
+    batch_size: !ref <test_batch_size>
+    num_workers: 6
+
+# BPE parameters
+token_type: char  # ["unigram", "bpe", "char"]
+character_coverage: 1.0
+
+# Model parameters
+# activation: !name:torch.nn.LeakyReLU
+wav2vec_output_dim: 1024
+dnn_neurons: 1024
+freeze_wav2vec: False
+freeze_feature_extractor: True
+dropout: 0.15
+warmup_steps: 500
+
+# Outputs
+output_neurons: 29  # BPE size, index(blank/eos/bos) = 0
+
+# Decoding parameters
+# Be sure that the bos and eos index match with the BPEs ones
+blank_index: 0
+bos_index: 1
+eos_index: 2
+
+#
+# Functions and classes
+#
+epoch_counter: !new:speechbrain.utils.epoch_loop.EpochCounter
+    limit: !ref <number_of_epochs>
+
+augmentation: !new:speechbrain.lobes.augment.TimeDomainSpecAugment
+    sample_rate: !ref <sample_rate>
+    speeds: [95, 100, 105]
+
+enc: !new:speechbrain.nnet.containers.Sequential
+    input_shape: [null, null, !ref <wav2vec_output_dim>]
+    linear1: !name:speechbrain.nnet.linear.Linear
+        n_neurons: !ref <dnn_neurons>
+        bias: True
+    bn1: !name:speechbrain.nnet.normalization.BatchNorm1d
+    activation: !new:torch.nn.LeakyReLU
+    drop: !new:torch.nn.Dropout
+        p: !ref <dropout>
+    linear2: !name:speechbrain.nnet.linear.Linear
+        n_neurons: !ref <dnn_neurons>
+        bias: True
+    bn2: !name:speechbrain.nnet.normalization.BatchNorm1d
+    activation2: !new:torch.nn.LeakyReLU
+    drop2: !new:torch.nn.Dropout
+        p: !ref <dropout>
+    linear3: !name:speechbrain.nnet.linear.Linear
+        n_neurons: !ref <dnn_neurons>
+        bias: True
+    bn3: !name:speechbrain.nnet.normalization.BatchNorm1d
+    activation3: !new:torch.nn.LeakyReLU
+
+wav2vec2: !new:speechbrain.lobes.models.huggingface_wav2vec.HuggingFaceWav2Vec2
+    source: /gpfsscratch/rech/nou/uzn19yk/wav2vec2-large-lv60/
+    output_norm: True
+    freeze: !ref <freeze_wav2vec>
+    freeze_feature_extractor: !ref <freeze_feature_extractor>
+    save_path: !ref <wav2vec2_folder>
+
+#####
+# Uncomment this block if you prefer to use a Fairseq pretrained model instead
+# of a HuggingFace one. Here, we provide an URL that is obtained from the
+# Fairseq github for the multilingual XLSR.
+#
+#wav2vec2_url: https://dl.fbaipublicfiles.com/fairseq/wav2vec/xlsr_53_56k.pt
+#wav2vec2: !new:speechbrain.lobes.models.fairseq_wav2vec.FairseqWav2Vec2
+#    pretrained_path: !ref <wav2vec2_url>
+#    output_norm: True
+#    freeze: False
+#    save_path: !ref <save_folder>/wav2vec2_checkpoint/model.pt
+#####
+
+ctc_lin: !new:speechbrain.nnet.linear.Linear
+    input_size: !ref <dnn_neurons>
+    n_neurons: !ref <output_neurons>
+
+log_softmax: !new:speechbrain.nnet.activations.Softmax
+    apply_log: True
+
+ctc_cost: !name:speechbrain.nnet.losses.ctc_loss
+    blank_index: !ref <blank_index>
+
+modules:
+    wav2vec2: !ref <wav2vec2>
+    enc: !ref <enc>
+    ctc_lin: !ref <ctc_lin>
+
+model: !new:torch.nn.ModuleList
+    - [!ref <enc>, !ref <ctc_lin>]
+
+model_opt_class: !name:torch.optim.Adadelta
+    lr: !ref <lr>
+    rho: 0.95
+    eps: 1.e-8
+
+wav2vec_opt_class: !name:torch.optim.Adam
+    lr: !ref <lr_wav2vec>
+
+lr_annealing_model: !new:speechbrain.nnet.schedulers.NewBobScheduler
+    initial_value: !ref <lr>
+    improvement_threshold: 0.0025
+    annealing_factor: 0.8
+    patient: 0
+
+lr_annealing_wav2vec: !new:speechbrain.nnet.schedulers.NewBobScheduler
+    initial_value: !ref <lr_wav2vec>
+    improvement_threshold: 0.0025
+    annealing_factor: 0.9
+    patient: 0
+
+checkpointer: !new:speechbrain.utils.checkpoints.Checkpointer
+    checkpoints_dir: !ref <save_folder>
+    recoverables:
+        wav2vec2: !ref <wav2vec2>
+        model: !ref <model>
+        scheduler_model: !ref <lr_annealing_model>
+        scheduler_wav2vec: !ref <lr_annealing_wav2vec>
+        counter: !ref <epoch_counter>
+
+train_logger: !new:speechbrain.utils.train_logger.FileTrainLogger
+    save_file: !ref <train_log>
+
+error_rate_computer: !name:speechbrain.utils.metric_stats.ErrorRateStats
+
+cer_computer: !name:speechbrain.utils.metric_stats.ErrorRateStats
+    split_tokens: True

EnglishCV/train_with_wav2vec.py

@@ -0,0 +1,388 @@
+#!/usr/bin/env python3
+import sys
+import torch
+import logging
+import speechbrain as sb
+import torchaudio
+from hyperpyyaml import load_hyperpyyaml
+from speechbrain.tokenizers.SentencePiece import SentencePiece
+from speechbrain.utils.data_utils import undo_padding
+from speechbrain.utils.distributed import run_on_main
+
+"""Recipe for training a sequence-to-sequence ASR system with CommonVoice.
+The system employs a wav2vec2 encoder and a CTC decoder.
+Decoding is performed with greedy decoding (will be extended to beam search).
+
+To run this recipe, do the following:
+> python train_with_wav2vec2.py hparams/train_with_wav2vec2.yaml
+
+With the default hyperparameters, the system employs a pretrained wav2vec2 encoder.
+The wav2vec2 model is pretrained following the model given in the hprams file.
+It may be dependent on the language.
+
+The neural network is trained with CTC on sub-word units estimated with
+Byte Pairwise Encoding (BPE).
+
+The experiment file is flexible enough to support a large variety of
+different systems. By properly changing the parameter files, you can try
+different encoders, decoders, tokens (e.g, characters instead of BPE),
+training languages (all CommonVoice languages), and many
+other possible variations.
+
+Authors
+ * Titouan Parcollet 2021
+"""
+
+logger = logging.getLogger(__name__)
+
+
+# Define training procedure
+class ASR(sb.core.Brain):
+    def compute_forward(self, batch, stage):
+        """Forward computations from the waveform batches to the output probabilities."""
+
+        batch = batch.to(self.device)
+        wavs, wav_lens = batch.sig
+        tokens_bos, _ = batch.tokens_bos
+        wavs, wav_lens = wavs.to(self.device), wav_lens.to(self.device)
+
+        if stage == sb.Stage.TRAIN:
+            if hasattr(self.hparams, "augmentation"):
+                wavs = self.hparams.augmentation(wavs, wav_lens)
+
+        # Forward pass
+        feats = self.modules.wav2vec2(wavs, wav_lens)
+        x = self.modules.enc(feats)
+        logits = self.modules.ctc_lin(x)
+        p_ctc = self.hparams.log_softmax(logits)
+
+        return p_ctc, wav_lens
+
+    def compute_objectives(self, predictions, batch, stage):
+        """Computes the loss (CTC) given predictions and targets."""
+
+        p_ctc, wav_lens = predictions
+
+        ids = batch.id
+        tokens_eos, tokens_eos_lens = batch.tokens_eos
+        tokens, tokens_lens = batch.tokens
+
+        loss = self.hparams.ctc_cost(p_ctc, tokens, wav_lens, tokens_lens)
+
+        if stage != sb.Stage.TRAIN:
+            # Decode token terms to words
+            sequence = sb.decoders.ctc_greedy_decode(
+                p_ctc, wav_lens, blank_id=self.hparams.blank_index
+            )
+
+            predicted_words = self.tokenizer(sequence, task="decode_from_list")
+
+            # Convert indices to words
+            target_words = undo_padding(tokens, tokens_lens)
+            target_words = self.tokenizer(target_words, task="decode_from_list")
+
+            self.wer_metric.append(ids, predicted_words, target_words)
+            self.cer_metric.append(ids, predicted_words, target_words)
+
+        return loss
+
+    def fit_batch(self, batch):
+        """Train the parameters given a single batch in input"""
+        should_step = self.step % self.grad_accumulation_factor == 0
+        # Managing automatic mixed precision
+        # TOFIX: CTC fine-tuning currently is unstable
+        # This is certainly due to CTC being done in fp16 instead of fp32
+        if self.auto_mix_prec:
+            with torch.cuda.amp.autocast():
+                with self.no_sync():
+                    outputs = self.compute_forward(batch, sb.Stage.TRAIN)
+                loss = self.compute_objectives(outputs, batch, sb.Stage.TRAIN)
+            with self.no_sync(not should_step):
+                self.scaler.scale(
+                    loss / self.grad_accumulation_factor
+                ).backward()
+            if should_step:
+
+                if not self.hparams.wav2vec2.freeze:
+                    self.scaler.unscale_(self.wav2vec_optimizer)
+                self.scaler.unscale_(self.model_optimizer)
+                if self.check_gradients(loss):
+                    if not self.hparams.wav2vec2.freeze:
+                        if self.optimizer_step >= self.hparams.warmup_steps:
+                            self.scaler.step(self.wav2vec_optimizer)
+                    self.scaler.step(self.model_optimizer)
+                self.scaler.update()
+                self.zero_grad()
+                self.optimizer_step += 1
+        else:
+            # This is mandatory because HF models have a weird behavior with DDP
+            # on the forward pass
+            with self.no_sync():
+                outputs = self.compute_forward(batch, sb.Stage.TRAIN)
+
+            loss = self.compute_objectives(outputs, batch, sb.Stage.TRAIN)
+
+            with self.no_sync(not should_step):
+                (loss / self.grad_accumulation_factor).backward()
+            if should_step:
+                if self.check_gradients(loss):
+                    if not self.hparams.wav2vec2.freeze:
+                        if self.optimizer_step >= self.hparams.warmup_steps:
+                            self.wav2vec_optimizer.step()
+                    self.model_optimizer.step()
+                self.zero_grad()
+                self.optimizer_step += 1
+
+        self.on_fit_batch_end(batch, outputs, loss, should_step)
+        return loss.detach().cpu()
+
+    def evaluate_batch(self, batch, stage):
+        """Computations needed for validation/test batches"""
+        predictions = self.compute_forward(batch, stage=stage)
+        with torch.no_grad():
+            loss = self.compute_objectives(predictions, batch, stage=stage)
+        return loss.detach()
+
+    def on_stage_start(self, stage, epoch):
+        """Gets called at the beginning of each epoch"""
+        if stage != sb.Stage.TRAIN:
+            self.cer_metric = self.hparams.cer_computer()
+            self.wer_metric = self.hparams.error_rate_computer()
+
+    def on_stage_end(self, stage, stage_loss, epoch):
+        """Gets called at the end of an epoch."""
+        # Compute/store important stats
+        stage_stats = {"loss": stage_loss}
+        if stage == sb.Stage.TRAIN:
+            self.train_stats = stage_stats
+        else:
+            stage_stats["CER"] = self.cer_metric.summarize("error_rate")
+            stage_stats["WER"] = self.wer_metric.summarize("error_rate")
+
+        # Perform end-of-iteration things, like annealing, logging, etc.
+        if stage == sb.Stage.VALID:
+            old_lr_model, new_lr_model = self.hparams.lr_annealing_model(
+                stage_stats["loss"]
+            )
+            old_lr_wav2vec, new_lr_wav2vec = self.hparams.lr_annealing_wav2vec(
+                stage_stats["loss"]
+            )
+            sb.nnet.schedulers.update_learning_rate(
+                self.model_optimizer, new_lr_model
+            )
+            if not self.hparams.wav2vec2.freeze:
+                sb.nnet.schedulers.update_learning_rate(
+                    self.wav2vec_optimizer, new_lr_wav2vec
+                )
+            self.hparams.train_logger.log_stats(
+                stats_meta={
+                    "epoch": epoch,
+                    "lr_model": old_lr_model,
+                    "lr_wav2vec": old_lr_wav2vec,
+                },
+                train_stats=self.train_stats,
+                valid_stats=stage_stats,
+            )
+            self.checkpointer.save_and_keep_only(
+                meta={"WER": stage_stats["WER"]}, min_keys=["WER"],
+            )
+        elif stage == sb.Stage.TEST:
+            self.hparams.train_logger.log_stats(
+                stats_meta={"Epoch loaded": self.hparams.epoch_counter.current},
+                test_stats=stage_stats,
+            )
+            with open(self.hparams.wer_file, "w") as w:
+                self.wer_metric.write_stats(w)
+
+    def init_optimizers(self):
+        "Initializes the wav2vec2 optimizer and model optimizer"
+
+        # If the wav2vec encoder is unfrozen, we create the optimizer
+        if not self.hparams.wav2vec2.freeze:
+            self.wav2vec_optimizer = self.hparams.wav2vec_opt_class(
+                self.modules.wav2vec2.parameters()
+            )
+            if self.checkpointer is not None:
+                self.checkpointer.add_recoverable(
+                    "wav2vec_opt", self.wav2vec_optimizer
+                )
+
+        self.model_optimizer = self.hparams.model_opt_class(
+            self.hparams.model.parameters()
+        )
+
+        if self.checkpointer is not None:
+            self.checkpointer.add_recoverable("modelopt", self.model_optimizer)
+
+    def zero_grad(self, set_to_none=False):
+        if not self.hparams.wav2vec2.freeze:
+            self.wav2vec_optimizer.zero_grad(set_to_none)
+        self.model_optimizer.zero_grad(set_to_none)
+
+
+# Define custom data procedure
+def dataio_prepare(hparams, tokenizer):
+    """This function prepares the datasets to be used in the brain class.
+    It also defines the data processing pipeline through user-defined functions."""
+
+    # 1. Define datasets
+    data_folder = hparams["data_folder"]
+
+    train_data = sb.dataio.dataset.DynamicItemDataset.from_csv(
+        csv_path=hparams["train_csv"], replacements={"data_root": data_folder},
+    )
+
+    if hparams["sorting"] == "ascending":
+        # we sort training data to speed up training and get better results.
+        train_data = train_data.filtered_sorted(
+            sort_key="duration",
+            key_max_value={"duration": hparams["avoid_if_longer_than"]},
+        )
+        # when sorting do not shuffle in dataloader ! otherwise is pointless
+        hparams["dataloader_options"]["shuffle"] = False
+
+    elif hparams["sorting"] == "descending":
+        train_data = train_data.filtered_sorted(
+            sort_key="duration",
+            reverse=True,
+            key_max_value={"duration": hparams["avoid_if_longer_than"]},
+        )
+        # when sorting do not shuffle in dataloader ! otherwise is pointless
+        hparams["dataloader_options"]["shuffle"] = False
+
+    elif hparams["sorting"] == "random":
+        pass
+
+    else:
+        raise NotImplementedError(
+            "sorting must be random, ascending or descending"
+        )
+
+    valid_data = sb.dataio.dataset.DynamicItemDataset.from_csv(
+        csv_path=hparams["valid_csv"], replacements={"data_root": data_folder},
+    )
+    # We also sort the validation data so it is faster to validate
+    valid_data = valid_data.filtered_sorted(sort_key="duration")
+
+    test_data = sb.dataio.dataset.DynamicItemDataset.from_csv(
+        csv_path=hparams["test_csv"], replacements={"data_root": data_folder},
+    )
+
+    # We also sort the validation data so it is faster to validate
+    test_data = test_data.filtered_sorted(sort_key="duration")
+
+    datasets = [train_data, valid_data, test_data]
+
+    # 2. Define audio pipeline:
+    @sb.utils.data_pipeline.takes("wav")
+    @sb.utils.data_pipeline.provides("sig")
+    def audio_pipeline(wav):
+        info = torchaudio.info(wav)
+        sig = sb.dataio.dataio.read_audio(wav)
+        resampled = torchaudio.transforms.Resample(
+            info.sample_rate, hparams["sample_rate"],
+        )(sig)
+        return resampled
+
+    sb.dataio.dataset.add_dynamic_item(datasets, audio_pipeline)
+
+    # 3. Define text pipeline:
+    @sb.utils.data_pipeline.takes("wrd")
+    @sb.utils.data_pipeline.provides(
+        "tokens_list", "tokens_bos", "tokens_eos", "tokens"
+    )
+    def text_pipeline(wrd):
+        tokens_list = tokenizer.sp.encode_as_ids(wrd)
+        yield tokens_list
+        tokens_bos = torch.LongTensor([hparams["bos_index"]] + (tokens_list))
+        yield tokens_bos
+        tokens_eos = torch.LongTensor(tokens_list + [hparams["eos_index"]])
+        yield tokens_eos
+        tokens = torch.LongTensor(tokens_list)
+        yield tokens
+
+    sb.dataio.dataset.add_dynamic_item(datasets, text_pipeline)
+
+    # 4. Set output:
+    sb.dataio.dataset.set_output_keys(
+        datasets, ["id", "sig", "tokens_bos", "tokens_eos", "tokens"],
+    )
+    return train_data, valid_data, test_data
+
+
+if __name__ == "__main__":
+
+    # Load hyperparameters file with command-line overrides
+    hparams_file, run_opts, overrides = sb.parse_arguments(sys.argv[1:])
+    with open(hparams_file) as fin:
+        hparams = load_hyperpyyaml(fin, overrides)
+
+    # If --distributed_launch then
+    # create ddp_group with the right communication protocol
+    sb.utils.distributed.ddp_init_group(run_opts)
+
+    # Dataset preparation (parsing CommonVoice)
+    from common_voice_prepare import prepare_common_voice  # noqa
+
+    # Create experiment directory
+    sb.create_experiment_directory(
+        experiment_directory=hparams["output_folder"],
+        hyperparams_to_save=hparams_file,
+        overrides=overrides,
+    )
+
+    # Due to DDP, we do the preparation ONLY on the main python process
+    run_on_main(
+        prepare_common_voice,
+        kwargs={
+            "data_folder": hparams["data_folder"],
+            "save_folder": hparams["save_folder"],
+            "train_tsv_file": hparams["train_tsv_file"],
+            "dev_tsv_file": hparams["dev_tsv_file"],
+            "test_tsv_file": hparams["test_tsv_file"],
+            "accented_letters": hparams["accented_letters"],
+            "language": hparams["language"],
+            "skip_prep": hparams["skip_prep"],
+        },
+    )
+
+    # Defining tokenizer and loading it
+    tokenizer = SentencePiece(
+        model_dir=hparams["save_folder"],
+        vocab_size=hparams["output_neurons"],
+        annotation_train=hparams["train_csv"],
+        annotation_read="wrd",
+        model_type=hparams["token_type"],
+        character_coverage=hparams["character_coverage"],
+    )
+
+    # Create the datasets objects as well as tokenization and encoding :-D
+    train_data, valid_data, test_data = dataio_prepare(hparams, tokenizer)
+
+    # Trainer initialization
+    asr_brain = ASR(
+        modules=hparams["modules"],
+        hparams=hparams,
+        run_opts=run_opts,
+        checkpointer=hparams["checkpointer"],
+    )
+
+    # Adding objects to trainer.
+    asr_brain.tokenizer = tokenizer
+
+    # Training
+    asr_brain.fit(
+        asr_brain.hparams.epoch_counter,
+        train_data,
+        valid_data,
+        train_loader_kwargs=hparams["dataloader_options"],
+        valid_loader_kwargs=hparams["test_dataloader_options"],
+    )
+
+    # Test
+    asr_brain.hparams.wer_file = hparams["output_folder"] + "/wer_test.txt"
+    asr_brain.evaluate(
+        test_data,
+        min_key="WER",
+        test_loader_kwargs=hparams["test_dataloader_options"],
+    )

README.md

@@ -1,3 +1,21 @@
 ---
 license: apache-2.0
 ---
+# Tunisian Arabic ASR Model with wav2vec2 and code switching
+This repository provides all the necessary tools to perform automatic speech recognition from an end-to-end system pretrained on Tunisian arabic dialect. This model utilizes a code_switching approach and can process english , french and tunisian arabic
+## Performance
+the performance of the mode is : 
+| Release Version |WER (%) | CER (%) |
+|-----------------|---------|---------|
+| v1.0            |29.47    | 12.44   |
+## Pipeline
+The architecture comprises three components: 
+ * French ASR pretrained with wav2vec2 on french corporas
+ * English ASR pretrained with wav2vec2 on english corporas
+  * Custom Tunisian ASR pretrained using wav2vec on a tunisian arabic corpora
+  All three models will process the audio data. Subsequently, the resulting posteriorgrams will be combined and utilized as input for the Mixer, which will produce the final posteriorgrams.
+## Install
+```python
+pip install speechbrain transformers
+```
+

TunisianASR/README.md

@@ -0,0 +1,21 @@
+# Tunisian Arabic ASR Model with wav2vec2
+
+This repository provides all the necessary tools to perform automatic speech recognition from an end-to-end system pretrained on Tunisian arabic dialect
+
+## Performance
+the performance of the mode is : 
+| Release Version | |WER (%) | CER (%) |
+|-----------------|----|---------|---------|
+| v1.0            | Without LM |11.82    | 6.33   |
+## Dataset
+This ASR model was trained on :
+* TARIC : The corpus, named TARIC (Tunisian Arabic Railway Interaction Corpus) has a collection of audio recordings and transcriptions from dialogues in the Tunisian Railway Transport Network. - [Taric Corpus](https://aclanthology.org/L14-1385/) -
+* STAC :A corpus of spoken Tunisian Arabic - [STAC Corpus](https://www.researchgate.net/publication/307583782_Spoken_Tunisian_Arabic_Corpus_STAC_Transcription_and_Annotation)
+* IWSLT : A Tunisian conversational speech - [IWSLT Corpus](https://iwslt.org/2022/dialect)-
+* Tunspeech : Our custom dataset
+
+## Install
+```python
+pip install speechbrain transformers
+```
+

TunisianASR/outdomain.arpa

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:24654c1d236bb1bd367125131c847c4a734e69914eda71a6786964c20440d8fe
+size 324243244

TunisianASR/semi_wavlm_large_tunisian_ctc/1234/hyperparams.yaml

@@ -0,0 +1,194 @@
+# Generated 2023-09-08 from:
+# /gpfsdsstore/projects/rech/nou/uzn19yk/switched_code_tunisian/train/tunisian_asr/hparams/train_semi.yaml
+# yamllint disable
+# ################################
+# Model: wav2vec2 + DNN + CTC
+# Augmentation: SpecAugment
+# Authors: Titouan Parcollet 2021
+# ################################
+
+# Seed needs to be set at top of yaml, before objects with parameters are made
+seed: 1234
+__set_seed: !!python/object/apply:torch.manual_seed [1234]
+output_folder: results/semi_wavlm_large_tunisian_ctc/1234
+wer_file: results/semi_wavlm_large_tunisian_ctc/1234/wer.txt
+save_folder: results/semi_wavlm_large_tunisian_ctc/1234/save
+train_log: results/semi_wavlm_large_tunisian_ctc/1234/train_log.txt
+
+# URL for the biggest LeBenchmark wav2vec french.
+wav2vec2_folder: results/semi_wavlm_large_tunisian_ctc/1234/save/wav2vec2_checkpoint
+
+# Data files
+data_folder: /gpfsscratch/rech/nou/uzn19yk/tunisian_junk  # e.g, /localscratch/cv-corpus-5.1-2020-06-22/fr
+train_tsv_file: /gpfsscratch/rech/nou/uzn19yk/tunisian_junk/train.tsv # Standard CommonVoice .tsv files
+dev_tsv_file: /gpfsscratch/rech/nou/uzn19yk/tunisian_junk/dev.tsv # Standard CommonVoice .tsv files
+test_tsv_file: /gpfsscratch/rech/nou/uzn19yk/tunisian_junk/test.tsv # Standard CommonVoice .tsv files
+accented_letters: true
+language: fr # use 'it' for Italian, 'rw' for Kinyarwanda, 'en' for english
+train_csv: /gpfsscratch/rech/nou/uzn19yk/tunisian_csvs/good_final/train_enhanced.csv
+valid_csv: /gpfsscratch/rech/nou/uzn19yk/tunisian_csvs/good_final/dev.csv
+test_csv:
+- /gpfsscratch/rech/nou/uzn19yk/tunisian_csvs/full_annotation_test.csv
+- /gpfsscratch/rech/nou/uzn19yk/tunisian_csvs/good_final/iwslt_test.csv
+- /gpfsscratch/rech/nou/uzn19yk/tunisian_csvs/good_final/taric_test.csv
+
+skip_prep: true # Skip data preparation
+
+use_language_modelling: true
+ngram_lm_path: arpas/outdomain.arpa
+
+# We remove utterance slonger than 10s in the train/dev/test sets as
+# longer sentences certainly correspond to "open microphones".
+avoid_if_longer_than: 10.0
+avoid_if_shorter_than: 1.2
+
+
+# Training parameters
+number_of_epochs: 12
+lr: 1.0
+lr_wav2vec: 0.0001
+sorting: ascending
+auto_mix_prec: false
+sample_rate: 16000
+ckpt_interval_minutes: 30 # save checkpoint every N min
+
+# With data_parallel batch_size is split into N jobs
+# With DDP batch_size is multiplied by N jobs
+# Must be 6 per GPU to fit 16GB of VRAM
+batch_size: 10
+test_batch_size: 4
+
+dataloader_options:
+  batch_size: 10
+  num_workers: 6
+test_dataloader_options:
+  batch_size: 4
+  num_workers: 6
+
+# BPE parameters
+token_type: char  # ["unigram", "bpe", "char"]
+character_coverage: 1.0
+
+# Model parameters
+# activation: !name:torch.nn.LeakyReLU
+wav2vec_output_dim: 1024
+dnn_neurons: 1024
+freeze_wav2vec: false
+freeze_feature_extractor: true
+dropout: 0.15
+warmup_steps: 500 # The wav2vec 2 model isn't updated for this amount of steps
+
+# Outputs
+output_neurons: 40  # BPE size, index(blank/eos/bos) = 0
+
+# Decoding parameters
+# Be sure that the bos and eos index match with the BPEs ones
+blank_index: 0
+unk_index: 1
+
+#
+# Functions and classes
+#
+epoch_counter: &id007 !new:speechbrain.utils.epoch_loop.EpochCounter
+
+  limit: 12
+
+augmentation: !new:speechbrain.lobes.augment.TimeDomainSpecAugment
+  sample_rate: 16000
+  speeds: [95, 100, 105]
+
+enc: &id002 !new:speechbrain.nnet.containers.Sequential
+  input_shape: [null, null, 1024]
+  linear1: !name:speechbrain.nnet.linear.Linear
+    n_neurons: 1024
+    bias: true
+  bn1: !name:speechbrain.nnet.normalization.BatchNorm1d
+  activation: !new:torch.nn.LeakyReLU
+  drop: !new:torch.nn.Dropout
+    p: 0.15
+  linear2: !name:speechbrain.nnet.linear.Linear
+    n_neurons: 1024
+    bias: true
+  bn2: !name:speechbrain.nnet.normalization.BatchNorm1d
+  activation2: !new:torch.nn.LeakyReLU
+  drop2: !new:torch.nn.Dropout
+    p: 0.15
+  linear3: !name:speechbrain.nnet.linear.Linear
+    n_neurons: 1024
+    bias: true
+  bn3: !name:speechbrain.nnet.normalization.BatchNorm1d
+  activation3: !new:torch.nn.LeakyReLU
+
+wav2vec2: &id001 !new:speechbrain.lobes.models.huggingface_wav2vec.HuggingFaceWav2Vec2
+  source: /gpfsstore/rech/nou/uzn19yk/wavlm/
+  output_norm: false
+  freeze: false
+  freeze_feature_extractor: true
+  save_path: results/semi_wavlm_large_tunisian_ctc/1234/save/wav2vec2_checkpoint
+
+#####
+# Uncomment this block if you prefer to use a Fairseq pretrained model instead
+# of a HuggingFace one. Here, we provide an URL that is obtained from the
+# Fairseq github for the multilingual XLSR.
+#
+#wav2vec2_url: https://dl.fbaipublicfiles.com/fairseq/wav2vec/xlsr_53_56k.pt
+#wav2vec2: !new:speechbrain.lobes.models.fairseq_wav2vec.FairseqWav2Vec2
+#    pretrained_path: !ref <wav2vec2_url>
+#    output_norm: True
+#    freeze: False
+#    save_path: !ref <save_folder>/wav2vec2_checkpoint/model.pt
+#####
+
+
+ctc_lin: &id003 !new:speechbrain.nnet.linear.Linear
+
+  input_size: 1024
+  n_neurons: 40
+
+log_softmax: !new:speechbrain.nnet.activations.Softmax
+  apply_log: true
+
+ctc_cost: !name:speechbrain.nnet.losses.ctc_loss
+  blank_index: 0
+
+modules:
+  wav2vec2: *id001
+  enc: *id002
+  ctc_lin: *id003
+model: &id004 !new:torch.nn.ModuleList
+- [*id002, *id003]
+model_opt_class: !name:torch.optim.Adadelta
+  lr: 1.0
+  rho: 0.95
+  eps: 1.e-8
+
+wav2vec_opt_class: !name:torch.optim.Adam
+  lr: 0.0001
+
+lr_annealing_model: &id005 !new:speechbrain.nnet.schedulers.NewBobScheduler
+  initial_value: 1.0
+  improvement_threshold: 0.0025
+  annealing_factor: 0.8
+  patient: 0
+
+lr_annealing_wav2vec: &id006 !new:speechbrain.nnet.schedulers.NewBobScheduler
+  initial_value: 0.0001
+  improvement_threshold: 0.0025
+  annealing_factor: 0.9
+  patient: 0
+
+checkpointer: !new:speechbrain.utils.checkpoints.Checkpointer
+  checkpoints_dir: results/semi_wavlm_large_tunisian_ctc/1234/save
+  recoverables:
+    wav2vec2: *id001
+    model: *id004
+    scheduler_model: *id005
+    scheduler_wav2vec: *id006
+    counter: *id007
+train_logger: !new:speechbrain.utils.train_logger.FileTrainLogger
+  save_file: results/semi_wavlm_large_tunisian_ctc/1234/train_log.txt
+
+error_rate_computer: !name:speechbrain.utils.metric_stats.ErrorRateStats
+
+cer_computer: !name:speechbrain.utils.metric_stats.ErrorRateStats
+  split_tokens: true

TunisianASR/semi_wavlm_large_tunisian_ctc/1234/save/CKPT+2023-09-05+01-09-23+00/CKPT.yaml

@@ -0,0 +1,4 @@
+# yamllint disable
+WER: 27.83210816487267
+end-of-epoch: true
+unixtime: 1693868963.5220973

TunisianASR/semi_wavlm_large_tunisian_ctc/1234/save/CKPT+2023-09-05+01-09-23+00/brain.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3947a24e8dff5a14299b9cf2fe66ffb4d738cb88717de7f0cf7e8547a76e9776
+size 51

TunisianASR/semi_wavlm_large_tunisian_ctc/1234/save/CKPT+2023-09-05+01-09-23+00/counter.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6b51d431df5d7f141cbececcf79edf3dd861c3b4069f0b11661a3eefacbba918
+size 2

TunisianASR/semi_wavlm_large_tunisian_ctc/1234/save/CKPT+2023-09-05+01-09-23+00/dataloader-TRAIN.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b363886c229e536bd3c84e0c3e89312d70e00422578e076a62df1b45c9390793
+size 5

TunisianASR/semi_wavlm_large_tunisian_ctc/1234/save/CKPT+2023-09-05+01-09-23+00/model.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bc1dbeca1e1f1340b08d8ebea6e492f474708dddbbe8cabbcdde5ee9660704f2
+size 12814446

TunisianASR/semi_wavlm_large_tunisian_ctc/1234/save/CKPT+2023-09-05+01-09-23+00/modelopt.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3af1791eb9a5bfbfc087d2c10b94634df24cad3ac503ce9ba280a3ecc4737781
+size 25575663

TunisianASR/semi_wavlm_large_tunisian_ctc/1234/save/CKPT+2023-09-05+01-09-23+00/scheduler_model.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c275ab9245b440d1586f72058d9edaac1a2fb3e7a52712aa9a9ad022b99a1c0d
+size 639

TunisianASR/semi_wavlm_large_tunisian_ctc/1234/save/CKPT+2023-09-05+01-09-23+00/scheduler_wav2vec.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a88187f7882dc3e10c108f1b7abfbd819285b34bded4e88e91c4ff699c1bb5d2
+size 643

TunisianASR/semi_wavlm_large_tunisian_ctc/1234/save/CKPT+2023-09-05+01-09-23+00/wav2vec2.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:788267bd25ef37623715fa21a975090e5e316fff05971375cd3f62e5160f0743
+size 1262005979

TunisianASR/semi_wavlm_large_tunisian_ctc/1234/save/CKPT+2023-09-05+01-09-23+00/wav2vec_opt.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:efa967fdd8067be7d88c18cd197980c9c91f344a3dff2b2518b8381c49f28b1e
+size 2490361859

TunisianASR/semi_wavlm_large_tunisian_ctc/1234/save/label_encoder.txt

@@ -0,0 +1,44 @@
+'ب' => 38
+'ا' => 1
+'ه' => 2
+'ي' => 3
+'و' => 4
+'ن' => 5
+'أ' => 6
+' ' => 7
+'م' => 8
+'ش' => 9
+'ل' => 10
+'س' => 11
+'ت' => 12
+'د' => 13
+'ر' => 14
+'ى' => 15
+'ح' => 16
+'ط' => 17
+'ع' => 18
+'ك' => 19
+'ف' => 20
+'ق' => 21
+'آ' => 22
+'ة' => 23
+'ج' => 24
+'ض' => 25
+'ز' => 26
+'ص' => 27
+'إ' => 28
+'ث' => 29
+'خ' => 30
+'ڨ' => 31
+'ذ' => 32
+'ظ' => 33
+'ء' => 34
+'غ' => 35
+'ئ' => 36
+'ؤ' => 37
+'<blank>' => 0
+1 => 39
+================
+'starting_index' => 0
+'unk_label' => 1
+'blank_label' => '<blank>'

TunisianASR/train_semi.yaml

@@ -0,0 +1,175 @@
+# ################################
+# Model: wav2vec2 + DNN + CTC
+# Augmentation: SpecAugment
+# Authors: Titouan Parcollet 2021
+# ################################
+
+# Seed needs to be set at top of yaml, before objects with parameters are made
+seed: 1234
+__set_seed: !!python/object/apply:torch.manual_seed [!ref <seed>]
+output_folder: !ref semi_wavlm_large_tunisian_ctc/<seed>
+wer_file: !ref <output_folder>/wer.txt
+save_folder: !ref <output_folder>/save
+train_log: !ref <output_folder>/train_log.txt
+
+# URL for the biggest LeBenchmark wav2vec french.
+wav2vec2_folder: !ref <save_folder>/wav2vec2_checkpoint
+
+# Data files
+data_folder: /path/to/data  # e.g, /localscratch/cv-corpus-5.1-2020-06-22/fr
+train_tsv_file: !ref <data_folder>/train.tsv  # Standard CommonVoice .tsv files
+dev_tsv_file: !ref <data_folder>/dev.tsv  # Standard CommonVoice .tsv files
+test_tsv_file: !ref <data_folder>/test.tsv  # Standard CommonVoice .tsv files
+accented_letters: True
+language: fr # use 'it' for Italian, 'rw' for Kinyarwanda, 'en' for english
+test_csv: 
+    - /path/to/test_data
+
+skip_prep: True # Skip data preparation
+
+use_language_modelling: True
+ngram_lm_path: outdomain.arpa
+
+# We remove utterance slonger than 10s in the train/dev/test sets as
+# longer sentences certainly correspond to "open microphones".
+avoid_if_longer_than: 10.0
+avoid_if_shorter_than: 1.2
+
+
+# Training parameters
+number_of_epochs: 12
+lr: 1.0
+lr_wav2vec: 0.0001
+sorting: ascending
+auto_mix_prec: False
+sample_rate: 16000
+ckpt_interval_minutes: 30 # save checkpoint every N min
+
+# With data_parallel batch_size is split into N jobs
+# With DDP batch_size is multiplied by N jobs
+# Must be 6 per GPU to fit 16GB of VRAM
+batch_size: 10
+test_batch_size: 4
+
+dataloader_options:
+    batch_size: !ref <batch_size>
+    num_workers: 6
+test_dataloader_options:
+    batch_size: !ref <test_batch_size>
+    num_workers: 6
+
+# BPE parameters
+token_type: char  # ["unigram", "bpe", "char"]
+character_coverage: 1.0
+
+# Model parameters
+# activation: !name:torch.nn.LeakyReLU
+wav2vec_output_dim: 1024
+dnn_neurons: 1024
+freeze_wav2vec: False
+freeze_feature_extractor: True
+dropout: 0.15
+warmup_steps: 500 # The wav2vec 2 model isn't updated for this amount of steps
+
+# Outputs
+output_neurons: 40  # BPE size, index(blank/eos/bos) = 0
+
+# Decoding parameters
+# Be sure that the bos and eos index match with the BPEs ones
+blank_index: 0
+unk_index: 1
+
+#
+# Functions and classes
+#
+epoch_counter: !new:speechbrain.utils.epoch_loop.EpochCounter
+    limit: !ref <number_of_epochs>
+
+augmentation: !new:speechbrain.lobes.augment.TimeDomainSpecAugment
+    sample_rate: !ref <sample_rate>
+    speeds: [95, 100, 105]
+
+enc: !new:speechbrain.nnet.containers.Sequential
+    input_shape: [null, null, !ref <wav2vec_output_dim>]
+    linear1: !name:speechbrain.nnet.linear.Linear
+        n_neurons: !ref <dnn_neurons>
+        bias: True
+    bn1: !name:speechbrain.nnet.normalization.BatchNorm1d
+    activation: !new:torch.nn.LeakyReLU
+    drop: !new:torch.nn.Dropout
+        p: !ref <dropout>
+    linear2: !name:speechbrain.nnet.linear.Linear
+        n_neurons: !ref <dnn_neurons>
+        bias: True
+    bn2: !name:speechbrain.nnet.normalization.BatchNorm1d
+    activation2: !new:torch.nn.LeakyReLU
+    drop2: !new:torch.nn.Dropout
+        p: !ref <dropout>
+    linear3: !name:speechbrain.nnet.linear.Linear
+        n_neurons: !ref <dnn_neurons>
+        bias: True
+    bn3: !name:speechbrain.nnet.normalization.BatchNorm1d
+    activation3: !new:torch.nn.LeakyReLU
+
+wav2vec2: !new:speechbrain.lobes.models.huggingface_wav2vec.HuggingFaceWav2Vec2
+    source: /gpfsstore/rech/nou/uzn19yk/wavlm/
+    output_norm: False
+    freeze: !ref <freeze_wav2vec>
+    freeze_feature_extractor: !ref <freeze_feature_extractor>
+    save_path: !ref <wav2vec2_folder>
+
+
+ctc_lin: !new:speechbrain.nnet.linear.Linear
+    input_size: !ref <dnn_neurons>
+    n_neurons: !ref <output_neurons>
+
+log_softmax: !new:speechbrain.nnet.activations.Softmax
+    apply_log: True
+
+ctc_cost: !name:speechbrain.nnet.losses.ctc_loss
+    blank_index: !ref <blank_index>
+
+modules:
+    wav2vec2: !ref <wav2vec2>
+    enc: !ref <enc>
+    ctc_lin: !ref <ctc_lin>
+
+model: !new:torch.nn.ModuleList
+    - [!ref <enc>, !ref <ctc_lin>]
+
+model_opt_class: !name:torch.optim.Adadelta
+    lr: !ref <lr>
+    rho: 0.95
+    eps: 1.e-8
+
+wav2vec_opt_class: !name:torch.optim.Adam
+    lr: !ref <lr_wav2vec>
+
+lr_annealing_model: !new:speechbrain.nnet.schedulers.NewBobScheduler
+    initial_value: !ref <lr>
+    improvement_threshold: 0.0025
+    annealing_factor: 0.8
+    patient: 0
+
+lr_annealing_wav2vec: !new:speechbrain.nnet.schedulers.NewBobScheduler
+    initial_value: !ref <lr_wav2vec>
+    improvement_threshold: 0.0025
+    annealing_factor: 0.9
+    patient: 0
+
+checkpointer: !new:speechbrain.utils.checkpoints.Checkpointer
+    checkpoints_dir: !ref <save_folder>
+    recoverables:
+        wav2vec2: !ref <wav2vec2>
+        model: !ref <model>
+        scheduler_model: !ref <lr_annealing_model>
+        scheduler_wav2vec: !ref <lr_annealing_wav2vec>
+        counter: !ref <epoch_counter>
+
+train_logger: !new:speechbrain.utils.train_logger.FileTrainLogger
+    save_file: !ref <train_log>
+
+error_rate_computer: !name:speechbrain.utils.metric_stats.ErrorRateStats
+
+cer_computer: !name:speechbrain.utils.metric_stats.ErrorRateStats
+    split_tokens: True

TunisianASR/train_with_wavlm.py

@@ -0,0 +1,399 @@
+#!/usr/bin/env python3
+import sys
+import torch
+import logging
+import speechbrain as sb
+from pathlib import Path
+import os
+import torchaudio
+from hyperpyyaml import load_hyperpyyaml
+from speechbrain.tokenizers.SentencePiece import SentencePiece
+from speechbrain.utils.data_utils import undo_padding
+from speechbrain.utils.distributed import run_on_main
+
+"""Recipe for training a sequence-to-sequence ASR system with CommonVoice.
+The system employs a wav2vec2 encoder and a CTC decoder.
+Decoding is performed with greedy decoding (will be extended to beam search).
+
+To run this recipe, do the following:
+> python train_with_wav2vec2.py hparams/train_with_wav2vec2.yaml
+
+With the default hyperparameters, the system employs a pretrained wav2vec2 encoder.
+The wav2vec2 model is pretrained following the model given in the hprams file.
+It may be dependent on the language.
+
+The neural network is trained with CTC on sub-word units estimated with
+Byte Pairwise Encoding (BPE).
+
+The experiment file is flexible enough to support a large variety of
+different systems. By properly changing the parameter files, you can try
+different encoders, decoders, tokens (e.g, characters instead of BPE),
+training languages (all CommonVoice languages), and many
+other possible variations.
+
+Authors
+ * Titouan Parcollet 2021
+"""
+
+logger = logging.getLogger(__name__)
+
+
+# Define training procedure
+class ASR(sb.core.Brain):
+    def compute_forward(self, batch, stage):
+        """Forward computations from the waveform batches to the output probabilities."""
+
+        batch = batch.to(self.device)
+        wavs, wav_lens = batch.sig
+        wavs, wav_lens = wavs.to(self.device), wav_lens.to(self.device)
+        if stage == sb.Stage.TRAIN:
+            if hasattr(self.hparams, "augmentation"):
+                wavs = self.hparams.augmentation(wavs, wav_lens)
+
+        # Forward pass
+        feats = self.modules.wav2vec2(wavs, wav_lens)
+        x = self.modules.enc(feats)
+        logits = self.modules.ctc_lin(x)
+        p_ctc = self.hparams.log_softmax(logits)
+
+        return p_ctc, wav_lens
+
+    def compute_objectives(self, predictions, batch, stage):
+        """Computes the loss (CTC) given predictions and targets."""
+
+        p_ctc, wav_lens = predictions
+
+        ids = batch.id
+        tokens, tokens_lens = batch.tokens
+
+        loss = self.hparams.ctc_cost(p_ctc, tokens, wav_lens, tokens_lens)
+
+        if stage != sb.Stage.TRAIN:
+            predicted_tokens = sb.decoders.ctc_greedy_decode(
+                p_ctc, wav_lens, blank_id=self.hparams.blank_index
+            )
+            # Decode token terms to words
+            if self.hparams.use_language_modelling:
+                predicted_words = []
+                for logs in p_ctc:
+                    text = decoder.decode(logs.detach().cpu().numpy())
+                    predicted_words.append(text.split(" "))
+            else:
+                predicted_words = [
+                    "".join(self.tokenizer.decode_ndim(utt_seq)).split(" ")
+                    for utt_seq in predicted_tokens
+                ]
+            # Convert indices to words
+            target_words = [wrd.split(" ") for wrd in batch.wrd]
+
+            self.wer_metric.append(ids, predicted_words, target_words)
+            self.cer_metric.append(ids, predicted_words, target_words)
+
+        return loss
+
+    def fit_batch(self, batch):
+        """Train the parameters given a single batch in input"""
+        should_step = self.step % self.grad_accumulation_factor == 0
+        # Managing automatic mixed precision
+        # TOFIX: CTC fine-tuning currently is unstable
+        # This is certainly due to CTC being done in fp16 instead of fp32
+        if self.auto_mix_prec:
+            with torch.cuda.amp.autocast():
+                with self.no_sync():
+                    outputs = self.compute_forward(batch, sb.Stage.TRAIN)
+                loss = self.compute_objectives(outputs, batch, sb.Stage.TRAIN)
+            with self.no_sync(not should_step):
+                self.scaler.scale(
+                    loss / self.grad_accumulation_factor
+                ).backward()
+            if should_step:
+
+                if not self.hparams.wav2vec2.freeze:
+                    self.scaler.unscale_(self.wav2vec_optimizer)
+                self.scaler.unscale_(self.model_optimizer)
+                if self.check_gradients(loss):
+                    if not self.hparams.wav2vec2.freeze:
+                        if self.optimizer_step >= self.hparams.warmup_steps:
+                            self.scaler.step(self.wav2vec_optimizer)
+                    self.scaler.step(self.model_optimizer)
+                self.scaler.update()
+                self.zero_grad()
+                self.optimizer_step += 1
+        else:
+            # This is mandatory because HF models have a weird behavior with DDP
+            # on the forward pass
+            with self.no_sync():
+                outputs = self.compute_forward(batch, sb.Stage.TRAIN)
+
+            loss = self.compute_objectives(outputs, batch, sb.Stage.TRAIN)
+
+            with self.no_sync(not should_step):
+                (loss / self.grad_accumulation_factor).backward()
+            if should_step:
+                if self.check_gradients(loss):
+                    if not self.hparams.wav2vec2.freeze:
+                        if self.optimizer_step >= self.hparams.warmup_steps:
+                            self.wav2vec_optimizer.step()
+                    self.model_optimizer.step()
+                self.zero_grad()
+                self.optimizer_step += 1
+
+        self.on_fit_batch_end(batch, outputs, loss, should_step)
+        return loss.detach().cpu()
+
+    def evaluate_batch(self, batch, stage):
+        """Computations needed for validation/test batches"""
+        predictions = self.compute_forward(batch, stage=stage)
+        with torch.no_grad():
+            loss = self.compute_objectives(predictions, batch, stage=stage)
+        return loss.detach()
+
+    def on_stage_start(self, stage, epoch):
+        """Gets called at the beginning of each epoch"""
+        if stage != sb.Stage.TRAIN:
+            self.cer_metric = self.hparams.cer_computer()
+            self.wer_metric = self.hparams.error_rate_computer()
+
+    def on_stage_end(self, stage, stage_loss, epoch):
+        """Gets called at the end of an epoch."""
+        # Compute/store important stats
+        stage_stats = {"loss": stage_loss}
+        if stage == sb.Stage.TRAIN:
+            self.train_stats = stage_stats
+        else:
+            stage_stats["CER"] = self.cer_metric.summarize("error_rate")
+            stage_stats["WER"] = self.wer_metric.summarize("error_rate")
+
+        # Perform end-of-iteration things, like annealing, logging, etc.
+        if stage == sb.Stage.VALID:
+            old_lr_model, new_lr_model = self.hparams.lr_annealing_model(
+                stage_stats["loss"]
+            )
+            old_lr_wav2vec, new_lr_wav2vec = self.hparams.lr_annealing_wav2vec(
+                stage_stats["loss"]
+            )
+            sb.nnet.schedulers.update_learning_rate(
+                self.model_optimizer, new_lr_model
+            )
+            if not self.hparams.wav2vec2.freeze:
+                sb.nnet.schedulers.update_learning_rate(
+                    self.wav2vec_optimizer, new_lr_wav2vec
+                )
+            self.hparams.train_logger.log_stats(
+                stats_meta={
+                    "epoch": epoch,
+                    "lr_model": old_lr_model,
+                    "lr_wav2vec": old_lr_wav2vec,
+                },
+                train_stats=self.train_stats,
+                valid_stats=stage_stats,
+            )
+            self.checkpointer.save_and_keep_only(
+                meta={"WER": stage_stats["WER"]}, min_keys=["WER"],
+            )
+        elif stage == sb.Stage.TEST:
+            self.hparams.train_logger.log_stats(
+                stats_meta={"Epoch loaded": self.hparams.epoch_counter.current},
+                test_stats=stage_stats,
+            )
+            with open(self.hparams.wer_file, "w") as w:
+                self.wer_metric.write_stats(w)
+
+    def init_optimizers(self):
+        "Initializes the wav2vec2 optimizer and model optimizer"
+
+        # If the wav2vec encoder is unfrozen, we create the optimizer
+        if not self.hparams.wav2vec2.freeze:
+            self.wav2vec_optimizer = self.hparams.wav2vec_opt_class(
+                self.modules.wav2vec2.parameters()
+            )
+            if self.checkpointer is not None:
+                self.checkpointer.add_recoverable(
+                    "wav2vec_opt", self.wav2vec_optimizer
+                )
+
+        self.model_optimizer = self.hparams.model_opt_class(
+            self.hparams.model.parameters()
+        )
+
+        if self.checkpointer is not None:
+            self.checkpointer.add_recoverable("modelopt", self.model_optimizer)
+
+    def zero_grad(self, set_to_none=False):
+        if not self.hparams.wav2vec2.freeze:
+            self.wav2vec_optimizer.zero_grad(set_to_none)
+        self.model_optimizer.zero_grad(set_to_none)
+
+
+# Define custom data procedure
+def dataio_prepare(hparams):
+    """This function prepares the datasets to be used in the brain class.
+    It also defines the data processing pipeline through user-defined functions."""
+
+    # 1. Define datasets
+    data_folder = hparams["data_folder"]
+
+    train_data = sb.dataio.dataset.DynamicItemDataset.from_csv(
+        csv_path=hparams["train_csv"], replacements={"data_root": data_folder},
+    )
+
+    if hparams["sorting"] == "ascending":
+        # we sort training data to speed up training and get better results.
+        train_data = train_data.filtered_sorted(
+            sort_key="duration",
+            key_max_value={"duration": hparams["avoid_if_longer_than"]},
+        )
+        # when sorting do not shuffle in dataloader ! otherwise is pointless
+        hparams["dataloader_options"]["shuffle"] = False
+
+    elif hparams["sorting"] == "descending":
+        train_data = train_data.filtered_sorted(
+            sort_key="duration",
+            reverse=True,
+            key_max_value={"duration": hparams["avoid_if_longer_than"]},
+        )
+        # when sorting do not shuffle in dataloader ! otherwise is pointless
+        hparams["dataloader_options"]["shuffle"] = False
+
+    elif hparams["sorting"] == "random":
+        pass
+
+    else:
+        raise NotImplementedError(
+            "sorting must be random, ascending or descending"
+        )
+
+    valid_data = sb.dataio.dataset.DynamicItemDataset.from_csv(
+        csv_path=hparams["valid_csv"], replacements={"data_root": data_folder},
+    )
+    # We also sort the validation data so it is faster to validate
+    valid_data = valid_data.filtered_sorted(sort_key="duration")
+    test_datasets = {}
+    for csv_file in hparams["test_csv"]:
+        name = Path(csv_file).stem
+        test_datasets[name] = sb.dataio.dataset.DynamicItemDataset.from_csv(
+            csv_path=csv_file, replacements={"data_root": data_folder}
+        )
+        test_datasets[name] = test_datasets[name].filtered_sorted(
+            sort_key="duration"
+        )
+
+    datasets = [train_data, valid_data] + [i for k, i in test_datasets.items()]
+
+
+    # 2. Define audio pipeline:
+    @sb.utils.data_pipeline.takes("wav")
+    @sb.utils.data_pipeline.provides("sig")
+    def audio_pipeline(wav):
+        info = torchaudio.info(wav)
+        sig = sb.dataio.dataio.read_audio(wav)
+        resampled = torchaudio.transforms.Resample(
+            info.sample_rate, hparams["sample_rate"],
+        )(sig)
+        return resampled
+
+    sb.dataio.dataset.add_dynamic_item(datasets, audio_pipeline)
+    label_encoder = sb.dataio.encoder.CTCTextEncoder()
+
+    # 3. Define text pipeline:
+    @sb.utils.data_pipeline.takes("wrd")
+    @sb.utils.data_pipeline.provides(
+        "wrd", "char_list", "tokens_list", "tokens"
+    )
+    def text_pipeline(wrd):
+        yield wrd
+        char_list = list(wrd)
+        yield char_list
+        tokens_list = label_encoder.encode_sequence(char_list)
+        yield tokens_list
+        tokens = torch.LongTensor(tokens_list)
+        yield tokens
+
+    sb.dataio.dataset.add_dynamic_item(datasets, text_pipeline)
+    lab_enc_file = os.path.join(hparams["save_folder"], "label_encoder.txt")
+    special_labels = {
+        "blank_label": hparams["blank_index"],
+        "unk_label": hparams["unk_index"]
+    }
+    label_encoder.load_or_create(
+        path=lab_enc_file,
+        from_didatasets=[train_data],
+        output_key="char_list",
+        special_labels=special_labels,
+        sequence_input=True,
+    )
+
+    # 4. Set output:
+    sb.dataio.dataset.set_output_keys(
+        datasets, ["id", "sig", "wrd", "char_list", "tokens"],
+    )
+    return train_data, valid_data,test_datasets, label_encoder
+
+
+if __name__ == "__main__":
+
+    # Load hyperparameters file with command-line overrides
+    hparams_file, run_opts, overrides = sb.parse_arguments(sys.argv[1:])
+    with open(hparams_file) as fin:
+        hparams = load_hyperpyyaml(fin, overrides)
+
+    # If --distributed_launch then
+    # create ddp_group with the right communication protocol
+    sb.utils.distributed.ddp_init_group(run_opts)
+
+
+    # Create experiment directory
+    sb.create_experiment_directory(
+        experiment_directory=hparams["output_folder"],
+        hyperparams_to_save=hparams_file,
+        overrides=overrides,
+    )
+
+    # Due to DDP, we do the preparation ONLY on the main python process
+    # Defining tokenizer and loading it
+    # Create the datasets objects as well as tokenization and encoding :-D
+    train_data, valid_data, test_datasets, label_encoder = dataio_prepare(hparams)
+    if hparams["use_language_modelling"]:
+        print("using langauge_modeeling")
+        from pyctcdecode import build_ctcdecoder
+        ind2lab = label_encoder.ind2lab
+        print(ind2lab)
+        labels = [ind2lab[x] for x in range(len(ind2lab))]
+        labels = [""] + labels[1:-1] + ["1"] 
+        # Replace the <blank> token with a blank character, needed for PyCTCdecode
+        print(labels)
+        decoder = build_ctcdecoder(
+            labels,
+            kenlm_model_path=hparams["ngram_lm_path"],  # .arpa or .bin
+            alpha=0.5,  # Default by KenLM
+            beta=1.0,  # Default by KenLM
+        )
+    # Trainer initialization
+    asr_brain = ASR(
+        modules=hparams["modules"],
+        hparams=hparams,
+        run_opts=run_opts,
+        checkpointer=hparams["checkpointer"],
+    )
+
+    # Adding objects to trainer.
+    asr_brain.tokenizer = label_encoder
+
+    # Training
+    asr_brain.fit(
+        asr_brain.hparams.epoch_counter,
+        train_data,
+        valid_data,
+        train_loader_kwargs=hparams["dataloader_options"],
+        valid_loader_kwargs=hparams["test_dataloader_options"],
+    )
+
+    # Test
+    for k in test_datasets.keys():  # keys are test_clean, test_other etc
+        asr_brain.hparams.wer_file = os.path.join(
+            hparams["output_folder"], "wer_{}.txt".format(k)
+        )
+        asr_brain.evaluate(
+            test_datasets[k], test_loader_kwargs=hparams["test_dataloader_options"]
+        )
+

arpas/everything.arpa

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7bada7d41f63b1e5fd661ba66bccdfa93c3e5c391038ac6e52615a42ec0e0174
+size 345991397

asr-wav2vec2-commonvoice-fr/README.md

@@ -0,0 +1,130 @@
+---
+language:
+- fr
+thumbnail: null
+pipeline_tag: automatic-speech-recognition
+tags:
+- CTC
+- pytorch
+- speechbrain
+- Transformer
+- hf-asr-leaderboard
+license: apache-2.0
+datasets:
+- commonvoice
+metrics:
+- wer
+- cer
+model-index:
+- name: asr-wav2vec2-commonvoice-fr
+  results:
+  - task:
+      name: Automatic Speech Recognition
+      type: automatic-speech-recognition
+    dataset:
+      name: CommonVoice 6.1 (French)
+      type: mozilla-foundation/common_voice_6_1
+      config: fr
+      split: test
+      args:
+        language: fr
+    metrics:
+    - name: Test WER
+      type: wer
+      value: '9.96'
+---
+
+<iframe src="https://ghbtns.com/github-btn.html?user=speechbrain&repo=speechbrain&type=star&count=true&size=large&v=2" frameborder="0" scrolling="0" width="170" height="30" title="GitHub"></iframe>
+<br/><br/>
+
+# wav2vec 2.0 with CTC/Attention trained on CommonVoice French (No LM)
+
+This repository provides all the necessary tools to perform automatic speech
+recognition from an end-to-end system pretrained on CommonVoice (French Language) within
+SpeechBrain. For a better experience, we encourage you to learn more about
+[SpeechBrain](https://speechbrain.github.io).
+
+The performance of the model is the following:
+
+| Release | Test CER | Test WER | GPUs |
+|:-------------:|:--------------:|:--------------:| :--------:|
+| 24-08-21 | 3.19 | 9.96 | 2xV100 32GB |
+
+## Pipeline description
+
+This ASR system is composed of 2 different but linked blocks:
+- Tokenizer (unigram) that transforms words into subword units and trained with
+the train transcriptions (train.tsv) of CommonVoice (FR).
+- Acoustic model (wav2vec2.0 + CTC). A pretrained wav2vec 2.0 model ([LeBenchmark/wav2vec2-FR-7K-large](https://huggingface.co/LeBenchmark/wav2vec2-FR-7K-large)) is combined with two DNN layers and finetuned on CommonVoice FR.
+The obtained final acoustic representation is given to the CTC greedy decoder.
+
+The system is trained with recordings sampled at 16kHz (single channel).
+The code will automatically normalize your audio (i.e., resampling + mono channel selection) when calling *transcribe_file* if needed.
+
+## Install SpeechBrain
+
+First of all, please install tranformers and SpeechBrain with the following command:
+
+```
+pip install speechbrain transformers
+```
+
+Please notice that we encourage you to read our tutorials and learn more about
+[SpeechBrain](https://speechbrain.github.io).
+
+### Transcribing your own audio files (in French)
+
+```python
+from speechbrain.pretrained import EncoderASR
+
+asr_model = EncoderASR.from_hparams(source="speechbrain/asr-wav2vec2-commonvoice-fr", savedir="pretrained_models/asr-wav2vec2-commonvoice-fr")
+asr_model.transcribe_file('speechbrain/asr-wav2vec2-commonvoice-fr/example-fr.wav')
+
+```
+### Inference on GPU
+To perform inference on the GPU, add  `run_opts={"device":"cuda"}`  when calling the `from_hparams` method.
+
+### Training
+The model was trained with SpeechBrain.
+To train it from scratch follow these steps:
+1. Clone SpeechBrain:
+```bash
+git clone https://github.com/speechbrain/speechbrain/
+```
+2. Install it:
+```bash
+cd speechbrain
+pip install -r requirements.txt
+pip install -e .
+```
+
+3. Run Training:
+```bash
+cd recipes/CommonVoice/ASR/CTC/
+python train_with_wav2vec.py hparams/train_fr_with_wav2vec.yaml --data_folder=your_data_folder
+```
+
+You can find our training results (models, logs, etc) [here](https://drive.google.com/drive/folders/1T9DfdZwcNI9CURxhLCi8GA5JVz8adiY8?usp=sharing).
+
+### Limitations
+The SpeechBrain team does not provide any warranty on the performance achieved by this model when used on other datasets.
+
+#### Referencing SpeechBrain
+
+```
+@misc{SB2021,
+    author = {Ravanelli, Mirco and Parcollet, Titouan and Rouhe, Aku and Plantinga, Peter and Rastorgueva, Elena and Lugosch, Loren and Dawalatabad, Nauman and Ju-Chieh, Chou and Heba, Abdel and Grondin, Francois and Aris, William and Liao, Chien-Feng and Cornell, Samuele and Yeh, Sung-Lin and Na, Hwidong and Gao, Yan and Fu, Szu-Wei and Subakan, Cem and De Mori, Renato and Bengio, Yoshua },
+    title = {SpeechBrain},
+    year = {2021},
+    publisher = {GitHub},
+    journal = {GitHub repository},
+    howpublished = {\\\\url{https://github.com/speechbrain/speechbrain}},
+  }
+```
+
+#### About SpeechBrain
+SpeechBrain is an open-source and all-in-one speech toolkit. It is designed to be simple, extremely flexible, and user-friendly. Competitive or state-of-the-art performance is obtained in various domains.
+
+Website: https://speechbrain.github.io/
+
+GitHub: https://github.com/speechbrain/speechbrain

asr-wav2vec2-commonvoice-fr/asr.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:64ba475ed7be735d4ac054c2d537f22251b80f6ecb65cb04217eb0d1ed50a143
+size 12963902