Add wav2vec model with LM

README.md

@@ -0,0 +1,9 @@
+---
+language: 
+  - es
+tags:
+- es
+- robust-speech-event
+---
+
+# wav2vec2-xls-r-300m-36-tokens-es

added_tokens.json

@@ -0,0 +1 @@
+{"<s>": 36, "</s>": 37}

alphabet.json

@@ -0,0 +1 @@
+{"labels": [" ", "a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l", "m", "n", "o", "p", "q", "r", "s", "t", "u", "v", "w", "x", "y", "z", "\u00e1", "\u00e9", "\u00ed", "\u00f1", "\u00f3", "\u00fa", "\u00fc", "\u2047", "", "<s>", "</s>"], "is_bpe": false}

common_voice_es_test_eval_results.txt

@@ -0,0 +1,2 @@
+WER: 0.15404152388631323
+CER: 0.04559264378545272

config.json

@@ -0,0 +1,108 @@
+{
+  "_name_or_path": "facebook/wav2vec2-xls-r-300m",
+  "activation_dropout": 0.0,
+  "adapter_kernel_size": 3,
+  "adapter_stride": 2,
+  "add_adapter": false,
+  "apply_spec_augment": true,
+  "architectures": [
+    "Wav2Vec2ForCTC"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "classifier_proj_size": 256,
+  "codevector_dim": 768,
+  "contrastive_logits_temperature": 0.1,
+  "conv_bias": true,
+  "conv_dim": [
+    512,
+    512,
+    512,
+    512,
+    512,
+    512,
+    512
+  ],
+  "conv_kernel": [
+    10,
+    3,
+    3,
+    3,
+    3,
+    2,
+    2
+  ],
+  "conv_stride": [
+    5,
+    2,
+    2,
+    2,
+    2,
+    2,
+    2
+  ],
+  "ctc_loss_reduction": "mean",
+  "ctc_zero_infinity": false,
+  "diversity_loss_weight": 0.1,
+  "do_stable_layer_norm": true,
+  "eos_token_id": 2,
+  "feat_extract_activation": "gelu",
+  "feat_extract_dropout": 0.0,
+  "feat_extract_norm": "layer",
+  "feat_proj_dropout": 0.0,
+  "feat_quantizer_dropout": 0.0,
+  "final_dropout": 0.0,
+  "gradient_checkpointing": false,
+  "hidden_act": "gelu",
+  "hidden_dropout": 0.0,
+  "hidden_size": 1024,
+  "initializer_range": 0.02,
+  "intermediate_size": 4096,
+  "layer_norm_eps": 1e-05,
+  "layerdrop": 0.0,
+  "mask_feature_length": 10,
+  "mask_feature_min_masks": 0,
+  "mask_feature_prob": 0.0,
+  "mask_time_length": 10,
+  "mask_time_min_masks": 2,
+  "mask_time_prob": 0.05,
+  "model_type": "wav2vec2",
+  "num_adapter_layers": 3,
+  "num_attention_heads": 16,
+  "num_codevector_groups": 2,
+  "num_codevectors_per_group": 320,
+  "num_conv_pos_embedding_groups": 16,
+  "num_conv_pos_embeddings": 128,
+  "num_feat_extract_layers": 7,
+  "num_hidden_layers": 24,
+  "num_negatives": 100,
+  "output_hidden_size": 1024,
+  "pad_token_id": 35,
+  "proj_codevector_dim": 768,
+  "tdnn_dilation": [
+    1,
+    2,
+    3,
+    1,
+    1
+  ],
+  "tdnn_dim": [
+    512,
+    512,
+    512,
+    512,
+    1500
+  ],
+  "tdnn_kernel": [
+    5,
+    3,
+    3,
+    1,
+    1
+  ],
+  "torch_dtype": "float32",
+  "transformers_version": "4.16.0.dev0",
+  "use_weighted_layer_sum": false,
+  "vocab_size": 38,
+  "xvector_output_dim": 512
+}

eval.py

@@ -0,0 +1,131 @@
+#!/usr/bin/env python3
+from datasets import load_dataset, load_metric, Audio, Dataset
+from transformers import pipeline, AutoFeatureExtractor
+import re
+import argparse
+import unicodedata
+from typing import Dict
+
+
+def log_results(result: Dataset, args: Dict[str, str]):
+    """ DO NOT CHANGE. This function computes and logs the result metrics. """
+
+    log_outputs = args.log_outputs
+    dataset_id = "_".join(args.dataset.split("/") + [args.config, args.split])
+
+    # load metric
+    wer = load_metric("wer")
+    cer = load_metric("cer")
+
+    # compute metrics
+    wer_result = wer.compute(references=result["target"], predictions=result["prediction"])
+    cer_result = cer.compute(references=result["target"], predictions=result["prediction"])
+
+    # print & log results
+    result_str = (
+        f"WER: {wer_result}\n"
+        f"CER: {cer_result}"
+    )
+    print(result_str)
+
+    with open(f"{dataset_id}_eval_results.txt", "w") as f:
+        f.write(result_str)
+
+    # log all results in text file. Possibly interesting for analysis
+    if log_outputs is not None:
+        pred_file = f"log_{dataset_id}_predictions.txt"
+        target_file = f"log_{dataset_id}_targets.txt"
+
+        with open(pred_file, "w") as p, open(target_file, "w") as t:
+
+            # mapping function to write output
+            def write_to_file(batch, i):
+                p.write(f"{i}" + "\n")
+                p.write(batch["prediction"] + "\n")
+                t.write(f"{i}" + "\n")
+                t.write(batch["target"] + "\n")
+
+            result.map(write_to_file, with_indices=True)
+
+
+def normalize_text(text: str) -> str:
+    """ DO ADAPT FOR YOUR USE CASE. this function normalizes the target text. """
+
+    # chars_to_ignore_regex = '[,?.!\-\;\:\"“%‘”�—’…–]'  # noqa: W605 IMPORTANT: this should correspond to the chars that were ignored during training
+    chars_to_ignore_regex = ',\?\¿\.\!\¡\;\；\:\""\%\"\�\ʿ\·\჻\~\՞\؟\،\।\॥\«\»\„\“\”\「\」\‘\’\《\》\(\)\[\]\{\}\=\`\_\+\<\>\…\–\°\´\ʾ\‹\›\©\®\—\→\。\、\﹂\﹁\‧\～\﹏\，\｛\｝\（\）\［\］\【\】\‥\〽\『\』\〝\〟\⟨\⟩\〜\：\！\？\♪\؛\/\\\º\−\^\ʻ\ˆ\≪\≫'
+    chars_to_ignore_regex += "$\&\'\-\|\¨\ª\ß\à\â\ã\ä\å\æ\ç\ê\ë\ì\î\ï\ð\ò\ô\õ\ö\ø\ù\û\ý\þ\ā\ă\ć\č\đ\ė\ę\ě\ğ\ī\ı\ł\ń\ō\ŏ\ő\œ\ř\ś\ş\š\ū\ź\ż\ž\ș\ț\ə\ʷ\ʽ\ː\́\̇\ϙ\а\б\в\г\д\е\и\й\к\л\н\о\п\р\с\т\ч\ш\ы\ь\ю\я\ё\ү\ө\ְ\ִ\ֵ\ָ\ֹ\ּ\ב\ה\ו\י\כ\ל\ם\מ\נ\ס\ק\ר\ש\ת\ا\ب\ة\د\ذ\ر\ل\م\ه\و\ي\ਆ\ਘ\ਤ\ਨ\ਮ\ਸ\ਾ\ਿ\ੰ\ṁ\ṃ\ṇ\ồ\‐\‑\―\し\の\ひ\ら\ゴ\ヒ\ミ\ム\ラ\㓁\口\周\夷\山\戌\日\本\比\毵\消\生\申\真\箱\网\罒\罓\肋\肌\背\良\蝦\鮓\鮨\fi\$\&\'\-\|\¨\ª\ß\à\â\ã\ä\å\æ\ç\ê\ë\ì\î\ï\ð\ò\ô\õ\ö\ø\ù\û\ý\þ\ā\ă\ć\č\đ\ė\ę\ě\ğ\ī\ı\ł\ń\ō\ŏ\ő\œ\ř\ś\ş\š\ū\ź\ż\ž\ș\ț\ə\ʷ\ʽ\ː\́\̇\ϙ\а\б\в\г\д\е\и\й\к\л\н\о\п\р\с\т\ч\ш\ы\ь\ю\я\ё\ү\ө\ְ\ִ\ֵ\ָ\ֹ\ּ\ב\ה\ו\י\כ\ל\ם\מ\נ\ס\ק\ר\ש\ת\ا\ب\ة\د\ذ\ر\ل\م\ه\و\ي\ਆ\ਘ\ਤ\ਨ\ਮ\ਸ\ਾ\ਿ\ੰ\ṁ\ṃ\ṇ\ồ\‐\‑\―\し\の\ひ\ら\ゴ\ヒ\ミ\ム\ラ\㓁\口\周\夷\山\戌\日\本\比\毵\消\生\申\真\箱\网\罒\罓\肋\肌\背\良\蝦\鮓\鮨\fi\"
+    chars_to_ignore_regex = "[" + chars_to_ignore_regex + "]"
+
+    text = text.lower()
+    # normalize non-standard (stylized) unicode characters
+    text = unicodedata.normalize('NFKC', text)
+    # remove punctuation
+    text = re.sub(chars_to_ignore_regex, "", text)
+
+    # Let's also make sure we split on all kinds of newlines, spaces, etc...
+    text = " ".join(text.split())
+
+    return text
+
+
+def main(args):
+    # load dataset
+    dataset = load_dataset(args.dataset, args.config, split=args.split, use_auth_token=True)
+
+    # for testing: only process the first two examples as a test
+    # dataset = dataset.select(range(10))
+
+    # load processor
+    feature_extractor = AutoFeatureExtractor.from_pretrained(args.model_id)
+    sampling_rate = feature_extractor.sampling_rate
+
+    # resample audio
+    dataset = dataset.cast_column("audio", Audio(sampling_rate=sampling_rate))
+
+    # load eval pipeline
+    # asr = pipeline("automatic-speech-recognition", model=args.model_id)
+    asr = pipeline("automatic-speech-recognition", model=args.model_id, device=0)
+
+    # map function to decode audio
+    def map_to_pred(batch):
+        prediction = asr(batch["audio"]["array"], chunk_length_s=args.chunk_length_s, stride_length_s=args.stride_length_s)
+
+        batch["prediction"] = prediction["text"]
+        batch["target"] = normalize_text(batch["sentence"])
+        return batch
+
+    # run inference on all examples
+    result = dataset.map(map_to_pred, remove_columns=dataset.column_names)
+
+    # compute and log_results
+    # do not change function below
+    log_results(result, args)
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+
+    parser.add_argument(
+        "--model_id", type=str, required=True, help="Model identifier. Should be loadable with 🤗 Transformers"
+    )
+    parser.add_argument(
+        "--dataset", type=str, required=True, help="Dataset name to evaluate the `model_id`. Should be loadable with 🤗 Datasets"
+    )
+    parser.add_argument(
+        "--config", type=str, required=True, help="Config of the dataset. *E.g.* `'en'`  for Common Voice"
+    )
+    parser.add_argument(
+        "--split", type=str, required=True, help="Split of the dataset. *E.g.* `'test'`"
+    )
+    parser.add_argument(
+        "--chunk_length_s", type=float, default=None, help="Chunk length in seconds. Defaults to None. For long audio files a good value would be 5.0 seconds."
+    )
+    parser.add_argument(
+        "--stride_length_s", type=float, default=None, help="Stride of the audio chunks. Defaults to None. For long audio files a good value would be 1.0 seconds."
+    )
+    parser.add_argument(
+        "--log_outputs", action='store_true', help="If defined, write outputs to log file for analysis."
+    )
+    args = parser.parse_args()
+
+    main(args)

language_model/5gram.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9beb443cf2aab7939c2e484b1bdbbe03f57827dd0068db76a7601bf948656ae6
+size 111822055

language_model/attrs.json

@@ -0,0 +1 @@
+{"alpha": 0.5, "beta": 1.5, "unk_score_offset": -10.0, "score_boundary": true}

preprocessor_config.json

@@ -0,0 +1,10 @@
+{
+  "do_normalize": true,
+  "feature_extractor_type": "Wav2Vec2FeatureExtractor",
+  "feature_size": 1,
+  "padding_side": "right",
+  "padding_value": 0.0,
+  "processor_class": "Wav2Vec2ProcessorWithLM",
+  "return_attention_mask": true,
+  "sampling_rate": 16000
+}

pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e21c7d862ecf7b5327f2985887bca39fe968d49ea51449eed86bccb9b46dd6aa
+size 1262079473

special_tokens_map.json

@@ -0,0 +1 @@
+{"bos_token": "<s>", "eos_token": "</s>", "unk_token": "[UNK]", "pad_token": "[PAD]", "additional_special_tokens": [{"content": "<s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, {"content": "</s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, {"content": "<s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, {"content": "</s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}]}

tokenizer_config.json

@@ -0,0 +1 @@
+{"unk_token": "[UNK]", "bos_token": "<s>", "eos_token": "</s>", "pad_token": "[PAD]", "do_lower_case": false, "word_delimiter_token": "|", "special_tokens_map_file": null, "tokenizer_file": null, "name_or_path": "edugp/wav2vec2-xls-r-300m-36-tokens-es", "tokenizer_class": "Wav2Vec2CTCTokenizer", "processor_class": "Wav2Vec2ProcessorWithLM"}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7413e4b25523b52cee9dcbbc7b68c8bded63514be015752c5bda8705c9b1e0af
+size 3055

vocab.json

@@ -0,0 +1 @@
+{"a": 1, "b": 2, "c": 3, "d": 4, "e": 5, "f": 6, "g": 7, "h": 8, "i": 9, "j": 10, "k": 11, "l": 12, "m": 13, "n": 14, "o": 15, "p": 16, "q": 17, "r": 18, "s": 19, "t": 20, "u": 21, "v": 22, "w": 23, "x": 24, "y": 25, "z": 26, "á": 27, "é": 28, "í": 29, "ñ": 30, "ó": 31, "ú": 32, "ü": 33, "|": 0, "[UNK]": 34, "[PAD]": 35}