Create README.md

README.md

@@ -0,0 +1,381 @@
+---
+language: pt
+datasets:
+- common_voice 
+- mls
+- cetuc
+- lapsbm
+- voxforge
+- tedx
+- sid
+metrics:
+- wer
+tags:
+- audio
+- speech
+- wav2vec2
+- pt
+- portuguese-speech-corpus
+- automatic-speech-recognition
+- speech
+- PyTorch
+license: apache-2.0
+---
+
+# mls100-xlsr: Wav2vec 2.0 with LaPSBM Dataset
+
+This is a the demonstration of a fine-tuned Wav2vec model for Brazilian Portuguese using the [Multilingual Librispeech in Portuguese (MLS)](http://www.openslr.org/94/) dataset.
+
+In this notebook the model is tested against other available Brazilian Portuguese datasets. 
+
+| Dataset                        |  Train | Valid |  Test |
+|--------------------------------|-------:|------:|------:|
+| CETUC                          |        |    -- |  5.4h |
+| Common Voice                   |        |    -- |  9.5h |
+| LaPS BM                        |        |    -- |  0.1h |
+| MLS                            |  161h  |    -- |  3.7h |
+| Multilingual TEDx (Portuguese) |        |    -- |  1.8h |
+| SID                            |        |    -- |  1.0h |
+| VoxForge                       |        |    -- |  0.1h |
+| Total                          |  161h  |    -- | 21.6h |
+
+
+#### Summary
+
+|  | CETUC          | CV             | LaPS           | MLS            | SID            | TEDx           | VF             | AVG            |
+|----------------------|---------------|----------------|----------------|----------------|----------------|----------------|----------------|----------------|
+| mls100 (demonstration below) | 0.192 | 0.260 | 0.162 | 0.163 | 0.268 | 0.492 | 0.268 | 0.258 |
+| mls100 + 4-gram (demonstration below) | 0.087 | 0.173 | 0.077 | 0.126 | 0.245 | 0.415 | 0.218 | 0.191 |
+
+## Demonstration
+
+
+```python
+MODEL_NAME = "lgris/mls100-xlsr" 
+```
+
+### Imports and dependencies
+
+
+```python
+%%capture
+!pip install torch==1.8.2+cu111 torchvision==0.9.2+cu111 torchaudio===0.8.2 -f https://download.pytorch.org/whl/lts/1.8/torch_lts.html
+!pip install datasets
+!pip install jiwer
+!pip install transformers
+!pip install soundfile
+!pip install pyctcdecode
+!pip install https://github.com/kpu/kenlm/archive/master.zip
+```
+
+
+```python
+import jiwer
+import torchaudio
+from datasets import load_dataset, load_metric
+from transformers import (
+    Wav2Vec2ForCTC,
+    Wav2Vec2Processor,
+)
+from pyctcdecode import build_ctcdecoder
+import torch
+import re
+import sys
+```
+
+### Helpers
+
+
+```python
+chars_to_ignore_regex = '[\,\?\.\!\;\:\"]'  # noqa: W605
+
+def map_to_array(batch):
+    speech, _ = torchaudio.load(batch["path"])
+    batch["speech"] = speech.squeeze(0).numpy() 
+    batch["sampling_rate"] = 16_000 
+    batch["sentence"] = re.sub(chars_to_ignore_regex, '', batch["sentence"]).lower().replace("’", "'")
+    batch["target"] = batch["sentence"]
+    return batch
+```
+
+
+```python
+def calc_metrics(truths, hypos):
+    wers = []
+    mers = []
+    wils = []
+    for t, h in zip(truths, hypos):
+        try:
+            wers.append(jiwer.wer(t, h))
+            mers.append(jiwer.mer(t, h))
+            wils.append(jiwer.wil(t, h))
+        except: # Empty string?
+            pass
+    wer = sum(wers)/len(wers)
+    mer = sum(mers)/len(mers)
+    wil = sum(wils)/len(wils)
+    return wer, mer, wil
+```
+
+
+```python
+def load_data(dataset):
+    data_files = {'test': f'{dataset}/test.csv'}
+    dataset = load_dataset('csv', data_files=data_files)["test"]
+    return dataset.map(map_to_array)
+```
+
+### Model
+
+
+```python
+class STT:
+
+    def __init__(self, 
+                 model_name, 
+                 device='cuda' if torch.cuda.is_available() else 'cpu', 
+                 lm=None):
+        self.model_name = model_name
+        self.model = Wav2Vec2ForCTC.from_pretrained(model_name).to(device)
+        self.processor = Wav2Vec2Processor.from_pretrained(model_name)
+        self.vocab_dict = self.processor.tokenizer.get_vocab()
+        self.sorted_dict = {
+            k.lower(): v for k, v in sorted(self.vocab_dict.items(), 
+                                            key=lambda item: item[1])
+        }
+        self.device = device
+        self.lm = lm
+        if self.lm:            
+            self.lm_decoder = build_ctcdecoder(
+                list(self.sorted_dict.keys()),
+                self.lm
+            )
+
+    def batch_predict(self, batch):
+        features = self.processor(batch["speech"], 
+                                  sampling_rate=batch["sampling_rate"][0], 
+                                  padding=True, 
+                                  return_tensors="pt")
+        input_values = features.input_values.to(self.device)
+        attention_mask = features.attention_mask.to(self.device)
+        with torch.no_grad():
+            logits = self.model(input_values, attention_mask=attention_mask).logits
+        if self.lm:
+            logits = logits.cpu().numpy()
+            batch["predicted"] = []
+            for sample_logits in logits:
+                batch["predicted"].append(self.lm_decoder.decode(sample_logits))
+        else:
+            pred_ids = torch.argmax(logits, dim=-1)
+            batch["predicted"] = self.processor.batch_decode(pred_ids)
+        return batch
+```
+
+### Download datasets
+
+
+```python
+%%capture
+!gdown --id 1HFECzIizf-bmkQRLiQD0QVqcGtOG5upI
+!mkdir bp_dataset
+!unzip bp_dataset -d bp_dataset/
+```
+
+
+```python
+%cd bp_dataset/
+```
+
+    /content/bp_dataset
+
+
+### Tests 
+
+
+```python
+stt = STT(MODEL_NAME)
+```
+
+#### CETUC
+
+
+```python
+ds = load_data('cetuc_dataset')
+result = ds.map(stt.batch_predict, batched=True, batch_size=8) 
+wer, mer, wil = calc_metrics(result["sentence"], result["predicted"])
+print("CETUC WER:", wer)
+```
+    CETUC WER: 0.192586382955233
+
+
+#### Common Voice
+
+
+```python
+ds = load_data('commonvoice_dataset')
+result = ds.map(stt.batch_predict, batched=True, batch_size=8) 
+wer, mer, wil = calc_metrics(result["sentence"], result["predicted"])
+print("CV WER:", wer)
+```
+    CV WER: 0.2604333640312866
+
+
+#### LaPS
+
+
+```python
+ds = load_data('lapsbm_dataset')
+result = ds.map(stt.batch_predict, batched=True, batch_size=8) 
+wer, mer, wil = calc_metrics(result["sentence"], result["predicted"])
+print("Laps WER:", wer)
+```
+    Laps WER: 0.16259469696969692
+
+
+#### MLS
+
+
+```python
+ds = load_data('mls_dataset')
+result = ds.map(stt.batch_predict, batched=True, batch_size=8) 
+wer, mer, wil = calc_metrics(result["sentence"], result["predicted"])
+print("MLS WER:", wer)
+```
+    MLS WER: 0.16343014413283674
+
+
+#### SID
+
+
+```python
+ds = load_data('sid_dataset')
+result = ds.map(stt.batch_predict, batched=True, batch_size=8) 
+wer, mer, wil = calc_metrics(result["sentence"], result["predicted"])
+print("Sid WER:", wer)
+```
+    Sid WER: 0.2682880375992515
+
+
+#### TEDx
+
+
+```python
+ds = load_data('tedx_dataset')
+result = ds.map(stt.batch_predict, batched=True, batch_size=8) 
+wer, mer, wil = calc_metrics(result["sentence"], result["predicted"])
+print("TEDx WER:", wer)
+```
+    TEDx WER: 0.49252836581485837
+
+
+#### VoxForge
+
+
+```python
+ds = load_data('voxforge_dataset')
+result = ds.map(stt.batch_predict, batched=True, batch_size=8) 
+wer, mer, wil = calc_metrics(result["sentence"], result["predicted"])
+print("VoxForge WER:", wer)
+```
+    VoxForge WER: 0.2686972402597403
+
+
+### Tests with LM
+
+
+```python
+!rm -rf ~/.cache
+%cd /content/
+# !gdown --id '1d13Onxy9ubmJZORZ8FO2vnsnl36QMiUc' # trained with wikipedia;
+stt = STT(MODEL_NAME, lm='pt-BR-wiki.word.4-gram.arpa')
+# !gdown --id 1dLFldy7eguPtyJj5OAlI4Emnx0BpFywg  # trained with bp
+# stt = STT(MODEL_NAME, lm='pt-BR.word.4-gram.arpa')
+%cd bp_dataset/
+```
+
+    /content/bp_dataset
+
+
+#### CETUC
+
+
+```python
+ds = load_data('cetuc_dataset')
+result = ds.map(stt.batch_predict, batched=True, batch_size=8) 
+wer, mer, wil = calc_metrics(result["sentence"], result["predicted"])
+print("CETUC WER:", wer)
+```
+    CETUC WER: 0.0878818926974661
+
+
+#### Common Voice
+
+
+```python
+ds = load_data('commonvoice_dataset')
+result = ds.map(stt.batch_predict, batched=True, batch_size=8) 
+wer, mer, wil = calc_metrics(result["sentence"], result["predicted"])
+print("CV WER:", wer)
+```
+    CV WER: 0.173303354010221
+
+
+#### LaPS
+
+
+```python
+ds = load_data('lapsbm_dataset')
+result = ds.map(stt.batch_predict, batched=True, batch_size=8) 
+wer, mer, wil = calc_metrics(result["sentence"], result["predicted"])
+print("Laps WER:", wer)
+```
+    Laps WER: 0.07691919191919189
+
+
+#### MLS
+
+
+```python
+ds = load_data('mls_dataset')
+result = ds.map(stt.batch_predict, batched=True, batch_size=8) 
+wer, mer, wil = calc_metrics(result["sentence"], result["predicted"])
+print("MLS WER:", wer)
+```
+    MLS WER: 0.12624377042839321
+
+
+#### SID
+
+
+```python
+ds = load_data('sid_dataset')
+result = ds.map(stt.batch_predict, batched=True, batch_size=8) 
+wer, mer, wil = calc_metrics(result["sentence"], result["predicted"])
+print("Sid WER:", wer)
+```
+    Sid WER: 0.24545473435776916
+
+
+#### TEDx
+
+
+```python
+ds = load_data('tedx_dataset')
+result = ds.map(stt.batch_predict, batched=True, batch_size=8) 
+wer, mer, wil = calc_metrics(result["sentence"], result["predicted"])
+print("TEDx WER:", wer)
+```
+    TEDx WER: 0.4156272215612955
+
+
+#### VoxForge
+
+
+```python
+ds = load_data('voxforge_dataset')
+result = ds.map(stt.batch_predict, batched=True, batch_size=8) 
+wer, mer, wil = calc_metrics(result["sentence"], result["predicted"])
+print("VoxForge WER:", wer)
+```
+    VoxForge WER: 0.21832386363636366
+