Update: readme

README.md

@@ -81,31 +81,66 @@ print("Reference:", test_dataset["sentence"][:2])
 The model can be evaluated as follows on the Japanese test data of Common Voice.
 
 ```python
+!pip install torchaudio
+!pip install datasets transformers
+!pip install jiwer
+!pip install mecab-python3
+!pip install unidic-lite
+!python -m unidic download
+!pip install jaconv
+
 import torch
 import torchaudio
 from datasets import load_dataset, load_metric
 from transformers import Wav2Vec2ForCTC, Wav2Vec2Processor
 import re
+import MeCab
+from jaconv import kata2hira
+from typing import List
+
+# Japanese preprocessing
+tagger = MeCab.Tagger("-Owakati")
+chars_to_ignore_regex = '[\。\、\「\」\,\?\.\!\-\;\:\"\“\%\‘\”\�]'
+
+def text2kata(text):
+    node = tagger.parseToNode(text)
+    word_class = []
+    while node:
+        word = node.surface
+        wclass = node.feature.split(',')
+        if wclass[0] != u'BOS/EOS':
+            if len(wclass) <= 6:
+                word_class.append((word))
+            elif wclass[6] == None:
+                word_class.append((word))
+            else:
+                word_class.append((wclass[6]))
+        node = node.next
+    return ' '.join(word_class)
+
+def hiragana(text):
+    return kata2hira(text2kata(text))
 
 test_dataset = load_dataset("common_voice", "ja", split="test")
 wer = load_metric("wer")
+resampler = torchaudio.transforms.Resample(48_000, 16_000) # JSUT is already 16kHz
+# resampler = torchaudio.transforms.Resample(16_000, 16_000) # JSUT is already 16kHz
 
 processor = Wav2Vec2Processor.from_pretrained("qqhann/w2v_hf_jsut_xlsr53")
 model = Wav2Vec2ForCTC.from_pretrained("qqhann/w2v_hf_jsut_xlsr53")
 model.to("cuda")
 
-chars_to_ignore_regex = '[\\\\,\\\\?\\\\.\\\\!\\\\-\\\\;\\\\:\\\\"\\\\“]'  # TODO: adapt this list to include all special characters you removed from the data
-# resampler = torchaudio.transforms.Resample(48_000, 16_000) # JSUT is already 16kHz
-resampler = torchaudio.transforms.Resample(16_000, 16_000) # JSUT is already 16kHz
 
 # Preprocessing the datasets.
 # We need to read the aduio files as arrays
 def speech_file_to_array_fn(batch):
+    batch["sentence"] = hiragana(batch["sentence"]).strip()
     batch["sentence"] = re.sub(chars_to_ignore_regex, '', batch["sentence"]).lower()
     speech_array, sampling_rate = torchaudio.load(batch["path"])
     batch["speech"] = resampler(speech_array).squeeze().numpy()
     return batch
 
+
 test_dataset = test_dataset.map(speech_file_to_array_fn)
 
 # Preprocessing the datasets.
@@ -122,10 +157,16 @@ def evaluate(batch):
 
 result = test_dataset.map(evaluate, batched=True, batch_size=8)
 
+def cer_compute(predictions: List[str], references: List[str]):
+    p = [" ".join(list(" " + pred.replace(" ", ""))).strip() for pred in predictions]
+    r = [" ".join(list(" " + ref.replace(" ", ""))).strip() for ref in references]
+    return wer.compute(predictions=p, references=r)
+
 print("WER: {:2f}".format(100 * wer.compute(predictions=result["pred_strings"], references=result["sentence"])))
+print("CER: {:2f}".format(100 * cer_compute(predictions=result["pred_strings"], references=result["sentence"])))
 ```
 
-**Test Result**: 20.48 %
+**Test Result**: 51.72 %
 
 ## Training