Update README.md

README.md

@@ -44,7 +44,7 @@ import torchaudio
 from datasets import load_dataset
 from transformers import Wav2Vec2ForCTC, Wav2Vec2Processor
 
-test_dataset = load_dataset("", split="test[:2%]")
+test_dataset = load_dataset("ma_speech_corpus", split="test")
 
 processor = Wav2Vec2Processor.from_pretrained("othrif/wav2vec2-large-xlsr-arabic")
 model = Wav2Vec2ForCTC.from_pretrained("othrif/wav2vec2-large-xlsr-arabic")
@@ -83,20 +83,38 @@ from datasets import load_dataset, load_metric
 from transformers import Wav2Vec2ForCTC, Wav2Vec2Processor
 import re
 
-test_dataset = load_dataset("common_voice", "ar", split="test") 
+test_dataset = load_dataset("ma_speech_corpus", split="test") 
 wer = load_metric("wer")
 
-processor = Wav2Vec2Processor.from_pretrained("othrif/wav2vec2-large-xlsr-arabic") 
-model = Wav2Vec2ForCTC.from_pretrained("othrif/wav2vec2-large-xlsr-arabic")
+processor = Wav2Vec2Processor.from_pretrained("othrif/wav2vec2-large-xlsr-moroccan") 
+model = Wav2Vec2ForCTC.from_pretrained("othrif/wav2vec2-large-xlsr-moroccan")
 model.to("cuda")
 
-chars_to_ignore_regex = '[\\\\\\\\؛\\\\\\\\—\\\\\\\\_get\\\\\\\\«\\\\\\\\»\\\\\\\\ـ\\\\\\\\ـ\\\\\\\\,\\\\\\\\?\\\\\\\\.\\\\\\\\!\\\\\\\\-\\\\\\\\;\\\\\\\\:\\\\\\\\"\\\\\\\\“\\\\\\\\%\\\\\\\\‘\\\\\\\\”\\\\\\\\�\\\\\\\\#\\\\\\\\،\\\\\\\\☭,\\\\\\\\؟]'
-resampler = torchaudio.transforms.Resample(48_000, 16_000)
+chars_to_ignore_regex = '[0\,\?\.\!\-\;\:\"\“\%\‘\”\�\n\@\ـ\؟\*\ \#\'\ \…\\u2003]'
+#resampler = torchaudio.transforms.Resample(48_000, 16_000)
 
 # Preprocessing the datasets.
 # We need to read the audio files as arrays
 def speech_file_to_array_fn(batch):
-    batch["sentence"] = re.sub(chars_to_ignore_regex, '', batch["sentence"]).lower()
+    batch["text"] = re.sub(chars_to_ignore_regex, '', batch["text"]).lower()
+    batch["text"] = re.sub('[a-zA-z]', '', batch["text"]).lower() + " "
+        batch["text"] = re.sub('[ًٌٍَُِ~]', '', batch["text"]).lower() + " "
+
+        # batch["text"] = re.sub('\\n','', batch["text"])
+        batch["text"] = re.sub("[إأٱآا]", "ا", batch["text"])
+        batch["text"] = re.sub("ڸ", "ل", batch["text"])
+        noise = re.compile(""" ّ    | # Tashdid
+                             َ    | # Fatha
+                             ً    | # Tanwin Fath
+                             ُ    | # Damma
+                             ٌ    | # Tanwin Damm
+                             ِ    | # Kasra
+                             ٍ    | # Tanwin Kasr
+                             ْ    | # Sukun
+                             ـ     # Tatwil/Kashida
+                         """, re.VERBOSE)
+        batch["text"] = re.sub(noise, '', batch["text"])
+        batch["text"] = re.sub('ٖ', '', batch["text"]).lower() + " "
     speech_array, sampling_rate = torchaudio.load(batch["path"])
     batch["speech"] = resampler(speech_array).squeeze().numpy()
     return batch
@@ -117,7 +135,7 @@ def evaluate(batch):
 
 result = test_dataset.map(evaluate, batched=True, batch_size=8)
 
-print("WER: {:2f}".format(100 * wer.compute(predictions=result["pred_strings"], references=result["sentence"])))
+print("WER: {:2f}".format(100 * wer.compute(predictions=result["pred_strings"], references=result["text"])))
 ```
 
 **Test Result**: 44.51
@@ -125,6 +143,6 @@ print("WER: {:2f}".format(100 * wer.compute(predictions=result["pred_strings"],
 
 ## Training
 
-The Common Voice `train`, `validation` datasets were used for training.
+The  [MGB5](http://www.islrn.org/resources/938-639-614-524-5/) `train`, `validation` datasets were used for training.
 
-The script used for training can be found [here](https://huggingface.co/othrif/wav2vec2-large-xlsr-arabic/tree/main) 
+The script used for training can be found [here](https://github.com/othrif/xlsr-wav2vec2)