tanmaylaud
commited on
Commit
•
0c60532
1
Parent(s):
16efafc
Update README.md
Browse files
README.md
CHANGED
@@ -35,15 +35,16 @@ Fine-tuned facebook/wav2vec2-large-xlsr-53 on Hindi and Marathi using the OpenSL
|
|
35 |
pip install git+https://github.com/huggingface/transformers.git datasets librosa torch==1.7.0 torchaudio==0.7.0 jiwer
|
36 |
|
37 |
## Eval dataset:
|
38 |
-
|
39 |
-
|
40 |
-
|
41 |
-
|
42 |
-
|
43 |
-
|
44 |
-
|
45 |
-
|
46 |
-
|
|
|
47 |
## Usage
|
48 |
The model can be used directly (without a language model) as follows, assuming you have a dataset with Marathi text and path fields:
|
49 |
|
@@ -87,7 +88,7 @@ print("Prediction:", processor.batch_decode(predicted_ids))
|
|
87 |
print("Reference:", test_data["text"][:2])
|
88 |
```
|
89 |
|
90 |
-
#Code For Evaluation on OpenSLR (Hindi + Marathi : https://filebin.net/snrz6bt13usv8w2e/test_large.csv)
|
91 |
```python
|
92 |
import torchaudio
|
93 |
import torch
|
@@ -98,7 +99,7 @@ import re
|
|
98 |
test = Dataset.from_csv('test.csv')
|
99 |
|
100 |
|
101 |
-
chars_to_ignore_regex = '[
|
102 |
|
103 |
# Preprocessing the datasets.
|
104 |
# We need to read the audio files as arrays
|
@@ -138,7 +139,7 @@ import numpy as np
|
|
138 |
import re
|
139 |
from datasets import load_dataset
|
140 |
|
141 |
-
chars_to_ignore_regex = '[
|
142 |
|
143 |
# Preprocessing the datasets.
|
144 |
# We need to read the audio files as arrays
|
@@ -172,4 +173,5 @@ print("WER: {:2f}".format(100 * wer.compute(predictions=test_data["pred_strings"
|
|
172 |
|
173 |
Link to eval notebook : https://colab.research.google.com/drive/1nZRTgKfxCD9cvy90wikTHkg2il3zgcqW#scrollTo=cXWFbhb0d7DT
|
174 |
|
175 |
-
WER : 24.944955% (
|
|
35 |
pip install git+https://github.com/huggingface/transformers.git datasets librosa torch==1.7.0 torchaudio==0.7.0 jiwer
|
36 |
|
37 |
## Eval dataset:
|
38 |
+
```bash
|
39 |
+
wget https://www.openslr.org/resources/103/Marathi_test.zip -P data/marathi
|
40 |
+
unzip -P "K3[2?do9" data/marathi/Marathi_test.zip -d data/marathi/.
|
41 |
+
tar -xzf data/marathi/Marathi_test.tar.gz -C data/marathi/.
|
42 |
+
wget https://www.openslr.org/resources/103/Hindi_test.zip -P data/hindi
|
43 |
+
unzip -P "w9I2{3B*" data/hindi/Hindi_test.zip -d data/hindi/.
|
44 |
+
tar -xzf data/hindi/Hindi_test.tar.gz -C data/hindi/.
|
45 |
+
wget -O test.csv 'https://filebin.net/snrz6bt13usv8w2e/test_large.csv?t=ps3n99ho'
|
46 |
+
#If download does not work, paste this link in browser: https://filebin.net/snrz6bt13usv8w2e/test_large.csv
|
47 |
+
```
|
48 |
## Usage
|
49 |
The model can be used directly (without a language model) as follows, assuming you have a dataset with Marathi text and path fields:
|
50 |
|
88 |
print("Reference:", test_data["text"][:2])
|
89 |
```
|
90 |
|
91 |
+
# Code For Evaluation on OpenSLR (Hindi + Marathi : https://filebin.net/snrz6bt13usv8w2e/test_large.csv)
|
92 |
```python
|
93 |
import torchaudio
|
94 |
import torch
|
99 |
test = Dataset.from_csv('test.csv')
|
100 |
|
101 |
|
102 |
+
chars_to_ignore_regex = '[\\,\\?\\.\\!\\-\\;\\:\\"\\“\\%\\‘\\”\\�\\।]'
|
103 |
|
104 |
# Preprocessing the datasets.
|
105 |
# We need to read the audio files as arrays
|
139 |
import re
|
140 |
from datasets import load_dataset
|
141 |
|
142 |
+
chars_to_ignore_regex = '[\\,\\?\\.\\!\\-\\;\\:\\"\\“\\%\\‘\\”\\�\\।]'
|
143 |
|
144 |
# Preprocessing the datasets.
|
145 |
# We need to read the audio files as arrays
|
173 |
|
174 |
Link to eval notebook : https://colab.research.google.com/drive/1nZRTgKfxCD9cvy90wikTHkg2il3zgcqW#scrollTo=cXWFbhb0d7DT
|
175 |
|
176 |
+
WER : 24.944955% (OpenSLR Hindi+Marathi Test set : https://filebin.net/snrz6bt13usv8w2e/test_large.csv)
|
177 |
+
WER: 49.303944% (Common Voice Hindi Test Split)
|