Commit
•
a44d066
1
Parent(s):
a07ecad
add tokenizer
Browse files- special_tokens_map.json +1 -0
- tokenizer_config.json +1 -0
- vocab.json +1 -0
special_tokens_map.json
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
{"bos_token": "<s>", "eos_token": "</s>", "unk_token": "[UNK]", "pad_token": "[PAD]"}
|
tokenizer_config.json
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
{"unk_token": "[UNK]", "bos_token": "<s>", "eos_token": "</s>", "pad_token": "[PAD]", "do_lower_case": false, "word_delimiter_token": "|", "tokenizer_class": "Wav2Vec2CTCTokenizer"}
|
vocab.json
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
{"w": 0, "ठ": 1, "र": 2, "अ": 3, "य": 4, "u": 5, "e": 6, "औ": 7, "भ": 8, "ण": 9, "ल": 10, "ग": 11, "ब": 12, "च": 13, "व": 14, "p": 15, "इ": 16, "ॉ": 17, "ड़": 18, "ख": 19, "ौ": 20, "ओ": 21, "i": 22, "न": 23, "'": 24, "ु": 25, "उ": 26, "घ": 27, "े": 28, "ट": 29, "ी": 30, "r": 31, "स": 32, "ध": 33, "ः": 35, "ष": 36, "m": 37, "छ": 38, "ि": 39, "ै": 40, "l": 41, "प": 42, "ज": 43, "ई": 44, "फ": 45, "झ": 46, "ऑ": 47, "त": 48, "ो": 49, "ऐ": 50, "ह": 51, "a": 52, "ा": 53, "्": 54, "ू": 55, "़": 56, "श": 57, "ँ": 58, "क": 59, "f": 60, "ं": 61, "ढ": 62, "ए": 63, "ड": 64, "ऊ": 65, "म": 66, "थ": 67, "आ": 68, "द": 69, "ृ": 70, "|": 34, "[UNK]": 71, "[PAD]": 72}
|