add tokenizer

special_tokens_map.json

@@ -1 +1 @@
-{"bos_token": "<s>", "eos_token": "</s>", "unk_token": "[UNK]", "pad_token": "[PAD]", "additional_special_tokens": [{"content": "<s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, {"content": "</s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, {"content": "<s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, {"content": "</s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, {"content": "<s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, {"content": "</s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}]}
+{"bos_token": "<s>", "eos_token": "</s>", "unk_token": "<unk>", "pad_token": "<pad>"}

tokenizer_config.json

@@ -1 +1 @@
-{"unk_token": "[UNK]", "bos_token": "<s>", "eos_token": "</s>", "pad_token": "[PAD]", "do_lower_case": false, "word_delimiter_token": "|", "special_tokens_map_file": null, "tokenizer_file": null, "name_or_path": "anuragshas/wav2vec2-large-xls-r-300m-ur-cv8", "processor_class": "Wav2Vec2ProcessorWithLM", "tokenizer_class": "Wav2Vec2CTCTokenizer"}
+{"unk_token": "<unk>", "bos_token": "<s>", "eos_token": "</s>", "pad_token": "<pad>", "do_lower_case": false, "word_delimiter_token": "|", "tokenizer_class": "Wav2Vec2CTCTokenizer"}

vocab.json

@@ -1 +1 @@
-{"ء": 1, "آ": 2, "ؤ": 3, "ئ": 4, "ا": 5, "ب": 6, "ت": 7, "ث": 8, "ج": 9, "ح": 10, "خ": 11, "د": 12, "ذ": 13, "ر": 14, "ز": 15, "س": 16, "ش": 17, "ص": 18, "ض": 19, "ط": 20, "ظ": 21, "ع": 22, "غ": 23, "ف": 24, "ق": 25, "ل": 26, "م": 27, "ن": 28, "و": 29, "ى": 30, "ي": 31, "ً": 32, "َ": 33, "ُ": 34, "ِ": 35, "ّ": 36, "ٔ": 37, "ٰ": 38, "ٹ": 39, "پ": 40, "چ": 41, "ڈ": 42, "ڑ": 43, "ژ": 44, "ک": 45, "گ": 46, "ں": 47, "ھ": 48, "ہ": 49, "ۂ": 50, "ی": 51, "ے": 52, "|": 0, "[UNK]": 53, "[PAD]": 54}
+{"ء": 1, "آ": 2, "ئ": 3, "ا": 4, "ب": 5, "ت": 6, "ث": 7, "ج": 8, "ح": 9, "خ": 10, "د": 11, "ذ": 12, "ر": 13, "ز": 14, "س": 15, "ش": 16, "ص": 17, "ض": 18, "ط": 19, "ظ": 20, "ع": 21, "غ": 22, "ف": 23, "ق": 24, "ل": 25, "م": 26, "ن": 27, "و": 28, "ً": 29, "ٹ": 30, "پ": 31, "چ": 32, "ڈ": 33, "ڑ": 34, "ژ": 35, "ک": 36, "گ": 37, "ں": 38, "ھ": 39, "ہ": 40, "ی": 41, "ے": 42, "|": 0, "<unk>": 43, "<pad>": 44, "<s>": 45, "</s>": 46}