kingabzpro commited on
Commit
00918c5
1 Parent(s): 6e6a217

add tokenizer

Browse files
Files changed (2) hide show
  1. tokenizer_config.json +1 -1
  2. vocab.json +1 -1
tokenizer_config.json CHANGED
@@ -1 +1 @@
1
- {"unk_token": "<unk>", "bos_token": "<s>", "eos_token": "</s>", "pad_token": "<pad>", "do_lower_case": false, "word_delimiter_token": "|", "special_tokens_map_file": "/root/.cache/huggingface/transformers/898afae2faf3d20dd22149b80863bb5d448c67d9c9b821dd66753253a4d77c64.9d6cd81ef646692fb1c169a880161ea1cb95f49694f220aced9b704b457e51dd", "tokenizer_file": null, "name_or_path": "Harveenchadha/vakyansh-wav2vec2-urdu-urm-60", "tokenizer_class": "Wav2Vec2CTCTokenizer"}
 
1
+ {"unk_token": "<unk>", "bos_token": "<s>", "eos_token": "</s>", "pad_token": "<pad>", "do_lower_case": false, "word_delimiter_token": "|", "tokenizer_class": "Wav2Vec2CTCTokenizer"}
vocab.json CHANGED
@@ -1 +1 @@
1
- {"<s>": 0, "<pad>": 1, "</s>": 2, "<unk>": 3, "|": 4, "ء": 5, "آ": 6, "ؤ": 7, "ئ": 8, "ا": 9, "ب": 10, "ت": 11, "ث": 12, "ج": 13, "ح": 14, "خ": 15, "د": 16, "ذ": 17, "ر": 18, "ز": 19, "س": 20, "ش": 21, "ص": 22, "ض": 23, "ط": 24, "ظ": 25, "ع": 26, "غ": 27, "ف": 28, "ق": 29, "ل": 30, "م": 31, "ن": 32, "و": 33, "ٹ": 34, "پ": 35, "چ": 36, "ڈ": 37, "ڑ": 38, "ژ": 39, "ک": 40, "گ": 41, "ں": 42, "ھ": 43, "ہ": 44, "ی": 45, "ے": 46, "ۓ": 47}
 
1
+ {"،": 1, "؟": 2, "ء": 3, "آ": 4, "ؤ": 5, "ئ": 6, "ا": 7, "ب": 8, "ت": 9, "ث": 10, "ج": 11, "ح": 12, "خ": 13, "د": 14, "ذ": 15, "ر": 16, "ز": 17, "س": 18, "ش": 19, "ص": 20, "ض": 21, "ط": 22, "ظ": 23, "ع": 24, "غ": 25, "ف": 26, "ق": 27, "ل": 28, "م": 29, "ن": 30, "و": 31, "ى": 32, "ي": 33, "َ": 34, "ُ": 35, "ِ": 36, "ّ": 37, "ٔ": 38, "ٰ": 39, "ٹ": 40, "پ": 41, "چ": 42, "ڈ": 43, "ڑ": 44, "ژ": 45, "ک": 46, "گ": 47, "ں": 48, "ھ": 49, "ہ": 50, "ۂ": 51, "ی": 52, "ے": 53, "۔": 54, "|": 0, "<unk>": 55, "<pad>": 56, "<s>": 57, "</s>": 58}