add tokenizer

added_tokens.json

@@ -0,0 +1 @@
+{"<s>": 95, "</s>": 96}

special_tokens_map.json

@@ -0,0 +1 @@
+{"bos_token": "<s>", "eos_token": "</s>", "unk_token": "[UNK]", "pad_token": "[PAD]", "additional_special_tokens": [{"content": "<s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, {"content": "</s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}]}

tokenizer_config.json

@@ -0,0 +1 @@
+{"unk_token": "[UNK]", "bos_token": "<s>", "eos_token": "</s>", "pad_token": "[PAD]", "do_lower_case": false, "word_delimiter_token": "|", "special_tokens_map_file": null, "tokenizer_file": null, "name_or_path": "./", "tokenizer_class": "Wav2Vec2CTCTokenizer"}

vocab.json

@@ -0,0 +1 @@
+{"&": 1, "a": 2, "b": 3, "c": 4, "d": 5, "e": 6, "f": 7, "g": 8, "h": 9, "i": 10, "j": 11, "k": 12, "l": 13, "m": 14, "n": 15, "o": 16, "p": 17, "r": 18, "s": 19, "t": 20, "u": 21, "v": 22, "w": 23, "x": 24, "y": 25, "z": 26, "ँ": 27, "ं": 28, "ः": 29, "अ": 30, "आ": 31, "इ": 32, "ई": 33, "उ": 34, "ऊ": 35, "ऋ": 36, "ए": 37, "ऐ": 38, "ऑ": 39, "ओ": 40, "औ": 41, "क": 42, "ख": 43, "ग": 44, "घ": 45, "च": 46, "छ": 47, "ज": 48, "झ": 49, "ञ": 50, "ट": 51, "ठ": 52, "ड": 53, "ढ": 54, "ण": 55, "त": 56, "थ": 57, "द": 58, "ध": 59, "न": 60, "प": 61, "फ": 62, "ब": 63, "भ": 64, "म": 65, "य": 66, "र": 67, "ल": 68, "व": 69, "श": 70, "ष": 71, "स": 72, "ह": 73, "़": 74, "ा": 75, "ि": 76, "ी": 77, "ु": 78, "ू": 79, "ृ": 80, "ॅ": 81, "े": 82, "ै": 83, "ॉ": 84, "ो": 85, "ौ": 86, "्": 87, "क़": 88, "ग़": 89, "ज़": 90, "ड़": 91, "ढ़": 92, "|": 0, "[UNK]": 93, "[PAD]": 94}