add tokenizer

added_tokens.json

@@ -0,0 +1 @@
+{"<s>": 97, "</s>": 98}

special_tokens_map.json

@@ -0,0 +1 @@
+{"bos_token": "<s>", "eos_token": "</s>", "unk_token": "[UNK]", "pad_token": "[PAD]", "additional_special_tokens": [{"content": "<s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, {"content": "</s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}]}

tokenizer_config.json

@@ -0,0 +1 @@
+{"unk_token": "[UNK]", "bos_token": "<s>", "eos_token": "</s>", "pad_token": "[PAD]", "do_lower_case": false, "word_delimiter_token": "^", "special_tokens_map_file": null, "tokenizer_file": null, "name_or_path": "./", "tokenizer_class": "Wav2Vec2CTCTokenizer"}

vocab.json

@@ -0,0 +1 @@
+{"a": 1, "b": 2, "c": 3, "d": 4, "e": 5, "f": 6, "g": 7, "h": 8, "i": 9, "j": 10, "k": 11, "l": 12, "m": 13, "n": 14, "o": 15, "p": 16, "r": 17, "s": 18, "t": 19, "u": 20, "v": 21, "w": 22, "x": 23, "y": 24, "z": 25, "|": 26, "ँ": 27, "ं": 28, "ः": 29, "अ": 30, "आ": 31, "इ": 32, "ई": 33, "उ": 34, "ऊ": 35, "ऋ": 36, "ए": 37, "ऐ": 38, "ऑ": 39, "ओ": 40, "औ": 41, "क": 42, "ख": 43, "ग": 44, "घ": 45, "च": 46, "छ": 47, "ज": 48, "झ": 49, "ञ": 50, "ट": 51, "ठ": 52, "ड": 53, "ढ": 54, "ण": 55, "त": 56, "थ": 57, "द": 58, "ध": 59, "न": 60, "प": 61, "फ": 62, "ब": 63, "भ": 64, "म": 65, "य": 66, "र": 67, "ल": 68, "व": 69, "श": 70, "ष": 71, "स": 72, "ह": 73, "़": 74, "ा": 75, "ि": 76, "ी": 77, "ु": 78, "ू": 79, "ृ": 80, "ॅ": 81, "े": 82, "ै": 83, "ॉ": 84, "ो": 85, "ौ": 86, "्": 87, "क़": 88, "ग़": 89, "ज़": 90, "ड़": 91, "ढ़": 92, "।": 93, "’": 94, "^": 0, "[UNK]": 95, "[PAD]": 96}