add tokenizer

added_tokens.json

@@ -0,0 +1 @@
+{"<s>": 137, "</s>": 138}

special_tokens_map.json

@@ -0,0 +1 @@
+{"bos_token": "<s>", "eos_token": "</s>", "unk_token": "[UNK]", "pad_token": "[PAD]", "additional_special_tokens": [{"content": "<s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, {"content": "</s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}]}

tokenizer_config.json

@@ -0,0 +1 @@
+{"unk_token": "[UNK]", "bos_token": "<s>", "eos_token": "</s>", "pad_token": "[PAD]", "do_lower_case": false, "word_delimiter_token": "|", "special_tokens_map_file": null, "tokenizer_file": null, "name_or_path": "./", "tokenizer_class": "Wav2Vec2CTCTokenizer"}

vocab.json

@@ -0,0 +1 @@
+{"$": 1, "&": 2, "'": 3, "-": 4, "a": 5, "b": 6, "c": 7, "d": 8, "e": 9, "f": 10, "g": 11, "h": 12, "i": 13, "j": 14, "k": 15, "l": 16, "m": 17, "n": 18, "o": 19, "p": 20, "q": 21, "r": 22, "s": 23, "t": 24, "u": 25, "v": 26, "w": 27, "x": 28, "y": 29, "z": 30, "|": 0, "¨": 32, "ª": 33, "ß": 34, "à": 35, "á": 36, "â": 37, "ã": 38, "ä": 39, "å": 40, "æ": 41, "ç": 42, "é": 43, "ê": 44, "ë": 45, "ì": 46, "í": 47, "î": 48, "ï": 49, "ð": 50, "ñ": 51, "ò": 52, "ó": 53, "ô": 54, "õ": 55, "ö": 56, "ø": 57, "ù": 58, "ú": 59, "û": 60, "ü": 61, "ý": 62, "þ": 63, "ā": 64, "ă": 65, "ć": 66, "č": 67, "đ": 68, "ė": 69, "ę": 70, "ě": 71, "ğ": 72, "ī": 73, "ı": 74, "ł": 75, "ń": 76, "ō": 77, "ŏ": 78, "ő": 79, "œ": 80, "ř": 81, "ś": 82, "ş": 83, "š": 84, "ū": 85, "ź": 86, "ż": 87, "ž": 88, "ș": 89, "ț": 90, "ə": 91, "ʷ": 92, "ʽ": 93, "ː": 94, "́": 95, "̇": 96, "ϙ": 97, "а": 98, "б": 99, "в": 100, "г": 101, "д": 102, "е": 103, "и": 104, "й": 105, "к": 106, "л": 107, "н": 108, "о": 109, "п": 110, "р": 111, "с": 112, "т": 113, "ч": 114, "ш": 115, "ы": 116, "ь": 117, "ю": 118, "я": 119, "ё": 120, "ү": 121, "ө": 122, "ਆ": 123, "ਘ": 124, "ਤ": 125, "ਨ": 126, "ਮ": 127, "ਸ": 128, "ṁ": 129, "ṃ": 130, "ṇ": 131, "ồ": 132, "‐": 133, "‑": 134, "―": 135, "": 136, "[UNK]": 136, "[PAD]": 137}