Commit
•
468e15b
1
Parent(s):
bbde6cb
add tokenizer
Browse files- special_tokens_map.json +1 -0
- tokenizer_config.json +1 -0
- vocab.json +1 -0
special_tokens_map.json
ADDED
@@ -0,0 +1 @@
|
|
|
|
|
1 |
+
{"bos_token": "<s>", "eos_token": "</s>", "unk_token": "[UNK]", "pad_token": "[PAD]"}
|
tokenizer_config.json
ADDED
@@ -0,0 +1 @@
|
|
|
|
|
1 |
+
{"unk_token": "[UNK]", "bos_token": "<s>", "eos_token": "</s>", "pad_token": "[PAD]", "do_lower_case": false, "word_delimiter_token": "|", "tokenizer_class": "Wav2Vec2CTCTokenizer"}
|
vocab.json
ADDED
@@ -0,0 +1 @@
|
|
|
|
|
1 |
+
{"ø": 0, "ו": 1, "к": 2, "ş": 3, "ʽ": 4, "ü": 5, "}": 6, "ミ": 7, "ʷ": 8, "à": 9, "ل": 10, "a": 11, "כ": 12, "ੰ": 13, "…": 14, "罒": 15, "c": 16, "→": 17, "ū": 18, "ק": 19, "ț": 20, "ò": 21, "ù": 22, "ا": 23, "ਸ": 24, "ְ": 25, "ñ": 26, "é": 27, "ֹ": 28, "ː": 29, "ō": 30, "ð": 31, "т": 32, "ì": 33, "ת": 34, "ө": 35, "œ": 36, "ź": 37, "f": 38, "ʻ": 39, "ϙ": 40, "с": 41, "`": 42, "›": 43, "ʿ": 44, "=": 45, "ב": 46, "ё": 47, "w": 48, "ú": 49, "ŏ": 50, "夷": 51, "ô": 52, "ń": 53, "́": 54, "r": 55, "u": 56, "ï": 57, "í": 58, "ę": 59, "ү": 60, "л": 61, "п": 62, "ä": 63, "ı": 64, "ة": 65, "箱": 66, "מ": 67, "¨": 68, "н": 69, "р": 70, "v": 71, "'": 72, "ּ": 73, "»": 74, "ṇ": 75, "鮓": 76, "„": 77, "و": 78, "ồ": 79, "б": 80, "č": 81, "蝦": 82, "¿": 83, "þ": 84, "®": 85, "о": 86, "ł": 87, "の": 88, "良": 89, "đ": 90, "е": 91, "ш": 92, "ر": 93, "ṃ": 94, "י": 95, "ª": 96, "t": 97, "̇": 98, "山": 99, "ṁ": 100, "ਨ": 101, "ğ": 102, "ש": 103, "s": 104, "ő": 105, "ý": 106, "本": 107, "ム": 108, "ら": 109, "z": 110, "―": 111, "ひ": 112, "ß": 113, "日": 114, "≫": 115, "’": 116, "–": 117, "õ": 118, "ă": 119, "m": 120, "ö": 121, "ы": 122, "ã": 123, "p": 124, "я": 125, "ר": 126, "&": 127, "ś": 128, "ਾ": 129, "ָ": 130, "㓁": 131, "ș": 132, "—": 133, "ヒ": 134, "°": 135, "周": 136, "ي": 137, "â": 138, "y": 139, "ė": 140, "_": 141, "ë": 143, "æ": 144, "真": 145, "ب": 146, "l": 147, "´": 148, "î": 149, "ə": 150, "г": 151, "~": 152, "i": 153, "肌": 154, "å": 155, "‑": 156, "ć": 157, "戌": 158, "ਤ": 159, "ю": 160, "罓": 161, "q": 162, "ִ": 163, "比": 164, "ֵ": 165, "ó": 166, "≪": 167, "肋": 168, "生": 169, "fi": 170, "ç": 171, "ā": 172, "š": 173, "й": 174, "h": 175, "ਆ": 176, "鮨": 177, "": 178, "j": 179, "背": 180, "|": 142, "·": 182, "口": 183, "申": 184, "d": 185, "ਮ": 186, "‧": 187, "ě": 188, "k": 189, "נ": 190, "д": 191, "«": 192, "û": 193, "ラ": 194, "ל": 195, "ਿ": 196, "网": 197, "し": 198, "ч": 199, "د": 200, "g": 201, "ž": 202, "ī": 203, "ם": 204, "ゴ": 205, "ס": 206, "毵": 207, "b": 208, "ه": 209, "o": 210, "x": 211, "م": 212, "‹": 213, "а": 214, "ř": 215, "e": 216, "ਘ": 217, "ь": 218, "消": 219, "n": 220, "в": 221, "ذ": 222, "¡": 223, "á": 224, "ê": 225, "$": 226, "ה": 227, "‐": 228, "и": 229, "ż": 230, "{": 231, "[UNK]": 231, "[PAD]": 232}
|