Commit
•
a6f2dbc
1
Parent(s):
90457b5
add tokenizer
Browse files- special_tokens_map.json +1 -0
- tokenizer_config.json +1 -0
- vocab.json +1 -0
special_tokens_map.json
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
{"bos_token": "<s>", "eos_token": "</s>", "unk_token": "[UNK]", "pad_token": "[PAD]"}
|
tokenizer_config.json
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
{"unk_token": "[UNK]", "bos_token": "<s>", "eos_token": "</s>", "pad_token": "[PAD]", "do_lower_case": false, "word_delimiter_token": "|", "tokenizer_class": "Wav2Vec2CTCTokenizer"}
|
vocab.json
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
{"h": 0, "ы": 1, "r": 2, "ʻ": 3, "ł": 4, "ת": 5, "p": 6, "ס": 7, "ã": 8, "т": 9, "ם": 10, "ź": 11, "í": 12, "x": 13, "ä": 15, "מ": 16, "—": 17, "'": 18, "н": 19, "m": 20, "ê": 21, "k": 22, "ü": 23, "ּ": 24, "ć": 25, "d": 26, "š": 27, "–": 28, "ó": 29, "ž": 30, "ו": 31, "u": 32, "t": 33, "l": 34, "ө": 35, "о": 36, "w": 37, "f": 38, "ø": 39, "o": 40, "נ": 41, "к": 42, "¿": 43, "ш": 44, "°": 45, "б": 46, "n": 47, "é": 48, "y": 49, "ú": 50, "ë": 51, "ö": 52, "ñ": 53, "毵": 54, "ל": 55, "ş": 56, "a": 57, "â": 58, "œ": 59, "ò": 60, "j": 61, "g": 62, "р": 63, "ő": 64, "ָ": 65, "z": 66, "ֹ": 67, "כ": 68, "e": 69, "ė": 70, "á": 71, "s": 72, "¡": 73, "ְ": 74, "q": 75, "v": 76, "и": 77, "п": 78, "ש": 79, "=": 80, "´": 81, "ð": 82, "ō": 83, "i": 84, "ə": 85, "ū": 86, "c": 87, "е": 88, "ô": 89, "а": 90, "ֵ": 91, "ү": 92, "ï": 93, "л": 94, "ě": 95, "ִ": 96, "b": 97, "’": 98, "|": 14, "[UNK]": 99, "[PAD]": 100}
|