add tokenizer
Browse files- special_tokens_map.json +1 -0
- tokenizer_config.json +1 -0
- vocab.json +1 -0
special_tokens_map.json
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
| 1 |
+
{"bos_token": "<s>", "eos_token": "</s>", "unk_token": "[UNK]", "pad_token": "[PAD]"}
|
tokenizer_config.json
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
| 1 |
+
{"unk_token": "[UNK]", "bos_token": "<s>", "eos_token": "</s>", "pad_token": "[PAD]", "do_lower_case": false, "word_delimiter_token": "|", "tokenizer_class": "Wav2Vec2CTCTokenizer"}
|
vocab.json
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
| 1 |
+
{"ง": 0, "ึ": 1, "ห": 2, "ซ": 3, "ศ": 4, "ฎ": 5, "ษ": 6, "ฌ": 7, "อ": 8, "ำ": 9, "ๅ": 10, "ฉ": 11, "์": 12, "ผ": 13, "ื": 14, "ณ": 15, "ล": 16, "ไ": 17, "ข": 18, "ด": 19, "ส": 20, "ฯ": 21, "น": 22, "ะ": 23, "ี": 24, "โ": 25, "ธ": 26, "ว": 27, "ฐ": 28, "ฬ": 29, "บ": 30, "ป": 31, "ภ": 32, "๋": 33, "ช": 34, "ิ": 35, "ย": 36, "ม": 37, "เ": 38, "้": 39, "ั": 40, "ฝ": 41, "ต": 42, "่": 43, "ู": 44, "ฆ": 45, "ใ": 46, "ุ": 47, "ค": 48, "ฮ": 49, "ฟ": 50, "๊": 51, "ญ": 52, "ฤ": 53, "ฒ": 54, "ถ": 55, "ฏ": 56, "็": 57, "ฑ": 58, "ฃ": 59, "จ": 60, "า": 61, "ก": 63, "ร": 64, "ท": 65, "พ": 66, "แ": 67, "|": 62, "[UNK]": 68, "[PAD]": 69}
|