add tokenizer

added_tokens.json

@@ -0,0 +1 @@
+{"<s>": 73, "</s>": 74}

special_tokens_map.json

@@ -0,0 +1 @@
+{"bos_token": "<s>", "eos_token": "</s>", "unk_token": "[UNK]", "pad_token": "[PAD]", "additional_special_tokens": [{"content": "<s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, {"content": "</s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}]}

tokenizer_config.json

@@ -0,0 +1 @@
+{"unk_token": "[UNK]", "bos_token": "<s>", "eos_token": "</s>", "pad_token": "[PAD]", "do_lower_case": false, "word_delimiter_token": "|", "special_tokens_map_file": null, "tokenizer_file": null, "name_or_path": "./", "tokenizer_class": "Wav2Vec2CTCTokenizer"}

vocab.json

@@ -1 +1 @@
-{"A": 1, "B": 2, "C": 3, "D": 4, "E": 5, "F": 6, "G": 7, "H": 8, "I": 9, "J": 10, "K": 11, "L": 12, "M": 13, "N": 14, "O": 15, "P": 16, "Q": 17, "R": 18, "S": 19, "T": 20, "U": 21, "V": 22, "W": 23, "X": 24, "Y": 25, "Z": 26, "a": 27, "b": 28, "c": 29, "d": 30, "e": 31, "f": 32, "g": 33, "h": 34, "i": 35, "j": 36, "k": 37, "l": 38, "m": 39, "n": 40, "o": 41, "p": 42, "r": 43, "s": 44, "t": 45, "u": 46, "v": 47, "w": 48, "x": 49, "y": 50, "z": 51, "\u00c7": 52, "\u00d6": 53, "\u00dc": 54, "\u00e2": 55, "\u00e7": 56, "\u00eb": 57, "\u00ee": 58, "\u00f6": 59, "\u00fc": 60, "\u011f": 61, "\u0130": 62, "\u0131": 63, "\u015e": 64, "\u015f": 65, "|": 0, "[UNK]": 67, "[PAD]": 68}
+{"ក": 1, "ខ": 2, "គ": 3, "ឃ": 4, "ង": 5, "ច": 6, "ឆ": 7, "ជ": 8, "ឈ": 9, "ញ": 10, "ដ": 11, "ឋ": 12, "ឌ": 13, "ឍ": 14, "ណ": 15, "ត": 16, "ថ": 17, "ទ": 18, "ធ": 19, "ន": 20, "ប": 21, "ផ": 22, "ព": 23, "ភ": 24, "ម": 25, "យ": 26, "រ": 27, "ល": 28, "វ": 29, "ស": 30, "ហ": 31, "ឡ": 32, "អ": 33, "ឥ": 34, "ឧ": 35, "ឪ": 36, "ឫ": 37, "ឬ": 38, "ឭ": 39, "ឮ": 40, "ឯ": 41, "ឱ": 42, "ា": 43, "ិ": 44, "ី": 45, "ឹ": 46, "ឺ": 47, "ុ": 48, "ូ": 49, "ួ": 50, "ើ": 51, "ឿ": 52, "ៀ": 53, "េ": 54, "ែ": 55, "ៃ": 56, "ោ": 57, "ៅ": 58, "ំ": 59, "ះ": 60, "ៈ": 61, "៉": 62, "៊": 63, "់": 64, "៌": 65, "៍": 66, "៎": 67, "៏": 68, "័": 69, "្": 70, "|": 0, "[UNK]": 71, "[PAD]": 72}