20220507-074029 / vocab.json
lilitket's picture
add tokenizer
16bb557
{"'": 1, "(": 2, ")": 3, ",": 4, "-": 5, ".": 6, "0": 7, "1": 8, "2": 9, "3": 10, "4": 11, "5": 12, "6": 13, "7": 14, "8": 15, "9": 16, ":": 17, "a": 18, "b": 19, "c": 20, "d": 21, "e": 22, "f": 23, "g": 24, "h": 25, "i": 26, "j": 27, "k": 28, "l": 29, "m": 30, "n": 31, "o": 32, "p": 33, "r": 34, "s": 35, "t": 36, "u": 37, "v": 38, "w": 39, "x": 40, "y": 41, "Ա": 42, "Բ": 43, "Գ": 44, "Դ": 45, "Ե": 46, "Զ": 47, "Է": 48, "Ը": 49, "Թ": 50, "Ժ": 51, "Ի": 52, "Լ": 53, "Խ": 54, "Ծ": 55, "Կ": 56, "Հ": 57, "Ձ": 58, "Ճ": 59, "Մ": 60, "Յ": 61, "Ն": 62, "Շ": 63, "Ո": 64, "Չ": 65, "Պ": 66, "Ջ": 67, "Ռ": 68, "Ս": 69, "Վ": 70, "Տ": 71, "Ց": 72, "Փ": 73, "Ք": 74, "Օ": 75, "Ֆ": 76, "՛": 77, "՝": 78, "ա": 79, "բ": 80, "գ": 81, "դ": 82, "ե": 83, "զ": 84, "է": 85, "ը": 86, "թ": 87, "ժ": 88, "ի": 89, "լ": 90, "խ": 91, "ծ": 92, "կ": 93, "հ": 94, "ձ": 95, "ղ": 96, "ճ": 97, "մ": 98, "յ": 99, "ն": 100, "շ": 101, "ո": 102, "չ": 103, "պ": 104, "ջ": 105, "ռ": 106, "ս": 107, "վ": 108, "տ": 109, "ր": 110, "ց": 111, "ւ": 112, "փ": 113, "ք": 114, "օ": 115, "ֆ": 116, "և": 117, "։": 118, "|": 0, "[UNK]": 119, "[PAD]": 120}