hellosimple's picture
add tokenizer
ae4c288
{
"'": 19,
"[PAD]": 29,
"[UNK]": 28,
"a": 27,
"b": 21,
"c": 26,
"d": 5,
"e": 18,
"f": 15,
"g": 1,
"h": 24,
"i": 17,
"j": 22,
"k": 6,
"l": 2,
"m": 11,
"n": 10,
"o": 13,
"p": 16,
"q": 12,
"r": 20,
"s": 9,
"t": 4,
"u": 14,
"v": 7,
"w": 3,
"x": 0,
"y": 8,
"z": 23,
"|": 25
}