Jethuestad's picture
add tokenizer
74b9a3f
{
"'": 11,
"A": 6,
"B": 24,
"C": 0,
"D": 10,
"E": 17,
"F": 9,
"G": 13,
"H": 14,
"I": 8,
"J": 2,
"K": 16,
"L": 22,
"M": 26,
"N": 19,
"O": 1,
"P": 20,
"Q": 4,
"R": 15,
"S": 25,
"T": 3,
"U": 21,
"V": 18,
"W": 5,
"X": 12,
"Y": 7,
"Z": 23,
"[PAD]": 29,
"[UNK]": 28,
"|": 27
}