SreyanG-NVIDIA commited on
Commit
fabb7f9
·
1 Parent(s): 07ab43e

add tokenizer

Browse files
Files changed (1) hide show
  1. vocab.json +1 -1
vocab.json CHANGED
@@ -1 +1 @@
1
- {"M": 0, "F": 1, "L": 2, "S": 3, "H": 4, "A": 5, "K": 6, "Z": 7, "R": 8, "E": 9, "N": 10, "O": 11, "T": 12, "C": 13, "I": 14, "Y": 15, "P": 16, "J": 17, "V": 19, "G": 20, "Q": 21, "W": 22, "D": 23, "X": 24, "B": 25, "'": 26, "U": 27, "|": 18, "[UNK]": 28, "[PAD]": 29}
 
1
+ {"O": 0, "J": 1, "'": 2, "P": 3, "F": 4, "V": 5, "U": 6, "R": 7, "C": 9, "W": 10, "A": 11, "Q": 12, "I": 13, "D": 14, "K": 15, "M": 16, "X": 17, "T": 18, "N": 19, "H": 20, "B": 21, "E": 22, "Y": 23, "Z": 24, "L": 25, "G": 26, "S": 27, "|": 8, "[UNK]": 28, "[PAD]": 29}