HarryPotter09 commited on
Commit
7142e7d
1 Parent(s): 1e63e70

add tokenizer

Browse files
Files changed (1) hide show
  1. vocab.json +1 -1
vocab.json CHANGED
@@ -1 +1 @@
1
- {"<pad>": 0, "<unk>": 1, "|": 2, "A": 3, "B": 4, "C": 5, "D": 6, "E": 7, "F": 8, "G": 9, "H": 10, "I": 11, "J": 12, "K": 13, "L": 14, "M": 15, "N": 16, "O": 17, "P": 18, "Q": 19, "R": 20, "S": 21, "T": 22, "U": 23, "V": 24, "W": 25, "X": 26, "Y": 27, "Z": 28, "Å": 29, "Ä": 30, "Ö": 31}
 
1
+ {"<pad>": 0, "<s>": 1, "</s>": 2, "<unk>": 3, "|": 4, "A": 5, "B": 6, "C": 7, "D": 8, "E": 9, "F": 10, "G": 11, "H": 12, "I": 13, "J": 14, "K": 15, "L": 16, "M": 17, "N": 18, "O": 19, "P": 20, "Q": 21, "R": 22, "S": 23, "T": 24, "U": 25, "V": 26, "W": 27, "X": 28, "Y": 29, "Z": 30, "Å": 31, "Ä": 32, "Ö": 33}