add tokenizer
Browse files- tokenizer.json +11 -11
- vocab.txt +7 -7
tokenizer.json
CHANGED
@@ -183,17 +183,17 @@
|
|
183 |
"r": 56,
|
184 |
"s": 57,
|
185 |
"##\"": 58,
|
186 |
-
"##
|
187 |
-
"##
|
188 |
-
"##
|
189 |
-
"##
|
190 |
-
"##
|
191 |
-
"##
|
192 |
-
"##
|
193 |
-
"##
|
194 |
-
"##
|
195 |
-
"##
|
196 |
-
"##
|
197 |
}
|
198 |
}
|
199 |
}
|
|
|
183 |
"r": 56,
|
184 |
"s": 57,
|
185 |
"##\"": 58,
|
186 |
+
"##N": 59,
|
187 |
+
"##B": 60,
|
188 |
+
"##[": 61,
|
189 |
+
"##F": 62,
|
190 |
+
"##O": 63,
|
191 |
+
"##S": 64,
|
192 |
+
"##E": 65,
|
193 |
+
"##P": 66,
|
194 |
+
"##]": 67,
|
195 |
+
"##c": 68,
|
196 |
+
"##C": 69
|
197 |
}
|
198 |
}
|
199 |
}
|
vocab.txt
CHANGED
@@ -57,14 +57,14 @@ o
|
|
57 |
r
|
58 |
s
|
59 |
##"
|
60 |
-
##c
|
61 |
-
##C
|
62 |
-
##S
|
63 |
-
##E
|
64 |
-
##P
|
65 |
-
##]
|
66 |
##N
|
67 |
-
##F
|
68 |
##B
|
69 |
##[
|
|
|
70 |
##O
|
|
|
|
|
|
|
|
|
|
|
|
|
|
57 |
r
|
58 |
s
|
59 |
##"
|
|
|
|
|
|
|
|
|
|
|
|
|
60 |
##N
|
|
|
61 |
##B
|
62 |
##[
|
63 |
+
##F
|
64 |
##O
|
65 |
+
##S
|
66 |
+
##E
|
67 |
+
##P
|
68 |
+
##]
|
69 |
+
##c
|
70 |
+
##C
|