Upload tokenizer
Browse files- added_tokens.json +300 -301
- special_tokens_map.json +0 -15
added_tokens.json
CHANGED
@@ -1,303 +1,302 @@
|
|
1 |
{
|
2 |
-
"
|
3 |
-
"<
|
4 |
-
"<
|
5 |
-
"<
|
6 |
-
"<
|
7 |
-
"<
|
8 |
-
"<
|
9 |
-
"<
|
10 |
-
"<
|
11 |
-
"<
|
12 |
-
"<
|
13 |
-
"<
|
14 |
-
"<
|
15 |
-
"<
|
16 |
-
"<
|
17 |
-
"<
|
18 |
-
"<
|
19 |
-
"<
|
20 |
-
"<
|
21 |
-
"<
|
22 |
-
"<
|
23 |
-
"<
|
24 |
-
"<
|
25 |
-
"<
|
26 |
-
"<
|
27 |
-
"<
|
28 |
-
"<
|
29 |
-
"<
|
30 |
-
"<
|
31 |
-
"<
|
32 |
-
"<
|
33 |
-
"<
|
34 |
-
"<
|
35 |
-
"<
|
36 |
-
"<
|
37 |
-
"<
|
38 |
-
"<
|
39 |
-
"<
|
40 |
-
"<
|
41 |
-
"<
|
42 |
-
"<
|
43 |
-
"<
|
44 |
-
"<
|
45 |
-
"<
|
46 |
-
"<
|
47 |
-
"<
|
48 |
-
"<
|
49 |
-
"<
|
50 |
-
"<
|
51 |
-
"<
|
52 |
-
"<
|
53 |
-
"<
|
54 |
-
"<
|
55 |
-
"<
|
56 |
-
"<
|
57 |
-
"<
|
58 |
-
"<
|
59 |
-
"<
|
60 |
-
"<
|
61 |
-
"<
|
62 |
-
"<
|
63 |
-
"<
|
64 |
-
"<
|
65 |
-
"<
|
66 |
-
"<
|
67 |
-
"<
|
68 |
-
"<
|
69 |
-
"<
|
70 |
-
"<
|
71 |
-
"<
|
72 |
-
"<
|
73 |
-
"<
|
74 |
-
"<
|
75 |
-
"<
|
76 |
-
"<
|
77 |
-
"<
|
78 |
-
"<
|
79 |
-
"<
|
80 |
-
"<
|
81 |
-
"<
|
82 |
-
"<
|
83 |
-
"<
|
84 |
-
"<
|
85 |
-
"<
|
86 |
-
"<
|
87 |
-
"<
|
88 |
-
"<
|
89 |
-
"<
|
90 |
-
"<
|
91 |
-
"<
|
92 |
-
"<
|
93 |
-
"<
|
94 |
-
"<
|
95 |
-
"<
|
96 |
-
"<
|
97 |
-
"<
|
98 |
-
"<
|
99 |
-
"<
|
100 |
-
"<
|
101 |
-
"<
|
102 |
-
"<
|
103 |
-
"<
|
104 |
-
"<
|
105 |
-
"<
|
106 |
-
"<
|
107 |
-
"<
|
108 |
-
"<
|
109 |
-
"<
|
110 |
-
"<
|
111 |
-
"<
|
112 |
-
"<
|
113 |
-
"<
|
114 |
-
"<
|
115 |
-
"<
|
116 |
-
"<
|
117 |
-
"<
|
118 |
-
"<
|
119 |
-
"<
|
120 |
-
"<
|
121 |
-
"<
|
122 |
-
"<
|
123 |
-
"<
|
124 |
-
"<
|
125 |
-
"<
|
126 |
-
"<
|
127 |
-
"<
|
128 |
-
"<
|
129 |
-
"<
|
130 |
-
"<
|
131 |
-
"<
|
132 |
-
"<
|
133 |
-
"<
|
134 |
-
"<
|
135 |
-
"<
|
136 |
-
"<
|
137 |
-
"<
|
138 |
-
"<
|
139 |
-
"<
|
140 |
-
"<
|
141 |
-
"<
|
142 |
-
"<
|
143 |
-
"<
|
144 |
-
"<
|
145 |
-
"<
|
146 |
-
"<
|
147 |
-
"<
|
148 |
-
"<
|
149 |
-
"<
|
150 |
-
"<
|
151 |
-
"<
|
152 |
-
"<
|
153 |
-
"<
|
154 |
-
"<
|
155 |
-
"<
|
156 |
-
"<
|
157 |
-
"<
|
158 |
-
"<
|
159 |
-
"<
|
160 |
-
"<
|
161 |
-
"<
|
162 |
-
"<
|
163 |
-
"<
|
164 |
-
"<
|
165 |
-
"<
|
166 |
-
"<
|
167 |
-
"<
|
168 |
-
"<
|
169 |
-
"<
|
170 |
-
"<
|
171 |
-
"<
|
172 |
-
"<
|
173 |
-
"<
|
174 |
-
"<
|
175 |
-
"<
|
176 |
-
"<
|
177 |
-
"<
|
178 |
-
"<
|
179 |
-
"<
|
180 |
-
"<
|
181 |
-
"<
|
182 |
-
"<
|
183 |
-
"<
|
184 |
-
"<
|
185 |
-
"<
|
186 |
-
"<
|
187 |
-
"<
|
188 |
-
"<
|
189 |
-
"<
|
190 |
-
"<
|
191 |
-
"<
|
192 |
-
"<
|
193 |
-
"<
|
194 |
-
"<
|
195 |
-
"<
|
196 |
-
"<
|
197 |
-
"<
|
198 |
-
"<
|
199 |
-
"<
|
200 |
-
"<
|
201 |
-
"<
|
202 |
-
"<
|
203 |
-
"<
|
204 |
-
"<
|
205 |
-
"<
|
206 |
-
"<
|
207 |
-
"<
|
208 |
-
"<
|
209 |
-
"<
|
210 |
-
"<
|
211 |
-
"<
|
212 |
-
"<
|
213 |
-
"<
|
214 |
-
"<
|
215 |
-
"<
|
216 |
-
"<
|
217 |
-
"<
|
218 |
-
"<
|
219 |
-
"<
|
220 |
-
"<
|
221 |
-
"<
|
222 |
-
"<
|
223 |
-
"<
|
224 |
-
"<
|
225 |
-
"<
|
226 |
-
"<
|
227 |
-
"<
|
228 |
-
"<
|
229 |
-
"<
|
230 |
-
"<
|
231 |
-
"<
|
232 |
-
"<
|
233 |
-
"<
|
234 |
-
"<
|
235 |
-
"<
|
236 |
-
"<
|
237 |
-
"<
|
238 |
-
"<
|
239 |
-
"<
|
240 |
-
"<
|
241 |
-
"<
|
242 |
-
"<
|
243 |
-
"<
|
244 |
-
"<
|
245 |
-
"<
|
246 |
-
"<
|
247 |
-
"<
|
248 |
-
"<
|
249 |
-
"<
|
250 |
-
"<
|
251 |
-
"<
|
252 |
-
"<
|
253 |
-
"<
|
254 |
-
"<
|
255 |
-
"<
|
256 |
-
"<
|
257 |
-
"<
|
258 |
-
"<
|
259 |
-
"<
|
260 |
-
"<
|
261 |
-
"<
|
262 |
-
"<
|
263 |
-
"<
|
264 |
-
"<
|
265 |
-
"<
|
266 |
-
"<
|
267 |
-
"<
|
268 |
-
"<
|
269 |
-
"<
|
270 |
-
"<
|
271 |
-
"<
|
272 |
-
"<
|
273 |
-
"<
|
274 |
-
"<
|
275 |
-
"<
|
276 |
-
"<
|
277 |
-
"<
|
278 |
-
"<
|
279 |
-
"<
|
280 |
-
"<
|
281 |
-
"<
|
282 |
-
"<
|
283 |
-
"<
|
284 |
-
"<
|
285 |
-
"<
|
286 |
-
"<
|
287 |
-
"<
|
288 |
-
"<
|
289 |
-
"<
|
290 |
-
"<
|
291 |
-
"<
|
292 |
-
"<
|
293 |
-
"<
|
294 |
-
"<
|
295 |
-
"<
|
296 |
-
"<
|
297 |
-
"<
|
298 |
-
"<
|
299 |
-
"<
|
300 |
-
"<
|
301 |
-
"<
|
302 |
-
"<extra_id_9>": 256291
|
303 |
}
|
|
|
1 |
{
|
2 |
+
"<extra_id_0>": 256299,
|
3 |
+
"<extra_id_100>": 256199,
|
4 |
+
"<extra_id_101>": 256198,
|
5 |
+
"<extra_id_102>": 256197,
|
6 |
+
"<extra_id_103>": 256196,
|
7 |
+
"<extra_id_104>": 256195,
|
8 |
+
"<extra_id_105>": 256194,
|
9 |
+
"<extra_id_106>": 256193,
|
10 |
+
"<extra_id_107>": 256192,
|
11 |
+
"<extra_id_108>": 256191,
|
12 |
+
"<extra_id_109>": 256190,
|
13 |
+
"<extra_id_10>": 256289,
|
14 |
+
"<extra_id_110>": 256189,
|
15 |
+
"<extra_id_111>": 256188,
|
16 |
+
"<extra_id_112>": 256187,
|
17 |
+
"<extra_id_113>": 256186,
|
18 |
+
"<extra_id_114>": 256185,
|
19 |
+
"<extra_id_115>": 256184,
|
20 |
+
"<extra_id_116>": 256183,
|
21 |
+
"<extra_id_117>": 256182,
|
22 |
+
"<extra_id_118>": 256181,
|
23 |
+
"<extra_id_119>": 256180,
|
24 |
+
"<extra_id_11>": 256288,
|
25 |
+
"<extra_id_120>": 256179,
|
26 |
+
"<extra_id_121>": 256178,
|
27 |
+
"<extra_id_122>": 256177,
|
28 |
+
"<extra_id_123>": 256176,
|
29 |
+
"<extra_id_124>": 256175,
|
30 |
+
"<extra_id_125>": 256174,
|
31 |
+
"<extra_id_126>": 256173,
|
32 |
+
"<extra_id_127>": 256172,
|
33 |
+
"<extra_id_128>": 256171,
|
34 |
+
"<extra_id_129>": 256170,
|
35 |
+
"<extra_id_12>": 256287,
|
36 |
+
"<extra_id_130>": 256169,
|
37 |
+
"<extra_id_131>": 256168,
|
38 |
+
"<extra_id_132>": 256167,
|
39 |
+
"<extra_id_133>": 256166,
|
40 |
+
"<extra_id_134>": 256165,
|
41 |
+
"<extra_id_135>": 256164,
|
42 |
+
"<extra_id_136>": 256163,
|
43 |
+
"<extra_id_137>": 256162,
|
44 |
+
"<extra_id_138>": 256161,
|
45 |
+
"<extra_id_139>": 256160,
|
46 |
+
"<extra_id_13>": 256286,
|
47 |
+
"<extra_id_140>": 256159,
|
48 |
+
"<extra_id_141>": 256158,
|
49 |
+
"<extra_id_142>": 256157,
|
50 |
+
"<extra_id_143>": 256156,
|
51 |
+
"<extra_id_144>": 256155,
|
52 |
+
"<extra_id_145>": 256154,
|
53 |
+
"<extra_id_146>": 256153,
|
54 |
+
"<extra_id_147>": 256152,
|
55 |
+
"<extra_id_148>": 256151,
|
56 |
+
"<extra_id_149>": 256150,
|
57 |
+
"<extra_id_14>": 256285,
|
58 |
+
"<extra_id_150>": 256149,
|
59 |
+
"<extra_id_151>": 256148,
|
60 |
+
"<extra_id_152>": 256147,
|
61 |
+
"<extra_id_153>": 256146,
|
62 |
+
"<extra_id_154>": 256145,
|
63 |
+
"<extra_id_155>": 256144,
|
64 |
+
"<extra_id_156>": 256143,
|
65 |
+
"<extra_id_157>": 256142,
|
66 |
+
"<extra_id_158>": 256141,
|
67 |
+
"<extra_id_159>": 256140,
|
68 |
+
"<extra_id_15>": 256284,
|
69 |
+
"<extra_id_160>": 256139,
|
70 |
+
"<extra_id_161>": 256138,
|
71 |
+
"<extra_id_162>": 256137,
|
72 |
+
"<extra_id_163>": 256136,
|
73 |
+
"<extra_id_164>": 256135,
|
74 |
+
"<extra_id_165>": 256134,
|
75 |
+
"<extra_id_166>": 256133,
|
76 |
+
"<extra_id_167>": 256132,
|
77 |
+
"<extra_id_168>": 256131,
|
78 |
+
"<extra_id_169>": 256130,
|
79 |
+
"<extra_id_16>": 256283,
|
80 |
+
"<extra_id_170>": 256129,
|
81 |
+
"<extra_id_171>": 256128,
|
82 |
+
"<extra_id_172>": 256127,
|
83 |
+
"<extra_id_173>": 256126,
|
84 |
+
"<extra_id_174>": 256125,
|
85 |
+
"<extra_id_175>": 256124,
|
86 |
+
"<extra_id_176>": 256123,
|
87 |
+
"<extra_id_177>": 256122,
|
88 |
+
"<extra_id_178>": 256121,
|
89 |
+
"<extra_id_179>": 256120,
|
90 |
+
"<extra_id_17>": 256282,
|
91 |
+
"<extra_id_180>": 256119,
|
92 |
+
"<extra_id_181>": 256118,
|
93 |
+
"<extra_id_182>": 256117,
|
94 |
+
"<extra_id_183>": 256116,
|
95 |
+
"<extra_id_184>": 256115,
|
96 |
+
"<extra_id_185>": 256114,
|
97 |
+
"<extra_id_186>": 256113,
|
98 |
+
"<extra_id_187>": 256112,
|
99 |
+
"<extra_id_188>": 256111,
|
100 |
+
"<extra_id_189>": 256110,
|
101 |
+
"<extra_id_18>": 256281,
|
102 |
+
"<extra_id_190>": 256109,
|
103 |
+
"<extra_id_191>": 256108,
|
104 |
+
"<extra_id_192>": 256107,
|
105 |
+
"<extra_id_193>": 256106,
|
106 |
+
"<extra_id_194>": 256105,
|
107 |
+
"<extra_id_195>": 256104,
|
108 |
+
"<extra_id_196>": 256103,
|
109 |
+
"<extra_id_197>": 256102,
|
110 |
+
"<extra_id_198>": 256101,
|
111 |
+
"<extra_id_199>": 256100,
|
112 |
+
"<extra_id_19>": 256280,
|
113 |
+
"<extra_id_1>": 256298,
|
114 |
+
"<extra_id_200>": 256099,
|
115 |
+
"<extra_id_201>": 256098,
|
116 |
+
"<extra_id_202>": 256097,
|
117 |
+
"<extra_id_203>": 256096,
|
118 |
+
"<extra_id_204>": 256095,
|
119 |
+
"<extra_id_205>": 256094,
|
120 |
+
"<extra_id_206>": 256093,
|
121 |
+
"<extra_id_207>": 256092,
|
122 |
+
"<extra_id_208>": 256091,
|
123 |
+
"<extra_id_209>": 256090,
|
124 |
+
"<extra_id_20>": 256279,
|
125 |
+
"<extra_id_210>": 256089,
|
126 |
+
"<extra_id_211>": 256088,
|
127 |
+
"<extra_id_212>": 256087,
|
128 |
+
"<extra_id_213>": 256086,
|
129 |
+
"<extra_id_214>": 256085,
|
130 |
+
"<extra_id_215>": 256084,
|
131 |
+
"<extra_id_216>": 256083,
|
132 |
+
"<extra_id_217>": 256082,
|
133 |
+
"<extra_id_218>": 256081,
|
134 |
+
"<extra_id_219>": 256080,
|
135 |
+
"<extra_id_21>": 256278,
|
136 |
+
"<extra_id_220>": 256079,
|
137 |
+
"<extra_id_221>": 256078,
|
138 |
+
"<extra_id_222>": 256077,
|
139 |
+
"<extra_id_223>": 256076,
|
140 |
+
"<extra_id_224>": 256075,
|
141 |
+
"<extra_id_225>": 256074,
|
142 |
+
"<extra_id_226>": 256073,
|
143 |
+
"<extra_id_227>": 256072,
|
144 |
+
"<extra_id_228>": 256071,
|
145 |
+
"<extra_id_229>": 256070,
|
146 |
+
"<extra_id_22>": 256277,
|
147 |
+
"<extra_id_230>": 256069,
|
148 |
+
"<extra_id_231>": 256068,
|
149 |
+
"<extra_id_232>": 256067,
|
150 |
+
"<extra_id_233>": 256066,
|
151 |
+
"<extra_id_234>": 256065,
|
152 |
+
"<extra_id_235>": 256064,
|
153 |
+
"<extra_id_236>": 256063,
|
154 |
+
"<extra_id_237>": 256062,
|
155 |
+
"<extra_id_238>": 256061,
|
156 |
+
"<extra_id_239>": 256060,
|
157 |
+
"<extra_id_23>": 256276,
|
158 |
+
"<extra_id_240>": 256059,
|
159 |
+
"<extra_id_241>": 256058,
|
160 |
+
"<extra_id_242>": 256057,
|
161 |
+
"<extra_id_243>": 256056,
|
162 |
+
"<extra_id_244>": 256055,
|
163 |
+
"<extra_id_245>": 256054,
|
164 |
+
"<extra_id_246>": 256053,
|
165 |
+
"<extra_id_247>": 256052,
|
166 |
+
"<extra_id_248>": 256051,
|
167 |
+
"<extra_id_249>": 256050,
|
168 |
+
"<extra_id_24>": 256275,
|
169 |
+
"<extra_id_250>": 256049,
|
170 |
+
"<extra_id_251>": 256048,
|
171 |
+
"<extra_id_252>": 256047,
|
172 |
+
"<extra_id_253>": 256046,
|
173 |
+
"<extra_id_254>": 256045,
|
174 |
+
"<extra_id_255>": 256044,
|
175 |
+
"<extra_id_256>": 256043,
|
176 |
+
"<extra_id_257>": 256042,
|
177 |
+
"<extra_id_258>": 256041,
|
178 |
+
"<extra_id_259>": 256040,
|
179 |
+
"<extra_id_25>": 256274,
|
180 |
+
"<extra_id_260>": 256039,
|
181 |
+
"<extra_id_261>": 256038,
|
182 |
+
"<extra_id_262>": 256037,
|
183 |
+
"<extra_id_263>": 256036,
|
184 |
+
"<extra_id_264>": 256035,
|
185 |
+
"<extra_id_265>": 256034,
|
186 |
+
"<extra_id_266>": 256033,
|
187 |
+
"<extra_id_267>": 256032,
|
188 |
+
"<extra_id_268>": 256031,
|
189 |
+
"<extra_id_269>": 256030,
|
190 |
+
"<extra_id_26>": 256273,
|
191 |
+
"<extra_id_270>": 256029,
|
192 |
+
"<extra_id_271>": 256028,
|
193 |
+
"<extra_id_272>": 256027,
|
194 |
+
"<extra_id_273>": 256026,
|
195 |
+
"<extra_id_274>": 256025,
|
196 |
+
"<extra_id_275>": 256024,
|
197 |
+
"<extra_id_276>": 256023,
|
198 |
+
"<extra_id_277>": 256022,
|
199 |
+
"<extra_id_278>": 256021,
|
200 |
+
"<extra_id_279>": 256020,
|
201 |
+
"<extra_id_27>": 256272,
|
202 |
+
"<extra_id_280>": 256019,
|
203 |
+
"<extra_id_281>": 256018,
|
204 |
+
"<extra_id_282>": 256017,
|
205 |
+
"<extra_id_283>": 256016,
|
206 |
+
"<extra_id_284>": 256015,
|
207 |
+
"<extra_id_285>": 256014,
|
208 |
+
"<extra_id_286>": 256013,
|
209 |
+
"<extra_id_287>": 256012,
|
210 |
+
"<extra_id_288>": 256011,
|
211 |
+
"<extra_id_289>": 256010,
|
212 |
+
"<extra_id_28>": 256271,
|
213 |
+
"<extra_id_290>": 256009,
|
214 |
+
"<extra_id_291>": 256008,
|
215 |
+
"<extra_id_292>": 256007,
|
216 |
+
"<extra_id_293>": 256006,
|
217 |
+
"<extra_id_294>": 256005,
|
218 |
+
"<extra_id_295>": 256004,
|
219 |
+
"<extra_id_296>": 256003,
|
220 |
+
"<extra_id_297>": 256002,
|
221 |
+
"<extra_id_298>": 256001,
|
222 |
+
"<extra_id_299>": 256000,
|
223 |
+
"<extra_id_29>": 256270,
|
224 |
+
"<extra_id_2>": 256297,
|
225 |
+
"<extra_id_30>": 256269,
|
226 |
+
"<extra_id_31>": 256268,
|
227 |
+
"<extra_id_32>": 256267,
|
228 |
+
"<extra_id_33>": 256266,
|
229 |
+
"<extra_id_34>": 256265,
|
230 |
+
"<extra_id_35>": 256264,
|
231 |
+
"<extra_id_36>": 256263,
|
232 |
+
"<extra_id_37>": 256262,
|
233 |
+
"<extra_id_38>": 256261,
|
234 |
+
"<extra_id_39>": 256260,
|
235 |
+
"<extra_id_3>": 256296,
|
236 |
+
"<extra_id_40>": 256259,
|
237 |
+
"<extra_id_41>": 256258,
|
238 |
+
"<extra_id_42>": 256257,
|
239 |
+
"<extra_id_43>": 256256,
|
240 |
+
"<extra_id_44>": 256255,
|
241 |
+
"<extra_id_45>": 256254,
|
242 |
+
"<extra_id_46>": 256253,
|
243 |
+
"<extra_id_47>": 256252,
|
244 |
+
"<extra_id_48>": 256251,
|
245 |
+
"<extra_id_49>": 256250,
|
246 |
+
"<extra_id_4>": 256295,
|
247 |
+
"<extra_id_50>": 256249,
|
248 |
+
"<extra_id_51>": 256248,
|
249 |
+
"<extra_id_52>": 256247,
|
250 |
+
"<extra_id_53>": 256246,
|
251 |
+
"<extra_id_54>": 256245,
|
252 |
+
"<extra_id_55>": 256244,
|
253 |
+
"<extra_id_56>": 256243,
|
254 |
+
"<extra_id_57>": 256242,
|
255 |
+
"<extra_id_58>": 256241,
|
256 |
+
"<extra_id_59>": 256240,
|
257 |
+
"<extra_id_5>": 256294,
|
258 |
+
"<extra_id_60>": 256239,
|
259 |
+
"<extra_id_61>": 256238,
|
260 |
+
"<extra_id_62>": 256237,
|
261 |
+
"<extra_id_63>": 256236,
|
262 |
+
"<extra_id_64>": 256235,
|
263 |
+
"<extra_id_65>": 256234,
|
264 |
+
"<extra_id_66>": 256233,
|
265 |
+
"<extra_id_67>": 256232,
|
266 |
+
"<extra_id_68>": 256231,
|
267 |
+
"<extra_id_69>": 256230,
|
268 |
+
"<extra_id_6>": 256293,
|
269 |
+
"<extra_id_70>": 256229,
|
270 |
+
"<extra_id_71>": 256228,
|
271 |
+
"<extra_id_72>": 256227,
|
272 |
+
"<extra_id_73>": 256226,
|
273 |
+
"<extra_id_74>": 256225,
|
274 |
+
"<extra_id_75>": 256224,
|
275 |
+
"<extra_id_76>": 256223,
|
276 |
+
"<extra_id_77>": 256222,
|
277 |
+
"<extra_id_78>": 256221,
|
278 |
+
"<extra_id_79>": 256220,
|
279 |
+
"<extra_id_7>": 256292,
|
280 |
+
"<extra_id_80>": 256219,
|
281 |
+
"<extra_id_81>": 256218,
|
282 |
+
"<extra_id_82>": 256217,
|
283 |
+
"<extra_id_83>": 256216,
|
284 |
+
"<extra_id_84>": 256215,
|
285 |
+
"<extra_id_85>": 256214,
|
286 |
+
"<extra_id_86>": 256213,
|
287 |
+
"<extra_id_87>": 256212,
|
288 |
+
"<extra_id_88>": 256211,
|
289 |
+
"<extra_id_89>": 256210,
|
290 |
+
"<extra_id_8>": 256291,
|
291 |
+
"<extra_id_90>": 256209,
|
292 |
+
"<extra_id_91>": 256208,
|
293 |
+
"<extra_id_92>": 256207,
|
294 |
+
"<extra_id_93>": 256206,
|
295 |
+
"<extra_id_94>": 256205,
|
296 |
+
"<extra_id_95>": 256204,
|
297 |
+
"<extra_id_96>": 256203,
|
298 |
+
"<extra_id_97>": 256202,
|
299 |
+
"<extra_id_98>": 256201,
|
300 |
+
"<extra_id_99>": 256200,
|
301 |
+
"<extra_id_9>": 256290
|
|
|
302 |
}
|
special_tokens_map.json
CHANGED
@@ -1,21 +1,6 @@
|
|
1 |
{
|
2 |
-
"additional_special_tokens": [
|
3 |
-
"[eod]",
|
4 |
-
"[wiki]",
|
5 |
-
"[translate]",
|
6 |
-
"[convo]",
|
7 |
-
"[fc]",
|
8 |
-
"[ffc]",
|
9 |
-
"[code]",
|
10 |
-
"[book]",
|
11 |
-
"[c4]",
|
12 |
-
"[news]",
|
13 |
-
"[forum]",
|
14 |
-
"[eot]"
|
15 |
-
],
|
16 |
"bos_token": "<s>",
|
17 |
"eos_token": "</s>",
|
18 |
"pad_token": "<pad>",
|
19 |
-
"sep_token": "<::::>",
|
20 |
"unk_token": "<unk>"
|
21 |
}
|
|
|
1 |
{
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2 |
"bos_token": "<s>",
|
3 |
"eos_token": "</s>",
|
4 |
"pad_token": "<pad>",
|
|
|
5 |
"unk_token": "<unk>"
|
6 |
}
|