ArthurZ HF staff commited on
Commit
0ce92bb
1 Parent(s): 2316899

Upload tokenizer

Browse files
Files changed (2) hide show
  1. added_tokens.json +300 -301
  2. special_tokens_map.json +0 -15
added_tokens.json CHANGED
@@ -1,303 +1,302 @@
1
  {
2
- "<::::>": 256000,
3
- "<extra_id_0>": 256300,
4
- "<extra_id_100>": 256200,
5
- "<extra_id_101>": 256199,
6
- "<extra_id_102>": 256198,
7
- "<extra_id_103>": 256197,
8
- "<extra_id_104>": 256196,
9
- "<extra_id_105>": 256195,
10
- "<extra_id_106>": 256194,
11
- "<extra_id_107>": 256193,
12
- "<extra_id_108>": 256192,
13
- "<extra_id_109>": 256191,
14
- "<extra_id_10>": 256290,
15
- "<extra_id_110>": 256190,
16
- "<extra_id_111>": 256189,
17
- "<extra_id_112>": 256188,
18
- "<extra_id_113>": 256187,
19
- "<extra_id_114>": 256186,
20
- "<extra_id_115>": 256185,
21
- "<extra_id_116>": 256184,
22
- "<extra_id_117>": 256183,
23
- "<extra_id_118>": 256182,
24
- "<extra_id_119>": 256181,
25
- "<extra_id_11>": 256289,
26
- "<extra_id_120>": 256180,
27
- "<extra_id_121>": 256179,
28
- "<extra_id_122>": 256178,
29
- "<extra_id_123>": 256177,
30
- "<extra_id_124>": 256176,
31
- "<extra_id_125>": 256175,
32
- "<extra_id_126>": 256174,
33
- "<extra_id_127>": 256173,
34
- "<extra_id_128>": 256172,
35
- "<extra_id_129>": 256171,
36
- "<extra_id_12>": 256288,
37
- "<extra_id_130>": 256170,
38
- "<extra_id_131>": 256169,
39
- "<extra_id_132>": 256168,
40
- "<extra_id_133>": 256167,
41
- "<extra_id_134>": 256166,
42
- "<extra_id_135>": 256165,
43
- "<extra_id_136>": 256164,
44
- "<extra_id_137>": 256163,
45
- "<extra_id_138>": 256162,
46
- "<extra_id_139>": 256161,
47
- "<extra_id_13>": 256287,
48
- "<extra_id_140>": 256160,
49
- "<extra_id_141>": 256159,
50
- "<extra_id_142>": 256158,
51
- "<extra_id_143>": 256157,
52
- "<extra_id_144>": 256156,
53
- "<extra_id_145>": 256155,
54
- "<extra_id_146>": 256154,
55
- "<extra_id_147>": 256153,
56
- "<extra_id_148>": 256152,
57
- "<extra_id_149>": 256151,
58
- "<extra_id_14>": 256286,
59
- "<extra_id_150>": 256150,
60
- "<extra_id_151>": 256149,
61
- "<extra_id_152>": 256148,
62
- "<extra_id_153>": 256147,
63
- "<extra_id_154>": 256146,
64
- "<extra_id_155>": 256145,
65
- "<extra_id_156>": 256144,
66
- "<extra_id_157>": 256143,
67
- "<extra_id_158>": 256142,
68
- "<extra_id_159>": 256141,
69
- "<extra_id_15>": 256285,
70
- "<extra_id_160>": 256140,
71
- "<extra_id_161>": 256139,
72
- "<extra_id_162>": 256138,
73
- "<extra_id_163>": 256137,
74
- "<extra_id_164>": 256136,
75
- "<extra_id_165>": 256135,
76
- "<extra_id_166>": 256134,
77
- "<extra_id_167>": 256133,
78
- "<extra_id_168>": 256132,
79
- "<extra_id_169>": 256131,
80
- "<extra_id_16>": 256284,
81
- "<extra_id_170>": 256130,
82
- "<extra_id_171>": 256129,
83
- "<extra_id_172>": 256128,
84
- "<extra_id_173>": 256127,
85
- "<extra_id_174>": 256126,
86
- "<extra_id_175>": 256125,
87
- "<extra_id_176>": 256124,
88
- "<extra_id_177>": 256123,
89
- "<extra_id_178>": 256122,
90
- "<extra_id_179>": 256121,
91
- "<extra_id_17>": 256283,
92
- "<extra_id_180>": 256120,
93
- "<extra_id_181>": 256119,
94
- "<extra_id_182>": 256118,
95
- "<extra_id_183>": 256117,
96
- "<extra_id_184>": 256116,
97
- "<extra_id_185>": 256115,
98
- "<extra_id_186>": 256114,
99
- "<extra_id_187>": 256113,
100
- "<extra_id_188>": 256112,
101
- "<extra_id_189>": 256111,
102
- "<extra_id_18>": 256282,
103
- "<extra_id_190>": 256110,
104
- "<extra_id_191>": 256109,
105
- "<extra_id_192>": 256108,
106
- "<extra_id_193>": 256107,
107
- "<extra_id_194>": 256106,
108
- "<extra_id_195>": 256105,
109
- "<extra_id_196>": 256104,
110
- "<extra_id_197>": 256103,
111
- "<extra_id_198>": 256102,
112
- "<extra_id_199>": 256101,
113
- "<extra_id_19>": 256281,
114
- "<extra_id_1>": 256299,
115
- "<extra_id_200>": 256100,
116
- "<extra_id_201>": 256099,
117
- "<extra_id_202>": 256098,
118
- "<extra_id_203>": 256097,
119
- "<extra_id_204>": 256096,
120
- "<extra_id_205>": 256095,
121
- "<extra_id_206>": 256094,
122
- "<extra_id_207>": 256093,
123
- "<extra_id_208>": 256092,
124
- "<extra_id_209>": 256091,
125
- "<extra_id_20>": 256280,
126
- "<extra_id_210>": 256090,
127
- "<extra_id_211>": 256089,
128
- "<extra_id_212>": 256088,
129
- "<extra_id_213>": 256087,
130
- "<extra_id_214>": 256086,
131
- "<extra_id_215>": 256085,
132
- "<extra_id_216>": 256084,
133
- "<extra_id_217>": 256083,
134
- "<extra_id_218>": 256082,
135
- "<extra_id_219>": 256081,
136
- "<extra_id_21>": 256279,
137
- "<extra_id_220>": 256080,
138
- "<extra_id_221>": 256079,
139
- "<extra_id_222>": 256078,
140
- "<extra_id_223>": 256077,
141
- "<extra_id_224>": 256076,
142
- "<extra_id_225>": 256075,
143
- "<extra_id_226>": 256074,
144
- "<extra_id_227>": 256073,
145
- "<extra_id_228>": 256072,
146
- "<extra_id_229>": 256071,
147
- "<extra_id_22>": 256278,
148
- "<extra_id_230>": 256070,
149
- "<extra_id_231>": 256069,
150
- "<extra_id_232>": 256068,
151
- "<extra_id_233>": 256067,
152
- "<extra_id_234>": 256066,
153
- "<extra_id_235>": 256065,
154
- "<extra_id_236>": 256064,
155
- "<extra_id_237>": 256063,
156
- "<extra_id_238>": 256062,
157
- "<extra_id_239>": 256061,
158
- "<extra_id_23>": 256277,
159
- "<extra_id_240>": 256060,
160
- "<extra_id_241>": 256059,
161
- "<extra_id_242>": 256058,
162
- "<extra_id_243>": 256057,
163
- "<extra_id_244>": 256056,
164
- "<extra_id_245>": 256055,
165
- "<extra_id_246>": 256054,
166
- "<extra_id_247>": 256053,
167
- "<extra_id_248>": 256052,
168
- "<extra_id_249>": 256051,
169
- "<extra_id_24>": 256276,
170
- "<extra_id_250>": 256050,
171
- "<extra_id_251>": 256049,
172
- "<extra_id_252>": 256048,
173
- "<extra_id_253>": 256047,
174
- "<extra_id_254>": 256046,
175
- "<extra_id_255>": 256045,
176
- "<extra_id_256>": 256044,
177
- "<extra_id_257>": 256043,
178
- "<extra_id_258>": 256042,
179
- "<extra_id_259>": 256041,
180
- "<extra_id_25>": 256275,
181
- "<extra_id_260>": 256040,
182
- "<extra_id_261>": 256039,
183
- "<extra_id_262>": 256038,
184
- "<extra_id_263>": 256037,
185
- "<extra_id_264>": 256036,
186
- "<extra_id_265>": 256035,
187
- "<extra_id_266>": 256034,
188
- "<extra_id_267>": 256033,
189
- "<extra_id_268>": 256032,
190
- "<extra_id_269>": 256031,
191
- "<extra_id_26>": 256274,
192
- "<extra_id_270>": 256030,
193
- "<extra_id_271>": 256029,
194
- "<extra_id_272>": 256028,
195
- "<extra_id_273>": 256027,
196
- "<extra_id_274>": 256026,
197
- "<extra_id_275>": 256025,
198
- "<extra_id_276>": 256024,
199
- "<extra_id_277>": 256023,
200
- "<extra_id_278>": 256022,
201
- "<extra_id_279>": 256021,
202
- "<extra_id_27>": 256273,
203
- "<extra_id_280>": 256020,
204
- "<extra_id_281>": 256019,
205
- "<extra_id_282>": 256018,
206
- "<extra_id_283>": 256017,
207
- "<extra_id_284>": 256016,
208
- "<extra_id_285>": 256015,
209
- "<extra_id_286>": 256014,
210
- "<extra_id_287>": 256013,
211
- "<extra_id_288>": 256012,
212
- "<extra_id_289>": 256011,
213
- "<extra_id_28>": 256272,
214
- "<extra_id_290>": 256010,
215
- "<extra_id_291>": 256009,
216
- "<extra_id_292>": 256008,
217
- "<extra_id_293>": 256007,
218
- "<extra_id_294>": 256006,
219
- "<extra_id_295>": 256005,
220
- "<extra_id_296>": 256004,
221
- "<extra_id_297>": 256003,
222
- "<extra_id_298>": 256002,
223
- "<extra_id_299>": 256001,
224
- "<extra_id_29>": 256271,
225
- "<extra_id_2>": 256298,
226
- "<extra_id_30>": 256270,
227
- "<extra_id_31>": 256269,
228
- "<extra_id_32>": 256268,
229
- "<extra_id_33>": 256267,
230
- "<extra_id_34>": 256266,
231
- "<extra_id_35>": 256265,
232
- "<extra_id_36>": 256264,
233
- "<extra_id_37>": 256263,
234
- "<extra_id_38>": 256262,
235
- "<extra_id_39>": 256261,
236
- "<extra_id_3>": 256297,
237
- "<extra_id_40>": 256260,
238
- "<extra_id_41>": 256259,
239
- "<extra_id_42>": 256258,
240
- "<extra_id_43>": 256257,
241
- "<extra_id_44>": 256256,
242
- "<extra_id_45>": 256255,
243
- "<extra_id_46>": 256254,
244
- "<extra_id_47>": 256253,
245
- "<extra_id_48>": 256252,
246
- "<extra_id_49>": 256251,
247
- "<extra_id_4>": 256296,
248
- "<extra_id_50>": 256250,
249
- "<extra_id_51>": 256249,
250
- "<extra_id_52>": 256248,
251
- "<extra_id_53>": 256247,
252
- "<extra_id_54>": 256246,
253
- "<extra_id_55>": 256245,
254
- "<extra_id_56>": 256244,
255
- "<extra_id_57>": 256243,
256
- "<extra_id_58>": 256242,
257
- "<extra_id_59>": 256241,
258
- "<extra_id_5>": 256295,
259
- "<extra_id_60>": 256240,
260
- "<extra_id_61>": 256239,
261
- "<extra_id_62>": 256238,
262
- "<extra_id_63>": 256237,
263
- "<extra_id_64>": 256236,
264
- "<extra_id_65>": 256235,
265
- "<extra_id_66>": 256234,
266
- "<extra_id_67>": 256233,
267
- "<extra_id_68>": 256232,
268
- "<extra_id_69>": 256231,
269
- "<extra_id_6>": 256294,
270
- "<extra_id_70>": 256230,
271
- "<extra_id_71>": 256229,
272
- "<extra_id_72>": 256228,
273
- "<extra_id_73>": 256227,
274
- "<extra_id_74>": 256226,
275
- "<extra_id_75>": 256225,
276
- "<extra_id_76>": 256224,
277
- "<extra_id_77>": 256223,
278
- "<extra_id_78>": 256222,
279
- "<extra_id_79>": 256221,
280
- "<extra_id_7>": 256293,
281
- "<extra_id_80>": 256220,
282
- "<extra_id_81>": 256219,
283
- "<extra_id_82>": 256218,
284
- "<extra_id_83>": 256217,
285
- "<extra_id_84>": 256216,
286
- "<extra_id_85>": 256215,
287
- "<extra_id_86>": 256214,
288
- "<extra_id_87>": 256213,
289
- "<extra_id_88>": 256212,
290
- "<extra_id_89>": 256211,
291
- "<extra_id_8>": 256292,
292
- "<extra_id_90>": 256210,
293
- "<extra_id_91>": 256209,
294
- "<extra_id_92>": 256208,
295
- "<extra_id_93>": 256207,
296
- "<extra_id_94>": 256206,
297
- "<extra_id_95>": 256205,
298
- "<extra_id_96>": 256204,
299
- "<extra_id_97>": 256203,
300
- "<extra_id_98>": 256202,
301
- "<extra_id_99>": 256201,
302
- "<extra_id_9>": 256291
303
  }
 
1
  {
2
+ "<extra_id_0>": 256299,
3
+ "<extra_id_100>": 256199,
4
+ "<extra_id_101>": 256198,
5
+ "<extra_id_102>": 256197,
6
+ "<extra_id_103>": 256196,
7
+ "<extra_id_104>": 256195,
8
+ "<extra_id_105>": 256194,
9
+ "<extra_id_106>": 256193,
10
+ "<extra_id_107>": 256192,
11
+ "<extra_id_108>": 256191,
12
+ "<extra_id_109>": 256190,
13
+ "<extra_id_10>": 256289,
14
+ "<extra_id_110>": 256189,
15
+ "<extra_id_111>": 256188,
16
+ "<extra_id_112>": 256187,
17
+ "<extra_id_113>": 256186,
18
+ "<extra_id_114>": 256185,
19
+ "<extra_id_115>": 256184,
20
+ "<extra_id_116>": 256183,
21
+ "<extra_id_117>": 256182,
22
+ "<extra_id_118>": 256181,
23
+ "<extra_id_119>": 256180,
24
+ "<extra_id_11>": 256288,
25
+ "<extra_id_120>": 256179,
26
+ "<extra_id_121>": 256178,
27
+ "<extra_id_122>": 256177,
28
+ "<extra_id_123>": 256176,
29
+ "<extra_id_124>": 256175,
30
+ "<extra_id_125>": 256174,
31
+ "<extra_id_126>": 256173,
32
+ "<extra_id_127>": 256172,
33
+ "<extra_id_128>": 256171,
34
+ "<extra_id_129>": 256170,
35
+ "<extra_id_12>": 256287,
36
+ "<extra_id_130>": 256169,
37
+ "<extra_id_131>": 256168,
38
+ "<extra_id_132>": 256167,
39
+ "<extra_id_133>": 256166,
40
+ "<extra_id_134>": 256165,
41
+ "<extra_id_135>": 256164,
42
+ "<extra_id_136>": 256163,
43
+ "<extra_id_137>": 256162,
44
+ "<extra_id_138>": 256161,
45
+ "<extra_id_139>": 256160,
46
+ "<extra_id_13>": 256286,
47
+ "<extra_id_140>": 256159,
48
+ "<extra_id_141>": 256158,
49
+ "<extra_id_142>": 256157,
50
+ "<extra_id_143>": 256156,
51
+ "<extra_id_144>": 256155,
52
+ "<extra_id_145>": 256154,
53
+ "<extra_id_146>": 256153,
54
+ "<extra_id_147>": 256152,
55
+ "<extra_id_148>": 256151,
56
+ "<extra_id_149>": 256150,
57
+ "<extra_id_14>": 256285,
58
+ "<extra_id_150>": 256149,
59
+ "<extra_id_151>": 256148,
60
+ "<extra_id_152>": 256147,
61
+ "<extra_id_153>": 256146,
62
+ "<extra_id_154>": 256145,
63
+ "<extra_id_155>": 256144,
64
+ "<extra_id_156>": 256143,
65
+ "<extra_id_157>": 256142,
66
+ "<extra_id_158>": 256141,
67
+ "<extra_id_159>": 256140,
68
+ "<extra_id_15>": 256284,
69
+ "<extra_id_160>": 256139,
70
+ "<extra_id_161>": 256138,
71
+ "<extra_id_162>": 256137,
72
+ "<extra_id_163>": 256136,
73
+ "<extra_id_164>": 256135,
74
+ "<extra_id_165>": 256134,
75
+ "<extra_id_166>": 256133,
76
+ "<extra_id_167>": 256132,
77
+ "<extra_id_168>": 256131,
78
+ "<extra_id_169>": 256130,
79
+ "<extra_id_16>": 256283,
80
+ "<extra_id_170>": 256129,
81
+ "<extra_id_171>": 256128,
82
+ "<extra_id_172>": 256127,
83
+ "<extra_id_173>": 256126,
84
+ "<extra_id_174>": 256125,
85
+ "<extra_id_175>": 256124,
86
+ "<extra_id_176>": 256123,
87
+ "<extra_id_177>": 256122,
88
+ "<extra_id_178>": 256121,
89
+ "<extra_id_179>": 256120,
90
+ "<extra_id_17>": 256282,
91
+ "<extra_id_180>": 256119,
92
+ "<extra_id_181>": 256118,
93
+ "<extra_id_182>": 256117,
94
+ "<extra_id_183>": 256116,
95
+ "<extra_id_184>": 256115,
96
+ "<extra_id_185>": 256114,
97
+ "<extra_id_186>": 256113,
98
+ "<extra_id_187>": 256112,
99
+ "<extra_id_188>": 256111,
100
+ "<extra_id_189>": 256110,
101
+ "<extra_id_18>": 256281,
102
+ "<extra_id_190>": 256109,
103
+ "<extra_id_191>": 256108,
104
+ "<extra_id_192>": 256107,
105
+ "<extra_id_193>": 256106,
106
+ "<extra_id_194>": 256105,
107
+ "<extra_id_195>": 256104,
108
+ "<extra_id_196>": 256103,
109
+ "<extra_id_197>": 256102,
110
+ "<extra_id_198>": 256101,
111
+ "<extra_id_199>": 256100,
112
+ "<extra_id_19>": 256280,
113
+ "<extra_id_1>": 256298,
114
+ "<extra_id_200>": 256099,
115
+ "<extra_id_201>": 256098,
116
+ "<extra_id_202>": 256097,
117
+ "<extra_id_203>": 256096,
118
+ "<extra_id_204>": 256095,
119
+ "<extra_id_205>": 256094,
120
+ "<extra_id_206>": 256093,
121
+ "<extra_id_207>": 256092,
122
+ "<extra_id_208>": 256091,
123
+ "<extra_id_209>": 256090,
124
+ "<extra_id_20>": 256279,
125
+ "<extra_id_210>": 256089,
126
+ "<extra_id_211>": 256088,
127
+ "<extra_id_212>": 256087,
128
+ "<extra_id_213>": 256086,
129
+ "<extra_id_214>": 256085,
130
+ "<extra_id_215>": 256084,
131
+ "<extra_id_216>": 256083,
132
+ "<extra_id_217>": 256082,
133
+ "<extra_id_218>": 256081,
134
+ "<extra_id_219>": 256080,
135
+ "<extra_id_21>": 256278,
136
+ "<extra_id_220>": 256079,
137
+ "<extra_id_221>": 256078,
138
+ "<extra_id_222>": 256077,
139
+ "<extra_id_223>": 256076,
140
+ "<extra_id_224>": 256075,
141
+ "<extra_id_225>": 256074,
142
+ "<extra_id_226>": 256073,
143
+ "<extra_id_227>": 256072,
144
+ "<extra_id_228>": 256071,
145
+ "<extra_id_229>": 256070,
146
+ "<extra_id_22>": 256277,
147
+ "<extra_id_230>": 256069,
148
+ "<extra_id_231>": 256068,
149
+ "<extra_id_232>": 256067,
150
+ "<extra_id_233>": 256066,
151
+ "<extra_id_234>": 256065,
152
+ "<extra_id_235>": 256064,
153
+ "<extra_id_236>": 256063,
154
+ "<extra_id_237>": 256062,
155
+ "<extra_id_238>": 256061,
156
+ "<extra_id_239>": 256060,
157
+ "<extra_id_23>": 256276,
158
+ "<extra_id_240>": 256059,
159
+ "<extra_id_241>": 256058,
160
+ "<extra_id_242>": 256057,
161
+ "<extra_id_243>": 256056,
162
+ "<extra_id_244>": 256055,
163
+ "<extra_id_245>": 256054,
164
+ "<extra_id_246>": 256053,
165
+ "<extra_id_247>": 256052,
166
+ "<extra_id_248>": 256051,
167
+ "<extra_id_249>": 256050,
168
+ "<extra_id_24>": 256275,
169
+ "<extra_id_250>": 256049,
170
+ "<extra_id_251>": 256048,
171
+ "<extra_id_252>": 256047,
172
+ "<extra_id_253>": 256046,
173
+ "<extra_id_254>": 256045,
174
+ "<extra_id_255>": 256044,
175
+ "<extra_id_256>": 256043,
176
+ "<extra_id_257>": 256042,
177
+ "<extra_id_258>": 256041,
178
+ "<extra_id_259>": 256040,
179
+ "<extra_id_25>": 256274,
180
+ "<extra_id_260>": 256039,
181
+ "<extra_id_261>": 256038,
182
+ "<extra_id_262>": 256037,
183
+ "<extra_id_263>": 256036,
184
+ "<extra_id_264>": 256035,
185
+ "<extra_id_265>": 256034,
186
+ "<extra_id_266>": 256033,
187
+ "<extra_id_267>": 256032,
188
+ "<extra_id_268>": 256031,
189
+ "<extra_id_269>": 256030,
190
+ "<extra_id_26>": 256273,
191
+ "<extra_id_270>": 256029,
192
+ "<extra_id_271>": 256028,
193
+ "<extra_id_272>": 256027,
194
+ "<extra_id_273>": 256026,
195
+ "<extra_id_274>": 256025,
196
+ "<extra_id_275>": 256024,
197
+ "<extra_id_276>": 256023,
198
+ "<extra_id_277>": 256022,
199
+ "<extra_id_278>": 256021,
200
+ "<extra_id_279>": 256020,
201
+ "<extra_id_27>": 256272,
202
+ "<extra_id_280>": 256019,
203
+ "<extra_id_281>": 256018,
204
+ "<extra_id_282>": 256017,
205
+ "<extra_id_283>": 256016,
206
+ "<extra_id_284>": 256015,
207
+ "<extra_id_285>": 256014,
208
+ "<extra_id_286>": 256013,
209
+ "<extra_id_287>": 256012,
210
+ "<extra_id_288>": 256011,
211
+ "<extra_id_289>": 256010,
212
+ "<extra_id_28>": 256271,
213
+ "<extra_id_290>": 256009,
214
+ "<extra_id_291>": 256008,
215
+ "<extra_id_292>": 256007,
216
+ "<extra_id_293>": 256006,
217
+ "<extra_id_294>": 256005,
218
+ "<extra_id_295>": 256004,
219
+ "<extra_id_296>": 256003,
220
+ "<extra_id_297>": 256002,
221
+ "<extra_id_298>": 256001,
222
+ "<extra_id_299>": 256000,
223
+ "<extra_id_29>": 256270,
224
+ "<extra_id_2>": 256297,
225
+ "<extra_id_30>": 256269,
226
+ "<extra_id_31>": 256268,
227
+ "<extra_id_32>": 256267,
228
+ "<extra_id_33>": 256266,
229
+ "<extra_id_34>": 256265,
230
+ "<extra_id_35>": 256264,
231
+ "<extra_id_36>": 256263,
232
+ "<extra_id_37>": 256262,
233
+ "<extra_id_38>": 256261,
234
+ "<extra_id_39>": 256260,
235
+ "<extra_id_3>": 256296,
236
+ "<extra_id_40>": 256259,
237
+ "<extra_id_41>": 256258,
238
+ "<extra_id_42>": 256257,
239
+ "<extra_id_43>": 256256,
240
+ "<extra_id_44>": 256255,
241
+ "<extra_id_45>": 256254,
242
+ "<extra_id_46>": 256253,
243
+ "<extra_id_47>": 256252,
244
+ "<extra_id_48>": 256251,
245
+ "<extra_id_49>": 256250,
246
+ "<extra_id_4>": 256295,
247
+ "<extra_id_50>": 256249,
248
+ "<extra_id_51>": 256248,
249
+ "<extra_id_52>": 256247,
250
+ "<extra_id_53>": 256246,
251
+ "<extra_id_54>": 256245,
252
+ "<extra_id_55>": 256244,
253
+ "<extra_id_56>": 256243,
254
+ "<extra_id_57>": 256242,
255
+ "<extra_id_58>": 256241,
256
+ "<extra_id_59>": 256240,
257
+ "<extra_id_5>": 256294,
258
+ "<extra_id_60>": 256239,
259
+ "<extra_id_61>": 256238,
260
+ "<extra_id_62>": 256237,
261
+ "<extra_id_63>": 256236,
262
+ "<extra_id_64>": 256235,
263
+ "<extra_id_65>": 256234,
264
+ "<extra_id_66>": 256233,
265
+ "<extra_id_67>": 256232,
266
+ "<extra_id_68>": 256231,
267
+ "<extra_id_69>": 256230,
268
+ "<extra_id_6>": 256293,
269
+ "<extra_id_70>": 256229,
270
+ "<extra_id_71>": 256228,
271
+ "<extra_id_72>": 256227,
272
+ "<extra_id_73>": 256226,
273
+ "<extra_id_74>": 256225,
274
+ "<extra_id_75>": 256224,
275
+ "<extra_id_76>": 256223,
276
+ "<extra_id_77>": 256222,
277
+ "<extra_id_78>": 256221,
278
+ "<extra_id_79>": 256220,
279
+ "<extra_id_7>": 256292,
280
+ "<extra_id_80>": 256219,
281
+ "<extra_id_81>": 256218,
282
+ "<extra_id_82>": 256217,
283
+ "<extra_id_83>": 256216,
284
+ "<extra_id_84>": 256215,
285
+ "<extra_id_85>": 256214,
286
+ "<extra_id_86>": 256213,
287
+ "<extra_id_87>": 256212,
288
+ "<extra_id_88>": 256211,
289
+ "<extra_id_89>": 256210,
290
+ "<extra_id_8>": 256291,
291
+ "<extra_id_90>": 256209,
292
+ "<extra_id_91>": 256208,
293
+ "<extra_id_92>": 256207,
294
+ "<extra_id_93>": 256206,
295
+ "<extra_id_94>": 256205,
296
+ "<extra_id_95>": 256204,
297
+ "<extra_id_96>": 256203,
298
+ "<extra_id_97>": 256202,
299
+ "<extra_id_98>": 256201,
300
+ "<extra_id_99>": 256200,
301
+ "<extra_id_9>": 256290
 
302
  }
special_tokens_map.json CHANGED
@@ -1,21 +1,6 @@
1
  {
2
- "additional_special_tokens": [
3
- "[eod]",
4
- "[wiki]",
5
- "[translate]",
6
- "[convo]",
7
- "[fc]",
8
- "[ffc]",
9
- "[code]",
10
- "[book]",
11
- "[c4]",
12
- "[news]",
13
- "[forum]",
14
- "[eot]"
15
- ],
16
  "bos_token": "<s>",
17
  "eos_token": "</s>",
18
  "pad_token": "<pad>",
19
- "sep_token": "<::::>",
20
  "unk_token": "<unk>"
21
  }
 
1
  {
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2
  "bos_token": "<s>",
3
  "eos_token": "</s>",
4
  "pad_token": "<pad>",
 
5
  "unk_token": "<unk>"
6
  }