erfan226 commited on
Commit
7dc6fc8
1 Parent(s): a0dccdc

add tokenizer

Browse files
Files changed (3) hide show
  1. special_tokens_map.json +107 -1
  2. tokenizer.json +212 -207
  3. tokenizer_config.json +111 -1
special_tokens_map.json CHANGED
@@ -1 +1,107 @@
1
- {"eos_token": "</s>", "unk_token": "<unk>", "pad_token": "<pad>", "additional_special_tokens": ["<extra_id_0>", "<extra_id_1>", "<extra_id_2>", "<extra_id_3>", "<extra_id_4>", "<extra_id_5>", "<extra_id_6>", "<extra_id_7>", "<extra_id_8>", "<extra_id_9>", "<extra_id_10>", "<extra_id_11>", "<extra_id_12>", "<extra_id_13>", "<extra_id_14>", "<extra_id_15>", "<extra_id_16>", "<extra_id_17>", "<extra_id_18>", "<extra_id_19>", "<extra_id_20>", "<extra_id_21>", "<extra_id_22>", "<extra_id_23>", "<extra_id_24>", "<extra_id_25>", "<extra_id_26>", "<extra_id_27>", "<extra_id_28>", "<extra_id_29>", "<extra_id_30>", "<extra_id_31>", "<extra_id_32>", "<extra_id_33>", "<extra_id_34>", "<extra_id_35>", "<extra_id_36>", "<extra_id_37>", "<extra_id_38>", "<extra_id_39>", "<extra_id_40>", "<extra_id_41>", "<extra_id_42>", "<extra_id_43>", "<extra_id_44>", "<extra_id_45>", "<extra_id_46>", "<extra_id_47>", "<extra_id_48>", "<extra_id_49>", "<extra_id_50>", "<extra_id_51>", "<extra_id_52>", "<extra_id_53>", "<extra_id_54>", "<extra_id_55>", "<extra_id_56>", "<extra_id_57>", "<extra_id_58>", "<extra_id_59>", "<extra_id_60>", "<extra_id_61>", "<extra_id_62>", "<extra_id_63>", "<extra_id_64>", "<extra_id_65>", "<extra_id_66>", "<extra_id_67>", "<extra_id_68>", "<extra_id_69>", "<extra_id_70>", "<extra_id_71>", "<extra_id_72>", "<extra_id_73>", "<extra_id_74>", "<extra_id_75>", "<extra_id_76>", "<extra_id_77>", "<extra_id_78>", "<extra_id_79>", "<extra_id_80>", "<extra_id_81>", "<extra_id_82>", "<extra_id_83>", "<extra_id_84>", "<extra_id_85>", "<extra_id_86>", "<extra_id_87>", "<extra_id_88>", "<extra_id_89>", "<extra_id_90>", "<extra_id_91>", "<extra_id_92>", "<extra_id_93>", "<extra_id_94>", "<extra_id_95>", "<extra_id_96>", "<extra_id_97>", "<extra_id_98>", "<extra_id_99>"]}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "additional_special_tokens": [
3
+ "<extra_id_0>",
4
+ "<extra_id_1>",
5
+ "<extra_id_2>",
6
+ "<extra_id_3>",
7
+ "<extra_id_4>",
8
+ "<extra_id_5>",
9
+ "<extra_id_6>",
10
+ "<extra_id_7>",
11
+ "<extra_id_8>",
12
+ "<extra_id_9>",
13
+ "<extra_id_10>",
14
+ "<extra_id_11>",
15
+ "<extra_id_12>",
16
+ "<extra_id_13>",
17
+ "<extra_id_14>",
18
+ "<extra_id_15>",
19
+ "<extra_id_16>",
20
+ "<extra_id_17>",
21
+ "<extra_id_18>",
22
+ "<extra_id_19>",
23
+ "<extra_id_20>",
24
+ "<extra_id_21>",
25
+ "<extra_id_22>",
26
+ "<extra_id_23>",
27
+ "<extra_id_24>",
28
+ "<extra_id_25>",
29
+ "<extra_id_26>",
30
+ "<extra_id_27>",
31
+ "<extra_id_28>",
32
+ "<extra_id_29>",
33
+ "<extra_id_30>",
34
+ "<extra_id_31>",
35
+ "<extra_id_32>",
36
+ "<extra_id_33>",
37
+ "<extra_id_34>",
38
+ "<extra_id_35>",
39
+ "<extra_id_36>",
40
+ "<extra_id_37>",
41
+ "<extra_id_38>",
42
+ "<extra_id_39>",
43
+ "<extra_id_40>",
44
+ "<extra_id_41>",
45
+ "<extra_id_42>",
46
+ "<extra_id_43>",
47
+ "<extra_id_44>",
48
+ "<extra_id_45>",
49
+ "<extra_id_46>",
50
+ "<extra_id_47>",
51
+ "<extra_id_48>",
52
+ "<extra_id_49>",
53
+ "<extra_id_50>",
54
+ "<extra_id_51>",
55
+ "<extra_id_52>",
56
+ "<extra_id_53>",
57
+ "<extra_id_54>",
58
+ "<extra_id_55>",
59
+ "<extra_id_56>",
60
+ "<extra_id_57>",
61
+ "<extra_id_58>",
62
+ "<extra_id_59>",
63
+ "<extra_id_60>",
64
+ "<extra_id_61>",
65
+ "<extra_id_62>",
66
+ "<extra_id_63>",
67
+ "<extra_id_64>",
68
+ "<extra_id_65>",
69
+ "<extra_id_66>",
70
+ "<extra_id_67>",
71
+ "<extra_id_68>",
72
+ "<extra_id_69>",
73
+ "<extra_id_70>",
74
+ "<extra_id_71>",
75
+ "<extra_id_72>",
76
+ "<extra_id_73>",
77
+ "<extra_id_74>",
78
+ "<extra_id_75>",
79
+ "<extra_id_76>",
80
+ "<extra_id_77>",
81
+ "<extra_id_78>",
82
+ "<extra_id_79>",
83
+ "<extra_id_80>",
84
+ "<extra_id_81>",
85
+ "<extra_id_82>",
86
+ "<extra_id_83>",
87
+ "<extra_id_84>",
88
+ "<extra_id_85>",
89
+ "<extra_id_86>",
90
+ "<extra_id_87>",
91
+ "<extra_id_88>",
92
+ "<extra_id_89>",
93
+ "<extra_id_90>",
94
+ "<extra_id_91>",
95
+ "<extra_id_92>",
96
+ "<extra_id_93>",
97
+ "<extra_id_94>",
98
+ "<extra_id_95>",
99
+ "<extra_id_96>",
100
+ "<extra_id_97>",
101
+ "<extra_id_98>",
102
+ "<extra_id_99>"
103
+ ],
104
+ "eos_token": "</s>",
105
+ "pad_token": "<pad>",
106
+ "unk_token": "<unk>"
107
+ }
tokenizer.json CHANGED
@@ -1,6 +1,11 @@
1
  {
2
  "version": "1.0",
3
- "truncation": null,
 
 
 
 
 
4
  "padding": {
5
  "strategy": "BatchLongest",
6
  "direction": "Right",
@@ -12,930 +17,930 @@
12
  "added_tokens": [
13
  {
14
  "id": 0,
15
- "special": true,
16
  "content": "<pad>",
17
  "single_word": false,
18
  "lstrip": false,
19
  "rstrip": false,
20
- "normalized": false
 
21
  },
22
  {
23
  "id": 1,
24
- "special": true,
25
  "content": "</s>",
26
  "single_word": false,
27
  "lstrip": false,
28
  "rstrip": false,
29
- "normalized": false
 
30
  },
31
  {
32
  "id": 2,
33
- "special": true,
34
  "content": "<unk>",
35
  "single_word": false,
36
  "lstrip": false,
37
  "rstrip": false,
38
- "normalized": false
 
39
  },
40
  {
41
  "id": 32003,
42
- "special": true,
43
  "content": "<extra_id_0>",
44
  "single_word": false,
45
  "lstrip": false,
46
  "rstrip": false,
47
- "normalized": false
 
48
  },
49
  {
50
  "id": 32004,
51
- "special": true,
52
  "content": "<extra_id_1>",
53
  "single_word": false,
54
  "lstrip": false,
55
  "rstrip": false,
56
- "normalized": false
 
57
  },
58
  {
59
  "id": 32005,
60
- "special": true,
61
  "content": "<extra_id_2>",
62
  "single_word": false,
63
  "lstrip": false,
64
  "rstrip": false,
65
- "normalized": false
 
66
  },
67
  {
68
  "id": 32006,
69
- "special": true,
70
  "content": "<extra_id_3>",
71
  "single_word": false,
72
  "lstrip": false,
73
  "rstrip": false,
74
- "normalized": false
 
75
  },
76
  {
77
  "id": 32007,
78
- "special": true,
79
  "content": "<extra_id_4>",
80
  "single_word": false,
81
  "lstrip": false,
82
  "rstrip": false,
83
- "normalized": false
 
84
  },
85
  {
86
  "id": 32008,
87
- "special": true,
88
  "content": "<extra_id_5>",
89
  "single_word": false,
90
  "lstrip": false,
91
  "rstrip": false,
92
- "normalized": false
 
93
  },
94
  {
95
  "id": 32009,
96
- "special": true,
97
  "content": "<extra_id_6>",
98
  "single_word": false,
99
  "lstrip": false,
100
  "rstrip": false,
101
- "normalized": false
 
102
  },
103
  {
104
  "id": 32010,
105
- "special": true,
106
  "content": "<extra_id_7>",
107
  "single_word": false,
108
  "lstrip": false,
109
  "rstrip": false,
110
- "normalized": false
 
111
  },
112
  {
113
  "id": 32011,
114
- "special": true,
115
  "content": "<extra_id_8>",
116
  "single_word": false,
117
  "lstrip": false,
118
  "rstrip": false,
119
- "normalized": false
 
120
  },
121
  {
122
  "id": 32012,
123
- "special": true,
124
  "content": "<extra_id_9>",
125
  "single_word": false,
126
  "lstrip": false,
127
  "rstrip": false,
128
- "normalized": false
 
129
  },
130
  {
131
  "id": 32013,
132
- "special": true,
133
  "content": "<extra_id_10>",
134
  "single_word": false,
135
  "lstrip": false,
136
  "rstrip": false,
137
- "normalized": false
 
138
  },
139
  {
140
  "id": 32014,
141
- "special": true,
142
  "content": "<extra_id_11>",
143
  "single_word": false,
144
  "lstrip": false,
145
  "rstrip": false,
146
- "normalized": false
 
147
  },
148
  {
149
  "id": 32015,
150
- "special": true,
151
  "content": "<extra_id_12>",
152
  "single_word": false,
153
  "lstrip": false,
154
  "rstrip": false,
155
- "normalized": false
 
156
  },
157
  {
158
  "id": 32016,
159
- "special": true,
160
  "content": "<extra_id_13>",
161
  "single_word": false,
162
  "lstrip": false,
163
  "rstrip": false,
164
- "normalized": false
 
165
  },
166
  {
167
  "id": 32017,
168
- "special": true,
169
  "content": "<extra_id_14>",
170
  "single_word": false,
171
  "lstrip": false,
172
  "rstrip": false,
173
- "normalized": false
 
174
  },
175
  {
176
  "id": 32018,
177
- "special": true,
178
  "content": "<extra_id_15>",
179
  "single_word": false,
180
  "lstrip": false,
181
  "rstrip": false,
182
- "normalized": false
 
183
  },
184
  {
185
  "id": 32019,
186
- "special": true,
187
  "content": "<extra_id_16>",
188
  "single_word": false,
189
  "lstrip": false,
190
  "rstrip": false,
191
- "normalized": false
 
192
  },
193
  {
194
  "id": 32020,
195
- "special": true,
196
  "content": "<extra_id_17>",
197
  "single_word": false,
198
  "lstrip": false,
199
  "rstrip": false,
200
- "normalized": false
 
201
  },
202
  {
203
  "id": 32021,
204
- "special": true,
205
  "content": "<extra_id_18>",
206
  "single_word": false,
207
  "lstrip": false,
208
  "rstrip": false,
209
- "normalized": false
 
210
  },
211
  {
212
  "id": 32022,
213
- "special": true,
214
  "content": "<extra_id_19>",
215
  "single_word": false,
216
  "lstrip": false,
217
  "rstrip": false,
218
- "normalized": false
 
219
  },
220
  {
221
  "id": 32023,
222
- "special": true,
223
  "content": "<extra_id_20>",
224
  "single_word": false,
225
  "lstrip": false,
226
  "rstrip": false,
227
- "normalized": false
 
228
  },
229
  {
230
  "id": 32024,
231
- "special": true,
232
  "content": "<extra_id_21>",
233
  "single_word": false,
234
  "lstrip": false,
235
  "rstrip": false,
236
- "normalized": false
 
237
  },
238
  {
239
  "id": 32025,
240
- "special": true,
241
  "content": "<extra_id_22>",
242
  "single_word": false,
243
  "lstrip": false,
244
  "rstrip": false,
245
- "normalized": false
 
246
  },
247
  {
248
  "id": 32026,
249
- "special": true,
250
  "content": "<extra_id_23>",
251
  "single_word": false,
252
  "lstrip": false,
253
  "rstrip": false,
254
- "normalized": false
 
255
  },
256
  {
257
  "id": 32027,
258
- "special": true,
259
  "content": "<extra_id_24>",
260
  "single_word": false,
261
  "lstrip": false,
262
  "rstrip": false,
263
- "normalized": false
 
264
  },
265
  {
266
  "id": 32028,
267
- "special": true,
268
  "content": "<extra_id_25>",
269
  "single_word": false,
270
  "lstrip": false,
271
  "rstrip": false,
272
- "normalized": false
 
273
  },
274
  {
275
  "id": 32029,
276
- "special": true,
277
  "content": "<extra_id_26>",
278
  "single_word": false,
279
  "lstrip": false,
280
  "rstrip": false,
281
- "normalized": false
 
282
  },
283
  {
284
  "id": 32030,
285
- "special": true,
286
  "content": "<extra_id_27>",
287
  "single_word": false,
288
  "lstrip": false,
289
  "rstrip": false,
290
- "normalized": false
 
291
  },
292
  {
293
  "id": 32031,
294
- "special": true,
295
  "content": "<extra_id_28>",
296
  "single_word": false,
297
  "lstrip": false,
298
  "rstrip": false,
299
- "normalized": false
 
300
  },
301
  {
302
  "id": 32032,
303
- "special": true,
304
  "content": "<extra_id_29>",
305
  "single_word": false,
306
  "lstrip": false,
307
  "rstrip": false,
308
- "normalized": false
 
309
  },
310
  {
311
  "id": 32033,
312
- "special": true,
313
  "content": "<extra_id_30>",
314
  "single_word": false,
315
  "lstrip": false,
316
  "rstrip": false,
317
- "normalized": false
 
318
  },
319
  {
320
  "id": 32034,
321
- "special": true,
322
  "content": "<extra_id_31>",
323
  "single_word": false,
324
  "lstrip": false,
325
  "rstrip": false,
326
- "normalized": false
 
327
  },
328
  {
329
  "id": 32035,
330
- "special": true,
331
  "content": "<extra_id_32>",
332
  "single_word": false,
333
  "lstrip": false,
334
  "rstrip": false,
335
- "normalized": false
 
336
  },
337
  {
338
  "id": 32036,
339
- "special": true,
340
  "content": "<extra_id_33>",
341
  "single_word": false,
342
  "lstrip": false,
343
  "rstrip": false,
344
- "normalized": false
 
345
  },
346
  {
347
  "id": 32037,
348
- "special": true,
349
  "content": "<extra_id_34>",
350
  "single_word": false,
351
  "lstrip": false,
352
  "rstrip": false,
353
- "normalized": false
 
354
  },
355
  {
356
  "id": 32038,
357
- "special": true,
358
  "content": "<extra_id_35>",
359
  "single_word": false,
360
  "lstrip": false,
361
  "rstrip": false,
362
- "normalized": false
 
363
  },
364
  {
365
  "id": 32039,
366
- "special": true,
367
  "content": "<extra_id_36>",
368
  "single_word": false,
369
  "lstrip": false,
370
  "rstrip": false,
371
- "normalized": false
 
372
  },
373
  {
374
  "id": 32040,
375
- "special": true,
376
  "content": "<extra_id_37>",
377
  "single_word": false,
378
  "lstrip": false,
379
  "rstrip": false,
380
- "normalized": false
 
381
  },
382
  {
383
  "id": 32041,
384
- "special": true,
385
  "content": "<extra_id_38>",
386
  "single_word": false,
387
  "lstrip": false,
388
  "rstrip": false,
389
- "normalized": false
 
390
  },
391
  {
392
  "id": 32042,
393
- "special": true,
394
  "content": "<extra_id_39>",
395
  "single_word": false,
396
  "lstrip": false,
397
  "rstrip": false,
398
- "normalized": false
 
399
  },
400
  {
401
  "id": 32043,
402
- "special": true,
403
  "content": "<extra_id_40>",
404
  "single_word": false,
405
  "lstrip": false,
406
  "rstrip": false,
407
- "normalized": false
 
408
  },
409
  {
410
  "id": 32044,
411
- "special": true,
412
  "content": "<extra_id_41>",
413
  "single_word": false,
414
  "lstrip": false,
415
  "rstrip": false,
416
- "normalized": false
 
417
  },
418
  {
419
  "id": 32045,
420
- "special": true,
421
  "content": "<extra_id_42>",
422
  "single_word": false,
423
  "lstrip": false,
424
  "rstrip": false,
425
- "normalized": false
 
426
  },
427
  {
428
  "id": 32046,
429
- "special": true,
430
  "content": "<extra_id_43>",
431
  "single_word": false,
432
  "lstrip": false,
433
  "rstrip": false,
434
- "normalized": false
 
435
  },
436
  {
437
  "id": 32047,
438
- "special": true,
439
  "content": "<extra_id_44>",
440
  "single_word": false,
441
  "lstrip": false,
442
  "rstrip": false,
443
- "normalized": false
 
444
  },
445
  {
446
  "id": 32048,
447
- "special": true,
448
  "content": "<extra_id_45>",
449
  "single_word": false,
450
  "lstrip": false,
451
  "rstrip": false,
452
- "normalized": false
 
453
  },
454
  {
455
  "id": 32049,
456
- "special": true,
457
  "content": "<extra_id_46>",
458
  "single_word": false,
459
  "lstrip": false,
460
  "rstrip": false,
461
- "normalized": false
 
462
  },
463
  {
464
  "id": 32050,
465
- "special": true,
466
  "content": "<extra_id_47>",
467
  "single_word": false,
468
  "lstrip": false,
469
  "rstrip": false,
470
- "normalized": false
 
471
  },
472
  {
473
  "id": 32051,
474
- "special": true,
475
  "content": "<extra_id_48>",
476
  "single_word": false,
477
  "lstrip": false,
478
  "rstrip": false,
479
- "normalized": false
 
480
  },
481
  {
482
  "id": 32052,
483
- "special": true,
484
  "content": "<extra_id_49>",
485
  "single_word": false,
486
  "lstrip": false,
487
  "rstrip": false,
488
- "normalized": false
 
489
  },
490
  {
491
  "id": 32053,
492
- "special": true,
493
  "content": "<extra_id_50>",
494
  "single_word": false,
495
  "lstrip": false,
496
  "rstrip": false,
497
- "normalized": false
 
498
  },
499
  {
500
  "id": 32054,
501
- "special": true,
502
  "content": "<extra_id_51>",
503
  "single_word": false,
504
  "lstrip": false,
505
  "rstrip": false,
506
- "normalized": false
 
507
  },
508
  {
509
  "id": 32055,
510
- "special": true,
511
  "content": "<extra_id_52>",
512
  "single_word": false,
513
  "lstrip": false,
514
  "rstrip": false,
515
- "normalized": false
 
516
  },
517
  {
518
  "id": 32056,
519
- "special": true,
520
  "content": "<extra_id_53>",
521
  "single_word": false,
522
  "lstrip": false,
523
  "rstrip": false,
524
- "normalized": false
 
525
  },
526
  {
527
  "id": 32057,
528
- "special": true,
529
  "content": "<extra_id_54>",
530
  "single_word": false,
531
  "lstrip": false,
532
  "rstrip": false,
533
- "normalized": false
 
534
  },
535
  {
536
  "id": 32058,
537
- "special": true,
538
  "content": "<extra_id_55>",
539
  "single_word": false,
540
  "lstrip": false,
541
  "rstrip": false,
542
- "normalized": false
 
543
  },
544
  {
545
  "id": 32059,
546
- "special": true,
547
  "content": "<extra_id_56>",
548
  "single_word": false,
549
  "lstrip": false,
550
  "rstrip": false,
551
- "normalized": false
 
552
  },
553
  {
554
  "id": 32060,
555
- "special": true,
556
  "content": "<extra_id_57>",
557
  "single_word": false,
558
  "lstrip": false,
559
  "rstrip": false,
560
- "normalized": false
 
561
  },
562
  {
563
  "id": 32061,
564
- "special": true,
565
  "content": "<extra_id_58>",
566
  "single_word": false,
567
  "lstrip": false,
568
  "rstrip": false,
569
- "normalized": false
 
570
  },
571
  {
572
  "id": 32062,
573
- "special": true,
574
  "content": "<extra_id_59>",
575
  "single_word": false,
576
  "lstrip": false,
577
  "rstrip": false,
578
- "normalized": false
 
579
  },
580
  {
581
  "id": 32063,
582
- "special": true,
583
  "content": "<extra_id_60>",
584
  "single_word": false,
585
  "lstrip": false,
586
  "rstrip": false,
587
- "normalized": false
 
588
  },
589
  {
590
  "id": 32064,
591
- "special": true,
592
  "content": "<extra_id_61>",
593
  "single_word": false,
594
  "lstrip": false,
595
  "rstrip": false,
596
- "normalized": false
 
597
  },
598
  {
599
  "id": 32065,
600
- "special": true,
601
  "content": "<extra_id_62>",
602
  "single_word": false,
603
  "lstrip": false,
604
  "rstrip": false,
605
- "normalized": false
 
606
  },
607
  {
608
  "id": 32066,
609
- "special": true,
610
  "content": "<extra_id_63>",
611
  "single_word": false,
612
  "lstrip": false,
613
  "rstrip": false,
614
- "normalized": false
 
615
  },
616
  {
617
  "id": 32067,
618
- "special": true,
619
  "content": "<extra_id_64>",
620
  "single_word": false,
621
  "lstrip": false,
622
  "rstrip": false,
623
- "normalized": false
 
624
  },
625
  {
626
  "id": 32068,
627
- "special": true,
628
  "content": "<extra_id_65>",
629
  "single_word": false,
630
  "lstrip": false,
631
  "rstrip": false,
632
- "normalized": false
 
633
  },
634
  {
635
  "id": 32069,
636
- "special": true,
637
  "content": "<extra_id_66>",
638
  "single_word": false,
639
  "lstrip": false,
640
  "rstrip": false,
641
- "normalized": false
 
642
  },
643
  {
644
  "id": 32070,
645
- "special": true,
646
  "content": "<extra_id_67>",
647
  "single_word": false,
648
  "lstrip": false,
649
  "rstrip": false,
650
- "normalized": false
 
651
  },
652
  {
653
  "id": 32071,
654
- "special": true,
655
  "content": "<extra_id_68>",
656
  "single_word": false,
657
  "lstrip": false,
658
  "rstrip": false,
659
- "normalized": false
 
660
  },
661
  {
662
  "id": 32072,
663
- "special": true,
664
  "content": "<extra_id_69>",
665
  "single_word": false,
666
  "lstrip": false,
667
  "rstrip": false,
668
- "normalized": false
 
669
  },
670
  {
671
  "id": 32073,
672
- "special": true,
673
  "content": "<extra_id_70>",
674
  "single_word": false,
675
  "lstrip": false,
676
  "rstrip": false,
677
- "normalized": false
 
678
  },
679
  {
680
  "id": 32074,
681
- "special": true,
682
  "content": "<extra_id_71>",
683
  "single_word": false,
684
  "lstrip": false,
685
  "rstrip": false,
686
- "normalized": false
 
687
  },
688
  {
689
  "id": 32075,
690
- "special": true,
691
  "content": "<extra_id_72>",
692
  "single_word": false,
693
  "lstrip": false,
694
  "rstrip": false,
695
- "normalized": false
 
696
  },
697
  {
698
  "id": 32076,
699
- "special": true,
700
  "content": "<extra_id_73>",
701
  "single_word": false,
702
  "lstrip": false,
703
  "rstrip": false,
704
- "normalized": false
 
705
  },
706
  {
707
  "id": 32077,
708
- "special": true,
709
  "content": "<extra_id_74>",
710
  "single_word": false,
711
  "lstrip": false,
712
  "rstrip": false,
713
- "normalized": false
 
714
  },
715
  {
716
  "id": 32078,
717
- "special": true,
718
  "content": "<extra_id_75>",
719
  "single_word": false,
720
  "lstrip": false,
721
  "rstrip": false,
722
- "normalized": false
 
723
  },
724
  {
725
  "id": 32079,
726
- "special": true,
727
  "content": "<extra_id_76>",
728
  "single_word": false,
729
  "lstrip": false,
730
  "rstrip": false,
731
- "normalized": false
 
732
  },
733
  {
734
  "id": 32080,
735
- "special": true,
736
  "content": "<extra_id_77>",
737
  "single_word": false,
738
  "lstrip": false,
739
  "rstrip": false,
740
- "normalized": false
 
741
  },
742
  {
743
  "id": 32081,
744
- "special": true,
745
  "content": "<extra_id_78>",
746
  "single_word": false,
747
  "lstrip": false,
748
  "rstrip": false,
749
- "normalized": false
 
750
  },
751
  {
752
  "id": 32082,
753
- "special": true,
754
  "content": "<extra_id_79>",
755
  "single_word": false,
756
  "lstrip": false,
757
  "rstrip": false,
758
- "normalized": false
 
759
  },
760
  {
761
  "id": 32083,
762
- "special": true,
763
  "content": "<extra_id_80>",
764
  "single_word": false,
765
  "lstrip": false,
766
  "rstrip": false,
767
- "normalized": false
 
768
  },
769
  {
770
  "id": 32084,
771
- "special": true,
772
  "content": "<extra_id_81>",
773
  "single_word": false,
774
  "lstrip": false,
775
  "rstrip": false,
776
- "normalized": false
 
777
  },
778
  {
779
  "id": 32085,
780
- "special": true,
781
  "content": "<extra_id_82>",
782
  "single_word": false,
783
  "lstrip": false,
784
  "rstrip": false,
785
- "normalized": false
 
786
  },
787
  {
788
  "id": 32086,
789
- "special": true,
790
  "content": "<extra_id_83>",
791
  "single_word": false,
792
  "lstrip": false,
793
  "rstrip": false,
794
- "normalized": false
 
795
  },
796
  {
797
  "id": 32087,
798
- "special": true,
799
  "content": "<extra_id_84>",
800
  "single_word": false,
801
  "lstrip": false,
802
  "rstrip": false,
803
- "normalized": false
 
804
  },
805
  {
806
  "id": 32088,
807
- "special": true,
808
  "content": "<extra_id_85>",
809
  "single_word": false,
810
  "lstrip": false,
811
  "rstrip": false,
812
- "normalized": false
 
813
  },
814
  {
815
  "id": 32089,
816
- "special": true,
817
  "content": "<extra_id_86>",
818
  "single_word": false,
819
  "lstrip": false,
820
  "rstrip": false,
821
- "normalized": false
 
822
  },
823
  {
824
  "id": 32090,
825
- "special": true,
826
  "content": "<extra_id_87>",
827
  "single_word": false,
828
  "lstrip": false,
829
  "rstrip": false,
830
- "normalized": false
 
831
  },
832
  {
833
  "id": 32091,
834
- "special": true,
835
  "content": "<extra_id_88>",
836
  "single_word": false,
837
  "lstrip": false,
838
  "rstrip": false,
839
- "normalized": false
 
840
  },
841
  {
842
  "id": 32092,
843
- "special": true,
844
  "content": "<extra_id_89>",
845
  "single_word": false,
846
  "lstrip": false,
847
  "rstrip": false,
848
- "normalized": false
 
849
  },
850
  {
851
  "id": 32093,
852
- "special": true,
853
  "content": "<extra_id_90>",
854
  "single_word": false,
855
  "lstrip": false,
856
  "rstrip": false,
857
- "normalized": false
 
858
  },
859
  {
860
  "id": 32094,
861
- "special": true,
862
  "content": "<extra_id_91>",
863
  "single_word": false,
864
  "lstrip": false,
865
  "rstrip": false,
866
- "normalized": false
 
867
  },
868
  {
869
  "id": 32095,
870
- "special": true,
871
  "content": "<extra_id_92>",
872
  "single_word": false,
873
  "lstrip": false,
874
  "rstrip": false,
875
- "normalized": false
 
876
  },
877
  {
878
  "id": 32096,
879
- "special": true,
880
  "content": "<extra_id_93>",
881
  "single_word": false,
882
  "lstrip": false,
883
  "rstrip": false,
884
- "normalized": false
 
885
  },
886
  {
887
  "id": 32097,
888
- "special": true,
889
  "content": "<extra_id_94>",
890
  "single_word": false,
891
  "lstrip": false,
892
  "rstrip": false,
893
- "normalized": false
 
894
  },
895
  {
896
  "id": 32098,
897
- "special": true,
898
  "content": "<extra_id_95>",
899
  "single_word": false,
900
  "lstrip": false,
901
  "rstrip": false,
902
- "normalized": false
 
903
  },
904
  {
905
  "id": 32099,
906
- "special": true,
907
  "content": "<extra_id_96>",
908
  "single_word": false,
909
  "lstrip": false,
910
  "rstrip": false,
911
- "normalized": false
 
912
  },
913
  {
914
  "id": 32100,
915
- "special": true,
916
  "content": "<extra_id_97>",
917
  "single_word": false,
918
  "lstrip": false,
919
  "rstrip": false,
920
- "normalized": false
 
921
  },
922
  {
923
  "id": 32101,
924
- "special": true,
925
  "content": "<extra_id_98>",
926
  "single_word": false,
927
  "lstrip": false,
928
  "rstrip": false,
929
- "normalized": false
 
930
  },
931
  {
932
  "id": 32102,
933
- "special": true,
934
  "content": "<extra_id_99>",
935
  "single_word": false,
936
  "lstrip": false,
937
  "rstrip": false,
938
- "normalized": false
 
939
  }
940
  ],
941
  "normalizer": {
 
1
  {
2
  "version": "1.0",
3
+ "truncation": {
4
+ "direction": "Right",
5
+ "max_length": 64,
6
+ "strategy": "LongestFirst",
7
+ "stride": 0
8
+ },
9
  "padding": {
10
  "strategy": "BatchLongest",
11
  "direction": "Right",
 
17
  "added_tokens": [
18
  {
19
  "id": 0,
 
20
  "content": "<pad>",
21
  "single_word": false,
22
  "lstrip": false,
23
  "rstrip": false,
24
+ "normalized": false,
25
+ "special": true
26
  },
27
  {
28
  "id": 1,
 
29
  "content": "</s>",
30
  "single_word": false,
31
  "lstrip": false,
32
  "rstrip": false,
33
+ "normalized": false,
34
+ "special": true
35
  },
36
  {
37
  "id": 2,
 
38
  "content": "<unk>",
39
  "single_word": false,
40
  "lstrip": false,
41
  "rstrip": false,
42
+ "normalized": false,
43
+ "special": true
44
  },
45
  {
46
  "id": 32003,
 
47
  "content": "<extra_id_0>",
48
  "single_word": false,
49
  "lstrip": false,
50
  "rstrip": false,
51
+ "normalized": false,
52
+ "special": true
53
  },
54
  {
55
  "id": 32004,
 
56
  "content": "<extra_id_1>",
57
  "single_word": false,
58
  "lstrip": false,
59
  "rstrip": false,
60
+ "normalized": false,
61
+ "special": true
62
  },
63
  {
64
  "id": 32005,
 
65
  "content": "<extra_id_2>",
66
  "single_word": false,
67
  "lstrip": false,
68
  "rstrip": false,
69
+ "normalized": false,
70
+ "special": true
71
  },
72
  {
73
  "id": 32006,
 
74
  "content": "<extra_id_3>",
75
  "single_word": false,
76
  "lstrip": false,
77
  "rstrip": false,
78
+ "normalized": false,
79
+ "special": true
80
  },
81
  {
82
  "id": 32007,
 
83
  "content": "<extra_id_4>",
84
  "single_word": false,
85
  "lstrip": false,
86
  "rstrip": false,
87
+ "normalized": false,
88
+ "special": true
89
  },
90
  {
91
  "id": 32008,
 
92
  "content": "<extra_id_5>",
93
  "single_word": false,
94
  "lstrip": false,
95
  "rstrip": false,
96
+ "normalized": false,
97
+ "special": true
98
  },
99
  {
100
  "id": 32009,
 
101
  "content": "<extra_id_6>",
102
  "single_word": false,
103
  "lstrip": false,
104
  "rstrip": false,
105
+ "normalized": false,
106
+ "special": true
107
  },
108
  {
109
  "id": 32010,
 
110
  "content": "<extra_id_7>",
111
  "single_word": false,
112
  "lstrip": false,
113
  "rstrip": false,
114
+ "normalized": false,
115
+ "special": true
116
  },
117
  {
118
  "id": 32011,
 
119
  "content": "<extra_id_8>",
120
  "single_word": false,
121
  "lstrip": false,
122
  "rstrip": false,
123
+ "normalized": false,
124
+ "special": true
125
  },
126
  {
127
  "id": 32012,
 
128
  "content": "<extra_id_9>",
129
  "single_word": false,
130
  "lstrip": false,
131
  "rstrip": false,
132
+ "normalized": false,
133
+ "special": true
134
  },
135
  {
136
  "id": 32013,
 
137
  "content": "<extra_id_10>",
138
  "single_word": false,
139
  "lstrip": false,
140
  "rstrip": false,
141
+ "normalized": false,
142
+ "special": true
143
  },
144
  {
145
  "id": 32014,
 
146
  "content": "<extra_id_11>",
147
  "single_word": false,
148
  "lstrip": false,
149
  "rstrip": false,
150
+ "normalized": false,
151
+ "special": true
152
  },
153
  {
154
  "id": 32015,
 
155
  "content": "<extra_id_12>",
156
  "single_word": false,
157
  "lstrip": false,
158
  "rstrip": false,
159
+ "normalized": false,
160
+ "special": true
161
  },
162
  {
163
  "id": 32016,
 
164
  "content": "<extra_id_13>",
165
  "single_word": false,
166
  "lstrip": false,
167
  "rstrip": false,
168
+ "normalized": false,
169
+ "special": true
170
  },
171
  {
172
  "id": 32017,
 
173
  "content": "<extra_id_14>",
174
  "single_word": false,
175
  "lstrip": false,
176
  "rstrip": false,
177
+ "normalized": false,
178
+ "special": true
179
  },
180
  {
181
  "id": 32018,
 
182
  "content": "<extra_id_15>",
183
  "single_word": false,
184
  "lstrip": false,
185
  "rstrip": false,
186
+ "normalized": false,
187
+ "special": true
188
  },
189
  {
190
  "id": 32019,
 
191
  "content": "<extra_id_16>",
192
  "single_word": false,
193
  "lstrip": false,
194
  "rstrip": false,
195
+ "normalized": false,
196
+ "special": true
197
  },
198
  {
199
  "id": 32020,
 
200
  "content": "<extra_id_17>",
201
  "single_word": false,
202
  "lstrip": false,
203
  "rstrip": false,
204
+ "normalized": false,
205
+ "special": true
206
  },
207
  {
208
  "id": 32021,
 
209
  "content": "<extra_id_18>",
210
  "single_word": false,
211
  "lstrip": false,
212
  "rstrip": false,
213
+ "normalized": false,
214
+ "special": true
215
  },
216
  {
217
  "id": 32022,
 
218
  "content": "<extra_id_19>",
219
  "single_word": false,
220
  "lstrip": false,
221
  "rstrip": false,
222
+ "normalized": false,
223
+ "special": true
224
  },
225
  {
226
  "id": 32023,
 
227
  "content": "<extra_id_20>",
228
  "single_word": false,
229
  "lstrip": false,
230
  "rstrip": false,
231
+ "normalized": false,
232
+ "special": true
233
  },
234
  {
235
  "id": 32024,
 
236
  "content": "<extra_id_21>",
237
  "single_word": false,
238
  "lstrip": false,
239
  "rstrip": false,
240
+ "normalized": false,
241
+ "special": true
242
  },
243
  {
244
  "id": 32025,
 
245
  "content": "<extra_id_22>",
246
  "single_word": false,
247
  "lstrip": false,
248
  "rstrip": false,
249
+ "normalized": false,
250
+ "special": true
251
  },
252
  {
253
  "id": 32026,
 
254
  "content": "<extra_id_23>",
255
  "single_word": false,
256
  "lstrip": false,
257
  "rstrip": false,
258
+ "normalized": false,
259
+ "special": true
260
  },
261
  {
262
  "id": 32027,
 
263
  "content": "<extra_id_24>",
264
  "single_word": false,
265
  "lstrip": false,
266
  "rstrip": false,
267
+ "normalized": false,
268
+ "special": true
269
  },
270
  {
271
  "id": 32028,
 
272
  "content": "<extra_id_25>",
273
  "single_word": false,
274
  "lstrip": false,
275
  "rstrip": false,
276
+ "normalized": false,
277
+ "special": true
278
  },
279
  {
280
  "id": 32029,
 
281
  "content": "<extra_id_26>",
282
  "single_word": false,
283
  "lstrip": false,
284
  "rstrip": false,
285
+ "normalized": false,
286
+ "special": true
287
  },
288
  {
289
  "id": 32030,
 
290
  "content": "<extra_id_27>",
291
  "single_word": false,
292
  "lstrip": false,
293
  "rstrip": false,
294
+ "normalized": false,
295
+ "special": true
296
  },
297
  {
298
  "id": 32031,
 
299
  "content": "<extra_id_28>",
300
  "single_word": false,
301
  "lstrip": false,
302
  "rstrip": false,
303
+ "normalized": false,
304
+ "special": true
305
  },
306
  {
307
  "id": 32032,
 
308
  "content": "<extra_id_29>",
309
  "single_word": false,
310
  "lstrip": false,
311
  "rstrip": false,
312
+ "normalized": false,
313
+ "special": true
314
  },
315
  {
316
  "id": 32033,
 
317
  "content": "<extra_id_30>",
318
  "single_word": false,
319
  "lstrip": false,
320
  "rstrip": false,
321
+ "normalized": false,
322
+ "special": true
323
  },
324
  {
325
  "id": 32034,
 
326
  "content": "<extra_id_31>",
327
  "single_word": false,
328
  "lstrip": false,
329
  "rstrip": false,
330
+ "normalized": false,
331
+ "special": true
332
  },
333
  {
334
  "id": 32035,
 
335
  "content": "<extra_id_32>",
336
  "single_word": false,
337
  "lstrip": false,
338
  "rstrip": false,
339
+ "normalized": false,
340
+ "special": true
341
  },
342
  {
343
  "id": 32036,
 
344
  "content": "<extra_id_33>",
345
  "single_word": false,
346
  "lstrip": false,
347
  "rstrip": false,
348
+ "normalized": false,
349
+ "special": true
350
  },
351
  {
352
  "id": 32037,
 
353
  "content": "<extra_id_34>",
354
  "single_word": false,
355
  "lstrip": false,
356
  "rstrip": false,
357
+ "normalized": false,
358
+ "special": true
359
  },
360
  {
361
  "id": 32038,
 
362
  "content": "<extra_id_35>",
363
  "single_word": false,
364
  "lstrip": false,
365
  "rstrip": false,
366
+ "normalized": false,
367
+ "special": true
368
  },
369
  {
370
  "id": 32039,
 
371
  "content": "<extra_id_36>",
372
  "single_word": false,
373
  "lstrip": false,
374
  "rstrip": false,
375
+ "normalized": false,
376
+ "special": true
377
  },
378
  {
379
  "id": 32040,
 
380
  "content": "<extra_id_37>",
381
  "single_word": false,
382
  "lstrip": false,
383
  "rstrip": false,
384
+ "normalized": false,
385
+ "special": true
386
  },
387
  {
388
  "id": 32041,
 
389
  "content": "<extra_id_38>",
390
  "single_word": false,
391
  "lstrip": false,
392
  "rstrip": false,
393
+ "normalized": false,
394
+ "special": true
395
  },
396
  {
397
  "id": 32042,
 
398
  "content": "<extra_id_39>",
399
  "single_word": false,
400
  "lstrip": false,
401
  "rstrip": false,
402
+ "normalized": false,
403
+ "special": true
404
  },
405
  {
406
  "id": 32043,
 
407
  "content": "<extra_id_40>",
408
  "single_word": false,
409
  "lstrip": false,
410
  "rstrip": false,
411
+ "normalized": false,
412
+ "special": true
413
  },
414
  {
415
  "id": 32044,
 
416
  "content": "<extra_id_41>",
417
  "single_word": false,
418
  "lstrip": false,
419
  "rstrip": false,
420
+ "normalized": false,
421
+ "special": true
422
  },
423
  {
424
  "id": 32045,
 
425
  "content": "<extra_id_42>",
426
  "single_word": false,
427
  "lstrip": false,
428
  "rstrip": false,
429
+ "normalized": false,
430
+ "special": true
431
  },
432
  {
433
  "id": 32046,
 
434
  "content": "<extra_id_43>",
435
  "single_word": false,
436
  "lstrip": false,
437
  "rstrip": false,
438
+ "normalized": false,
439
+ "special": true
440
  },
441
  {
442
  "id": 32047,
 
443
  "content": "<extra_id_44>",
444
  "single_word": false,
445
  "lstrip": false,
446
  "rstrip": false,
447
+ "normalized": false,
448
+ "special": true
449
  },
450
  {
451
  "id": 32048,
 
452
  "content": "<extra_id_45>",
453
  "single_word": false,
454
  "lstrip": false,
455
  "rstrip": false,
456
+ "normalized": false,
457
+ "special": true
458
  },
459
  {
460
  "id": 32049,
 
461
  "content": "<extra_id_46>",
462
  "single_word": false,
463
  "lstrip": false,
464
  "rstrip": false,
465
+ "normalized": false,
466
+ "special": true
467
  },
468
  {
469
  "id": 32050,
 
470
  "content": "<extra_id_47>",
471
  "single_word": false,
472
  "lstrip": false,
473
  "rstrip": false,
474
+ "normalized": false,
475
+ "special": true
476
  },
477
  {
478
  "id": 32051,
 
479
  "content": "<extra_id_48>",
480
  "single_word": false,
481
  "lstrip": false,
482
  "rstrip": false,
483
+ "normalized": false,
484
+ "special": true
485
  },
486
  {
487
  "id": 32052,
 
488
  "content": "<extra_id_49>",
489
  "single_word": false,
490
  "lstrip": false,
491
  "rstrip": false,
492
+ "normalized": false,
493
+ "special": true
494
  },
495
  {
496
  "id": 32053,
 
497
  "content": "<extra_id_50>",
498
  "single_word": false,
499
  "lstrip": false,
500
  "rstrip": false,
501
+ "normalized": false,
502
+ "special": true
503
  },
504
  {
505
  "id": 32054,
 
506
  "content": "<extra_id_51>",
507
  "single_word": false,
508
  "lstrip": false,
509
  "rstrip": false,
510
+ "normalized": false,
511
+ "special": true
512
  },
513
  {
514
  "id": 32055,
 
515
  "content": "<extra_id_52>",
516
  "single_word": false,
517
  "lstrip": false,
518
  "rstrip": false,
519
+ "normalized": false,
520
+ "special": true
521
  },
522
  {
523
  "id": 32056,
 
524
  "content": "<extra_id_53>",
525
  "single_word": false,
526
  "lstrip": false,
527
  "rstrip": false,
528
+ "normalized": false,
529
+ "special": true
530
  },
531
  {
532
  "id": 32057,
 
533
  "content": "<extra_id_54>",
534
  "single_word": false,
535
  "lstrip": false,
536
  "rstrip": false,
537
+ "normalized": false,
538
+ "special": true
539
  },
540
  {
541
  "id": 32058,
 
542
  "content": "<extra_id_55>",
543
  "single_word": false,
544
  "lstrip": false,
545
  "rstrip": false,
546
+ "normalized": false,
547
+ "special": true
548
  },
549
  {
550
  "id": 32059,
 
551
  "content": "<extra_id_56>",
552
  "single_word": false,
553
  "lstrip": false,
554
  "rstrip": false,
555
+ "normalized": false,
556
+ "special": true
557
  },
558
  {
559
  "id": 32060,
 
560
  "content": "<extra_id_57>",
561
  "single_word": false,
562
  "lstrip": false,
563
  "rstrip": false,
564
+ "normalized": false,
565
+ "special": true
566
  },
567
  {
568
  "id": 32061,
 
569
  "content": "<extra_id_58>",
570
  "single_word": false,
571
  "lstrip": false,
572
  "rstrip": false,
573
+ "normalized": false,
574
+ "special": true
575
  },
576
  {
577
  "id": 32062,
 
578
  "content": "<extra_id_59>",
579
  "single_word": false,
580
  "lstrip": false,
581
  "rstrip": false,
582
+ "normalized": false,
583
+ "special": true
584
  },
585
  {
586
  "id": 32063,
 
587
  "content": "<extra_id_60>",
588
  "single_word": false,
589
  "lstrip": false,
590
  "rstrip": false,
591
+ "normalized": false,
592
+ "special": true
593
  },
594
  {
595
  "id": 32064,
 
596
  "content": "<extra_id_61>",
597
  "single_word": false,
598
  "lstrip": false,
599
  "rstrip": false,
600
+ "normalized": false,
601
+ "special": true
602
  },
603
  {
604
  "id": 32065,
 
605
  "content": "<extra_id_62>",
606
  "single_word": false,
607
  "lstrip": false,
608
  "rstrip": false,
609
+ "normalized": false,
610
+ "special": true
611
  },
612
  {
613
  "id": 32066,
 
614
  "content": "<extra_id_63>",
615
  "single_word": false,
616
  "lstrip": false,
617
  "rstrip": false,
618
+ "normalized": false,
619
+ "special": true
620
  },
621
  {
622
  "id": 32067,
 
623
  "content": "<extra_id_64>",
624
  "single_word": false,
625
  "lstrip": false,
626
  "rstrip": false,
627
+ "normalized": false,
628
+ "special": true
629
  },
630
  {
631
  "id": 32068,
 
632
  "content": "<extra_id_65>",
633
  "single_word": false,
634
  "lstrip": false,
635
  "rstrip": false,
636
+ "normalized": false,
637
+ "special": true
638
  },
639
  {
640
  "id": 32069,
 
641
  "content": "<extra_id_66>",
642
  "single_word": false,
643
  "lstrip": false,
644
  "rstrip": false,
645
+ "normalized": false,
646
+ "special": true
647
  },
648
  {
649
  "id": 32070,
 
650
  "content": "<extra_id_67>",
651
  "single_word": false,
652
  "lstrip": false,
653
  "rstrip": false,
654
+ "normalized": false,
655
+ "special": true
656
  },
657
  {
658
  "id": 32071,
 
659
  "content": "<extra_id_68>",
660
  "single_word": false,
661
  "lstrip": false,
662
  "rstrip": false,
663
+ "normalized": false,
664
+ "special": true
665
  },
666
  {
667
  "id": 32072,
 
668
  "content": "<extra_id_69>",
669
  "single_word": false,
670
  "lstrip": false,
671
  "rstrip": false,
672
+ "normalized": false,
673
+ "special": true
674
  },
675
  {
676
  "id": 32073,
 
677
  "content": "<extra_id_70>",
678
  "single_word": false,
679
  "lstrip": false,
680
  "rstrip": false,
681
+ "normalized": false,
682
+ "special": true
683
  },
684
  {
685
  "id": 32074,
 
686
  "content": "<extra_id_71>",
687
  "single_word": false,
688
  "lstrip": false,
689
  "rstrip": false,
690
+ "normalized": false,
691
+ "special": true
692
  },
693
  {
694
  "id": 32075,
 
695
  "content": "<extra_id_72>",
696
  "single_word": false,
697
  "lstrip": false,
698
  "rstrip": false,
699
+ "normalized": false,
700
+ "special": true
701
  },
702
  {
703
  "id": 32076,
 
704
  "content": "<extra_id_73>",
705
  "single_word": false,
706
  "lstrip": false,
707
  "rstrip": false,
708
+ "normalized": false,
709
+ "special": true
710
  },
711
  {
712
  "id": 32077,
 
713
  "content": "<extra_id_74>",
714
  "single_word": false,
715
  "lstrip": false,
716
  "rstrip": false,
717
+ "normalized": false,
718
+ "special": true
719
  },
720
  {
721
  "id": 32078,
 
722
  "content": "<extra_id_75>",
723
  "single_word": false,
724
  "lstrip": false,
725
  "rstrip": false,
726
+ "normalized": false,
727
+ "special": true
728
  },
729
  {
730
  "id": 32079,
 
731
  "content": "<extra_id_76>",
732
  "single_word": false,
733
  "lstrip": false,
734
  "rstrip": false,
735
+ "normalized": false,
736
+ "special": true
737
  },
738
  {
739
  "id": 32080,
 
740
  "content": "<extra_id_77>",
741
  "single_word": false,
742
  "lstrip": false,
743
  "rstrip": false,
744
+ "normalized": false,
745
+ "special": true
746
  },
747
  {
748
  "id": 32081,
 
749
  "content": "<extra_id_78>",
750
  "single_word": false,
751
  "lstrip": false,
752
  "rstrip": false,
753
+ "normalized": false,
754
+ "special": true
755
  },
756
  {
757
  "id": 32082,
 
758
  "content": "<extra_id_79>",
759
  "single_word": false,
760
  "lstrip": false,
761
  "rstrip": false,
762
+ "normalized": false,
763
+ "special": true
764
  },
765
  {
766
  "id": 32083,
 
767
  "content": "<extra_id_80>",
768
  "single_word": false,
769
  "lstrip": false,
770
  "rstrip": false,
771
+ "normalized": false,
772
+ "special": true
773
  },
774
  {
775
  "id": 32084,
 
776
  "content": "<extra_id_81>",
777
  "single_word": false,
778
  "lstrip": false,
779
  "rstrip": false,
780
+ "normalized": false,
781
+ "special": true
782
  },
783
  {
784
  "id": 32085,
 
785
  "content": "<extra_id_82>",
786
  "single_word": false,
787
  "lstrip": false,
788
  "rstrip": false,
789
+ "normalized": false,
790
+ "special": true
791
  },
792
  {
793
  "id": 32086,
 
794
  "content": "<extra_id_83>",
795
  "single_word": false,
796
  "lstrip": false,
797
  "rstrip": false,
798
+ "normalized": false,
799
+ "special": true
800
  },
801
  {
802
  "id": 32087,
 
803
  "content": "<extra_id_84>",
804
  "single_word": false,
805
  "lstrip": false,
806
  "rstrip": false,
807
+ "normalized": false,
808
+ "special": true
809
  },
810
  {
811
  "id": 32088,
 
812
  "content": "<extra_id_85>",
813
  "single_word": false,
814
  "lstrip": false,
815
  "rstrip": false,
816
+ "normalized": false,
817
+ "special": true
818
  },
819
  {
820
  "id": 32089,
 
821
  "content": "<extra_id_86>",
822
  "single_word": false,
823
  "lstrip": false,
824
  "rstrip": false,
825
+ "normalized": false,
826
+ "special": true
827
  },
828
  {
829
  "id": 32090,
 
830
  "content": "<extra_id_87>",
831
  "single_word": false,
832
  "lstrip": false,
833
  "rstrip": false,
834
+ "normalized": false,
835
+ "special": true
836
  },
837
  {
838
  "id": 32091,
 
839
  "content": "<extra_id_88>",
840
  "single_word": false,
841
  "lstrip": false,
842
  "rstrip": false,
843
+ "normalized": false,
844
+ "special": true
845
  },
846
  {
847
  "id": 32092,
 
848
  "content": "<extra_id_89>",
849
  "single_word": false,
850
  "lstrip": false,
851
  "rstrip": false,
852
+ "normalized": false,
853
+ "special": true
854
  },
855
  {
856
  "id": 32093,
 
857
  "content": "<extra_id_90>",
858
  "single_word": false,
859
  "lstrip": false,
860
  "rstrip": false,
861
+ "normalized": false,
862
+ "special": true
863
  },
864
  {
865
  "id": 32094,
 
866
  "content": "<extra_id_91>",
867
  "single_word": false,
868
  "lstrip": false,
869
  "rstrip": false,
870
+ "normalized": false,
871
+ "special": true
872
  },
873
  {
874
  "id": 32095,
 
875
  "content": "<extra_id_92>",
876
  "single_word": false,
877
  "lstrip": false,
878
  "rstrip": false,
879
+ "normalized": false,
880
+ "special": true
881
  },
882
  {
883
  "id": 32096,
 
884
  "content": "<extra_id_93>",
885
  "single_word": false,
886
  "lstrip": false,
887
  "rstrip": false,
888
+ "normalized": false,
889
+ "special": true
890
  },
891
  {
892
  "id": 32097,
 
893
  "content": "<extra_id_94>",
894
  "single_word": false,
895
  "lstrip": false,
896
  "rstrip": false,
897
+ "normalized": false,
898
+ "special": true
899
  },
900
  {
901
  "id": 32098,
 
902
  "content": "<extra_id_95>",
903
  "single_word": false,
904
  "lstrip": false,
905
  "rstrip": false,
906
+ "normalized": false,
907
+ "special": true
908
  },
909
  {
910
  "id": 32099,
 
911
  "content": "<extra_id_96>",
912
  "single_word": false,
913
  "lstrip": false,
914
  "rstrip": false,
915
+ "normalized": false,
916
+ "special": true
917
  },
918
  {
919
  "id": 32100,
 
920
  "content": "<extra_id_97>",
921
  "single_word": false,
922
  "lstrip": false,
923
  "rstrip": false,
924
+ "normalized": false,
925
+ "special": true
926
  },
927
  {
928
  "id": 32101,
 
929
  "content": "<extra_id_98>",
930
  "single_word": false,
931
  "lstrip": false,
932
  "rstrip": false,
933
+ "normalized": false,
934
+ "special": true
935
  },
936
  {
937
  "id": 32102,
 
938
  "content": "<extra_id_99>",
939
  "single_word": false,
940
  "lstrip": false,
941
  "rstrip": false,
942
+ "normalized": false,
943
+ "special": true
944
  }
945
  ],
946
  "normalizer": {
tokenizer_config.json CHANGED
@@ -1 +1,111 @@
1
- {"eos_token": "</s>", "unk_token": "<unk>", "pad_token": "<pad>", "extra_ids": 100, "additional_special_tokens": ["<extra_id_0>", "<extra_id_1>", "<extra_id_2>", "<extra_id_3>", "<extra_id_4>", "<extra_id_5>", "<extra_id_6>", "<extra_id_7>", "<extra_id_8>", "<extra_id_9>", "<extra_id_10>", "<extra_id_11>", "<extra_id_12>", "<extra_id_13>", "<extra_id_14>", "<extra_id_15>", "<extra_id_16>", "<extra_id_17>", "<extra_id_18>", "<extra_id_19>", "<extra_id_20>", "<extra_id_21>", "<extra_id_22>", "<extra_id_23>", "<extra_id_24>", "<extra_id_25>", "<extra_id_26>", "<extra_id_27>", "<extra_id_28>", "<extra_id_29>", "<extra_id_30>", "<extra_id_31>", "<extra_id_32>", "<extra_id_33>", "<extra_id_34>", "<extra_id_35>", "<extra_id_36>", "<extra_id_37>", "<extra_id_38>", "<extra_id_39>", "<extra_id_40>", "<extra_id_41>", "<extra_id_42>", "<extra_id_43>", "<extra_id_44>", "<extra_id_45>", "<extra_id_46>", "<extra_id_47>", "<extra_id_48>", "<extra_id_49>", "<extra_id_50>", "<extra_id_51>", "<extra_id_52>", "<extra_id_53>", "<extra_id_54>", "<extra_id_55>", "<extra_id_56>", "<extra_id_57>", "<extra_id_58>", "<extra_id_59>", "<extra_id_60>", "<extra_id_61>", "<extra_id_62>", "<extra_id_63>", "<extra_id_64>", "<extra_id_65>", "<extra_id_66>", "<extra_id_67>", "<extra_id_68>", "<extra_id_69>", "<extra_id_70>", "<extra_id_71>", "<extra_id_72>", "<extra_id_73>", "<extra_id_74>", "<extra_id_75>", "<extra_id_76>", "<extra_id_77>", "<extra_id_78>", "<extra_id_79>", "<extra_id_80>", "<extra_id_81>", "<extra_id_82>", "<extra_id_83>", "<extra_id_84>", "<extra_id_85>", "<extra_id_86>", "<extra_id_87>", "<extra_id_88>", "<extra_id_89>", "<extra_id_90>", "<extra_id_91>", "<extra_id_92>", "<extra_id_93>", "<extra_id_94>", "<extra_id_95>", "<extra_id_96>", "<extra_id_97>", "<extra_id_98>", "<extra_id_99>"], "special_tokens_map_file": null, "name_or_path": "/gd2/MyDrive/models/persian-t5-style-paraphraser", "tokenizer_class": "T5Tokenizer"}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "additional_special_tokens": [
3
+ "<extra_id_0>",
4
+ "<extra_id_1>",
5
+ "<extra_id_2>",
6
+ "<extra_id_3>",
7
+ "<extra_id_4>",
8
+ "<extra_id_5>",
9
+ "<extra_id_6>",
10
+ "<extra_id_7>",
11
+ "<extra_id_8>",
12
+ "<extra_id_9>",
13
+ "<extra_id_10>",
14
+ "<extra_id_11>",
15
+ "<extra_id_12>",
16
+ "<extra_id_13>",
17
+ "<extra_id_14>",
18
+ "<extra_id_15>",
19
+ "<extra_id_16>",
20
+ "<extra_id_17>",
21
+ "<extra_id_18>",
22
+ "<extra_id_19>",
23
+ "<extra_id_20>",
24
+ "<extra_id_21>",
25
+ "<extra_id_22>",
26
+ "<extra_id_23>",
27
+ "<extra_id_24>",
28
+ "<extra_id_25>",
29
+ "<extra_id_26>",
30
+ "<extra_id_27>",
31
+ "<extra_id_28>",
32
+ "<extra_id_29>",
33
+ "<extra_id_30>",
34
+ "<extra_id_31>",
35
+ "<extra_id_32>",
36
+ "<extra_id_33>",
37
+ "<extra_id_34>",
38
+ "<extra_id_35>",
39
+ "<extra_id_36>",
40
+ "<extra_id_37>",
41
+ "<extra_id_38>",
42
+ "<extra_id_39>",
43
+ "<extra_id_40>",
44
+ "<extra_id_41>",
45
+ "<extra_id_42>",
46
+ "<extra_id_43>",
47
+ "<extra_id_44>",
48
+ "<extra_id_45>",
49
+ "<extra_id_46>",
50
+ "<extra_id_47>",
51
+ "<extra_id_48>",
52
+ "<extra_id_49>",
53
+ "<extra_id_50>",
54
+ "<extra_id_51>",
55
+ "<extra_id_52>",
56
+ "<extra_id_53>",
57
+ "<extra_id_54>",
58
+ "<extra_id_55>",
59
+ "<extra_id_56>",
60
+ "<extra_id_57>",
61
+ "<extra_id_58>",
62
+ "<extra_id_59>",
63
+ "<extra_id_60>",
64
+ "<extra_id_61>",
65
+ "<extra_id_62>",
66
+ "<extra_id_63>",
67
+ "<extra_id_64>",
68
+ "<extra_id_65>",
69
+ "<extra_id_66>",
70
+ "<extra_id_67>",
71
+ "<extra_id_68>",
72
+ "<extra_id_69>",
73
+ "<extra_id_70>",
74
+ "<extra_id_71>",
75
+ "<extra_id_72>",
76
+ "<extra_id_73>",
77
+ "<extra_id_74>",
78
+ "<extra_id_75>",
79
+ "<extra_id_76>",
80
+ "<extra_id_77>",
81
+ "<extra_id_78>",
82
+ "<extra_id_79>",
83
+ "<extra_id_80>",
84
+ "<extra_id_81>",
85
+ "<extra_id_82>",
86
+ "<extra_id_83>",
87
+ "<extra_id_84>",
88
+ "<extra_id_85>",
89
+ "<extra_id_86>",
90
+ "<extra_id_87>",
91
+ "<extra_id_88>",
92
+ "<extra_id_89>",
93
+ "<extra_id_90>",
94
+ "<extra_id_91>",
95
+ "<extra_id_92>",
96
+ "<extra_id_93>",
97
+ "<extra_id_94>",
98
+ "<extra_id_95>",
99
+ "<extra_id_96>",
100
+ "<extra_id_97>",
101
+ "<extra_id_98>",
102
+ "<extra_id_99>"
103
+ ],
104
+ "eos_token": "</s>",
105
+ "extra_ids": 100,
106
+ "name_or_path": "/content/drive/MyDrive/models/st-with-paraphraser-weights-sentinel-token",
107
+ "pad_token": "<pad>",
108
+ "special_tokens_map_file": null,
109
+ "tokenizer_class": "T5Tokenizer",
110
+ "unk_token": "<unk>"
111
+ }