Hugging Face's logo

getrajeev03
/
flan-t5-base-samsum

Text2Text Generation
Transformers
PyTorch
TensorBoard
t5
Generated from Trainer
Eval Results
Inference Endpoints
text-generation-inference
Model card Files Metrics Community
2
getrajeev03 commited on
Commit
ed39dfa
•
1 Parent(s): 9ca3b2a

End of training

Browse files
Files changed (5) hide show
  1. README.md +11 -11
  2. logs/events.out.tfevents.1687525193.notebook-rt222gpu1d283621de18747e3b0384bacd7ed0ecf-95f94bfsrpkk.225.0 +2 -2
  3. logs/events.out.tfevents.1687526155.notebook-rt222gpu1d283621de18747e3b0384bacd7ed0ecf-95f94bfsrpkk.225.1 +3 -0
  4. spiece.model +3 -0
  5. tokenizer.json +206 -206
README.md CHANGED
@@ -21,7 +21,7 @@ model-index:
21
  metrics:
22
  - name: Rouge1
23
  type: rouge
24
- value: 46.7607
25
  ---
26
 
27
  <!-- This model card has been generated automatically according to the information the Trainer had access to. You
@@ -31,12 +31,12 @@ should probably proofread and complete it, then remove this comment. -->
31
 
32
  This model is a fine-tuned version of [google/flan-t5-base](https://huggingface.co/google/flan-t5-base) on the samsum dataset.
33
  It achieves the following results on the evaluation set:
34
- - Loss: 1.3871
35
- - Rouge1: 46.7607
36
- - Rouge2: 23.2022
37
- - Rougel: 39.2704
38
- - Rougelsum: 43.0568
39
- - Gen Len: 17.2259
40
 
41
  ## Model description
42
 
@@ -67,12 +67,12 @@ The following hyperparameters were used during training:
67
 
68
  | Training Loss | Epoch | Step | Validation Loss | Rouge1 | Rouge2 | Rougel | Rougelsum | Gen Len |
69
  |:-------------:|:-----:|:----:|:---------------:|:-------:|:-------:|:-------:|:---------:|:-------:|
70
- | 1.4376 | 1.0 | 1842 | 1.3871 | 46.7607 | 23.2022 | 39.2704 | 43.0568 | 17.2259 |
71
 
72
 
73
  ### Framework versions
74
 
75
  - Transformers 4.30.2
76
- - Pytorch 2.0.1+cu118
77
- - Datasets 2.13.0
78
- - Tokenizers 0.13.3
 
21
  metrics:
22
  - name: Rouge1
23
  type: rouge
24
+ value: 46.7348
25
  ---
26
 
27
  <!-- This model card has been generated automatically according to the information the Trainer had access to. You
 
31
 
32
  This model is a fine-tuned version of [google/flan-t5-base](https://huggingface.co/google/flan-t5-base) on the samsum dataset.
33
  It achieves the following results on the evaluation set:
34
+ - Loss: 1.3861
35
+ - Rouge1: 46.7348
36
+ - Rouge2: 23.2044
37
+ - Rougel: 39.1916
38
+ - Rougelsum: 43.1409
39
+ - Gen Len: 17.1880
40
 
41
  ## Model description
42
 
 
67
 
68
  | Training Loss | Epoch | Step | Validation Loss | Rouge1 | Rouge2 | Rougel | Rougelsum | Gen Len |
69
  |:-------------:|:-----:|:----:|:---------------:|:-------:|:-------:|:-------:|:---------:|:-------:|
70
+ | 1.4378 | 1.0 | 1842 | 1.3861 | 46.7348 | 23.2044 | 39.1916 | 43.1409 | 17.1880 |
71
 
72
 
73
  ### Framework versions
74
 
75
  - Transformers 4.30.2
76
+ - Pytorch 1.12.1
77
+ - Datasets 2.13.1
78
+ - Tokenizers 0.11.0
logs/events.out.tfevents.1687525193.notebook-rt222gpu1d283621de18747e3b0384bacd7ed0ecf-95f94bfsrpkk.225.0 CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:b2ae41b629aebdb777e537e62eb989e860297be4f9e985d56e5ece245e7b4273
3
- size 6014
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:51d39339cb5745a63416b4735a5bd105d9fb8c58d0cf01b4876c375a30f469b2
3
+ size 6368
logs/events.out.tfevents.1687526155.notebook-rt222gpu1d283621de18747e3b0384bacd7ed0ecf-95f94bfsrpkk.225.1 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:4b81fd700213572dc14260dfe40dad34eadffbeb99a80a867f630581edee3ab5
3
+ size 613
spiece.model ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d60acb128cf7b7f2536e8f38a5b18a05535c9e14c7a355904270e15b0945ea86
3
+ size 791656
tokenizer.json CHANGED
@@ -19,930 +19,930 @@
19
  "added_tokens": [
20
  {
21
  "id": 0,
 
22
  "content": "<pad>",
23
  "single_word": false,
24
  "lstrip": false,
25
  "rstrip": false,
26
- "normalized": false,
27
- "special": true
28
  },
29
  {
30
  "id": 1,
 
31
  "content": "</s>",
32
  "single_word": false,
33
  "lstrip": false,
34
  "rstrip": false,
35
- "normalized": false,
36
- "special": true
37
  },
38
  {
39
  "id": 2,
 
40
  "content": "<unk>",
41
  "single_word": false,
42
  "lstrip": false,
43
  "rstrip": false,
44
- "normalized": false,
45
- "special": true
46
  },
47
  {
48
  "id": 32000,
 
49
  "content": "<extra_id_99>",
50
  "single_word": false,
51
  "lstrip": false,
52
  "rstrip": false,
53
- "normalized": false,
54
- "special": true
55
  },
56
  {
57
  "id": 32001,
 
58
  "content": "<extra_id_98>",
59
  "single_word": false,
60
  "lstrip": false,
61
  "rstrip": false,
62
- "normalized": false,
63
- "special": true
64
  },
65
  {
66
  "id": 32002,
 
67
  "content": "<extra_id_97>",
68
  "single_word": false,
69
  "lstrip": false,
70
  "rstrip": false,
71
- "normalized": false,
72
- "special": true
73
  },
74
  {
75
  "id": 32003,
 
76
  "content": "<extra_id_96>",
77
  "single_word": false,
78
  "lstrip": false,
79
  "rstrip": false,
80
- "normalized": false,
81
- "special": true
82
  },
83
  {
84
  "id": 32004,
 
85
  "content": "<extra_id_95>",
86
  "single_word": false,
87
  "lstrip": false,
88
  "rstrip": false,
89
- "normalized": false,
90
- "special": true
91
  },
92
  {
93
  "id": 32005,
 
94
  "content": "<extra_id_94>",
95
  "single_word": false,
96
  "lstrip": false,
97
  "rstrip": false,
98
- "normalized": false,
99
- "special": true
100
  },
101
  {
102
  "id": 32006,
 
103
  "content": "<extra_id_93>",
104
  "single_word": false,
105
  "lstrip": false,
106
  "rstrip": false,
107
- "normalized": false,
108
- "special": true
109
  },
110
  {
111
  "id": 32007,
 
112
  "content": "<extra_id_92>",
113
  "single_word": false,
114
  "lstrip": false,
115
  "rstrip": false,
116
- "normalized": false,
117
- "special": true
118
  },
119
  {
120
  "id": 32008,
 
121
  "content": "<extra_id_91>",
122
  "single_word": false,
123
  "lstrip": false,
124
  "rstrip": false,
125
- "normalized": false,
126
- "special": true
127
  },
128
  {
129
  "id": 32009,
 
130
  "content": "<extra_id_90>",
131
  "single_word": false,
132
  "lstrip": false,
133
  "rstrip": false,
134
- "normalized": false,
135
- "special": true
136
  },
137
  {
138
  "id": 32010,
 
139
  "content": "<extra_id_89>",
140
  "single_word": false,
141
  "lstrip": false,
142
  "rstrip": false,
143
- "normalized": false,
144
- "special": true
145
  },
146
  {
147
  "id": 32011,
 
148
  "content": "<extra_id_88>",
149
  "single_word": false,
150
  "lstrip": false,
151
  "rstrip": false,
152
- "normalized": false,
153
- "special": true
154
  },
155
  {
156
  "id": 32012,
 
157
  "content": "<extra_id_87>",
158
  "single_word": false,
159
  "lstrip": false,
160
  "rstrip": false,
161
- "normalized": false,
162
- "special": true
163
  },
164
  {
165
  "id": 32013,
 
166
  "content": "<extra_id_86>",
167
  "single_word": false,
168
  "lstrip": false,
169
  "rstrip": false,
170
- "normalized": false,
171
- "special": true
172
  },
173
  {
174
  "id": 32014,
 
175
  "content": "<extra_id_85>",
176
  "single_word": false,
177
  "lstrip": false,
178
  "rstrip": false,
179
- "normalized": false,
180
- "special": true
181
  },
182
  {
183
  "id": 32015,
 
184
  "content": "<extra_id_84>",
185
  "single_word": false,
186
  "lstrip": false,
187
  "rstrip": false,
188
- "normalized": false,
189
- "special": true
190
  },
191
  {
192
  "id": 32016,
 
193
  "content": "<extra_id_83>",
194
  "single_word": false,
195
  "lstrip": false,
196
  "rstrip": false,
197
- "normalized": false,
198
- "special": true
199
  },
200
  {
201
  "id": 32017,
 
202
  "content": "<extra_id_82>",
203
  "single_word": false,
204
  "lstrip": false,
205
  "rstrip": false,
206
- "normalized": false,
207
- "special": true
208
  },
209
  {
210
  "id": 32018,
 
211
  "content": "<extra_id_81>",
212
  "single_word": false,
213
  "lstrip": false,
214
  "rstrip": false,
215
- "normalized": false,
216
- "special": true
217
  },
218
  {
219
  "id": 32019,
 
220
  "content": "<extra_id_80>",
221
  "single_word": false,
222
  "lstrip": false,
223
  "rstrip": false,
224
- "normalized": false,
225
- "special": true
226
  },
227
  {
228
  "id": 32020,
 
229
  "content": "<extra_id_79>",
230
  "single_word": false,
231
  "lstrip": false,
232
  "rstrip": false,
233
- "normalized": false,
234
- "special": true
235
  },
236
  {
237
  "id": 32021,
 
238
  "content": "<extra_id_78>",
239
  "single_word": false,
240
  "lstrip": false,
241
  "rstrip": false,
242
- "normalized": false,
243
- "special": true
244
  },
245
  {
246
  "id": 32022,
 
247
  "content": "<extra_id_77>",
248
  "single_word": false,
249
  "lstrip": false,
250
  "rstrip": false,
251
- "normalized": false,
252
- "special": true
253
  },
254
  {
255
  "id": 32023,
 
256
  "content": "<extra_id_76>",
257
  "single_word": false,
258
  "lstrip": false,
259
  "rstrip": false,
260
- "normalized": false,
261
- "special": true
262
  },
263
  {
264
  "id": 32024,
 
265
  "content": "<extra_id_75>",
266
  "single_word": false,
267
  "lstrip": false,
268
  "rstrip": false,
269
- "normalized": false,
270
- "special": true
271
  },
272
  {
273
  "id": 32025,
 
274
  "content": "<extra_id_74>",
275
  "single_word": false,
276
  "lstrip": false,
277
  "rstrip": false,
278
- "normalized": false,
279
- "special": true
280
  },
281
  {
282
  "id": 32026,
 
283
  "content": "<extra_id_73>",
284
  "single_word": false,
285
  "lstrip": false,
286
  "rstrip": false,
287
- "normalized": false,
288
- "special": true
289
  },
290
  {
291
  "id": 32027,
 
292
  "content": "<extra_id_72>",
293
  "single_word": false,
294
  "lstrip": false,
295
  "rstrip": false,
296
- "normalized": false,
297
- "special": true
298
  },
299
  {
300
  "id": 32028,
 
301
  "content": "<extra_id_71>",
302
  "single_word": false,
303
  "lstrip": false,
304
  "rstrip": false,
305
- "normalized": false,
306
- "special": true
307
  },
308
  {
309
  "id": 32029,
 
310
  "content": "<extra_id_70>",
311
  "single_word": false,
312
  "lstrip": false,
313
  "rstrip": false,
314
- "normalized": false,
315
- "special": true
316
  },
317
  {
318
  "id": 32030,
 
319
  "content": "<extra_id_69>",
320
  "single_word": false,
321
  "lstrip": false,
322
  "rstrip": false,
323
- "normalized": false,
324
- "special": true
325
  },
326
  {
327
  "id": 32031,
 
328
  "content": "<extra_id_68>",
329
  "single_word": false,
330
  "lstrip": false,
331
  "rstrip": false,
332
- "normalized": false,
333
- "special": true
334
  },
335
  {
336
  "id": 32032,
 
337
  "content": "<extra_id_67>",
338
  "single_word": false,
339
  "lstrip": false,
340
  "rstrip": false,
341
- "normalized": false,
342
- "special": true
343
  },
344
  {
345
  "id": 32033,
 
346
  "content": "<extra_id_66>",
347
  "single_word": false,
348
  "lstrip": false,
349
  "rstrip": false,
350
- "normalized": false,
351
- "special": true
352
  },
353
  {
354
  "id": 32034,
 
355
  "content": "<extra_id_65>",
356
  "single_word": false,
357
  "lstrip": false,
358
  "rstrip": false,
359
- "normalized": false,
360
- "special": true
361
  },
362
  {
363
  "id": 32035,
 
364
  "content": "<extra_id_64>",
365
  "single_word": false,
366
  "lstrip": false,
367
  "rstrip": false,
368
- "normalized": false,
369
- "special": true
370
  },
371
  {
372
  "id": 32036,
 
373
  "content": "<extra_id_63>",
374
  "single_word": false,
375
  "lstrip": false,
376
  "rstrip": false,
377
- "normalized": false,
378
- "special": true
379
  },
380
  {
381
  "id": 32037,
 
382
  "content": "<extra_id_62>",
383
  "single_word": false,
384
  "lstrip": false,
385
  "rstrip": false,
386
- "normalized": false,
387
- "special": true
388
  },
389
  {
390
  "id": 32038,
 
391
  "content": "<extra_id_61>",
392
  "single_word": false,
393
  "lstrip": false,
394
  "rstrip": false,
395
- "normalized": false,
396
- "special": true
397
  },
398
  {
399
  "id": 32039,
 
400
  "content": "<extra_id_60>",
401
  "single_word": false,
402
  "lstrip": false,
403
  "rstrip": false,
404
- "normalized": false,
405
- "special": true
406
  },
407
  {
408
  "id": 32040,
 
409
  "content": "<extra_id_59>",
410
  "single_word": false,
411
  "lstrip": false,
412
  "rstrip": false,
413
- "normalized": false,
414
- "special": true
415
  },
416
  {
417
  "id": 32041,
 
418
  "content": "<extra_id_58>",
419
  "single_word": false,
420
  "lstrip": false,
421
  "rstrip": false,
422
- "normalized": false,
423
- "special": true
424
  },
425
  {
426
  "id": 32042,
 
427
  "content": "<extra_id_57>",
428
  "single_word": false,
429
  "lstrip": false,
430
  "rstrip": false,
431
- "normalized": false,
432
- "special": true
433
  },
434
  {
435
  "id": 32043,
 
436
  "content": "<extra_id_56>",
437
  "single_word": false,
438
  "lstrip": false,
439
  "rstrip": false,
440
- "normalized": false,
441
- "special": true
442
  },
443
  {
444
  "id": 32044,
 
445
  "content": "<extra_id_55>",
446
  "single_word": false,
447
  "lstrip": false,
448
  "rstrip": false,
449
- "normalized": false,
450
- "special": true
451
  },
452
  {
453
  "id": 32045,
 
454
  "content": "<extra_id_54>",
455
  "single_word": false,
456
  "lstrip": false,
457
  "rstrip": false,
458
- "normalized": false,
459
- "special": true
460
  },
461
  {
462
  "id": 32046,
 
463
  "content": "<extra_id_53>",
464
  "single_word": false,
465
  "lstrip": false,
466
  "rstrip": false,
467
- "normalized": false,
468
- "special": true
469
  },
470
  {
471
  "id": 32047,
 
472
  "content": "<extra_id_52>",
473
  "single_word": false,
474
  "lstrip": false,
475
  "rstrip": false,
476
- "normalized": false,
477
- "special": true
478
  },
479
  {
480
  "id": 32048,
 
481
  "content": "<extra_id_51>",
482
  "single_word": false,
483
  "lstrip": false,
484
  "rstrip": false,
485
- "normalized": false,
486
- "special": true
487
  },
488
  {
489
  "id": 32049,
 
490
  "content": "<extra_id_50>",
491
  "single_word": false,
492
  "lstrip": false,
493
  "rstrip": false,
494
- "normalized": false,
495
- "special": true
496
  },
497
  {
498
  "id": 32050,
 
499
  "content": "<extra_id_49>",
500
  "single_word": false,
501
  "lstrip": false,
502
  "rstrip": false,
503
- "normalized": false,
504
- "special": true
505
  },
506
  {
507
  "id": 32051,
 
508
  "content": "<extra_id_48>",
509
  "single_word": false,
510
  "lstrip": false,
511
  "rstrip": false,
512
- "normalized": false,
513
- "special": true
514
  },
515
  {
516
  "id": 32052,
 
517
  "content": "<extra_id_47>",
518
  "single_word": false,
519
  "lstrip": false,
520
  "rstrip": false,
521
- "normalized": false,
522
- "special": true
523
  },
524
  {
525
  "id": 32053,
 
526
  "content": "<extra_id_46>",
527
  "single_word": false,
528
  "lstrip": false,
529
  "rstrip": false,
530
- "normalized": false,
531
- "special": true
532
  },
533
  {
534
  "id": 32054,
 
535
  "content": "<extra_id_45>",
536
  "single_word": false,
537
  "lstrip": false,
538
  "rstrip": false,
539
- "normalized": false,
540
- "special": true
541
  },
542
  {
543
  "id": 32055,
 
544
  "content": "<extra_id_44>",
545
  "single_word": false,
546
  "lstrip": false,
547
  "rstrip": false,
548
- "normalized": false,
549
- "special": true
550
  },
551
  {
552
  "id": 32056,
 
553
  "content": "<extra_id_43>",
554
  "single_word": false,
555
  "lstrip": false,
556
  "rstrip": false,
557
- "normalized": false,
558
- "special": true
559
  },
560
  {
561
  "id": 32057,
 
562
  "content": "<extra_id_42>",
563
  "single_word": false,
564
  "lstrip": false,
565
  "rstrip": false,
566
- "normalized": false,
567
- "special": true
568
  },
569
  {
570
  "id": 32058,
 
571
  "content": "<extra_id_41>",
572
  "single_word": false,
573
  "lstrip": false,
574
  "rstrip": false,
575
- "normalized": false,
576
- "special": true
577
  },
578
  {
579
  "id": 32059,
 
580
  "content": "<extra_id_40>",
581
  "single_word": false,
582
  "lstrip": false,
583
  "rstrip": false,
584
- "normalized": false,
585
- "special": true
586
  },
587
  {
588
  "id": 32060,
 
589
  "content": "<extra_id_39>",
590
  "single_word": false,
591
  "lstrip": false,
592
  "rstrip": false,
593
- "normalized": false,
594
- "special": true
595
  },
596
  {
597
  "id": 32061,
 
598
  "content": "<extra_id_38>",
599
  "single_word": false,
600
  "lstrip": false,
601
  "rstrip": false,
602
- "normalized": false,
603
- "special": true
604
  },
605
  {
606
  "id": 32062,
 
607
  "content": "<extra_id_37>",
608
  "single_word": false,
609
  "lstrip": false,
610
  "rstrip": false,
611
- "normalized": false,
612
- "special": true
613
  },
614
  {
615
  "id": 32063,
 
616
  "content": "<extra_id_36>",
617
  "single_word": false,
618
  "lstrip": false,
619
  "rstrip": false,
620
- "normalized": false,
621
- "special": true
622
  },
623
  {
624
  "id": 32064,
 
625
  "content": "<extra_id_35>",
626
  "single_word": false,
627
  "lstrip": false,
628
  "rstrip": false,
629
- "normalized": false,
630
- "special": true
631
  },
632
  {
633
  "id": 32065,
 
634
  "content": "<extra_id_34>",
635
  "single_word": false,
636
  "lstrip": false,
637
  "rstrip": false,
638
- "normalized": false,
639
- "special": true
640
  },
641
  {
642
  "id": 32066,
 
643
  "content": "<extra_id_33>",
644
  "single_word": false,
645
  "lstrip": false,
646
  "rstrip": false,
647
- "normalized": false,
648
- "special": true
649
  },
650
  {
651
  "id": 32067,
 
652
  "content": "<extra_id_32>",
653
  "single_word": false,
654
  "lstrip": false,
655
  "rstrip": false,
656
- "normalized": false,
657
- "special": true
658
  },
659
  {
660
  "id": 32068,
 
661
  "content": "<extra_id_31>",
662
  "single_word": false,
663
  "lstrip": false,
664
  "rstrip": false,
665
- "normalized": false,
666
- "special": true
667
  },
668
  {
669
  "id": 32069,
 
670
  "content": "<extra_id_30>",
671
  "single_word": false,
672
  "lstrip": false,
673
  "rstrip": false,
674
- "normalized": false,
675
- "special": true
676
  },
677
  {
678
  "id": 32070,
 
679
  "content": "<extra_id_29>",
680
  "single_word": false,
681
  "lstrip": false,
682
  "rstrip": false,
683
- "normalized": false,
684
- "special": true
685
  },
686
  {
687
  "id": 32071,
 
688
  "content": "<extra_id_28>",
689
  "single_word": false,
690
  "lstrip": false,
691
  "rstrip": false,
692
- "normalized": false,
693
- "special": true
694
  },
695
  {
696
  "id": 32072,
 
697
  "content": "<extra_id_27>",
698
  "single_word": false,
699
  "lstrip": false,
700
  "rstrip": false,
701
- "normalized": false,
702
- "special": true
703
  },
704
  {
705
  "id": 32073,
 
706
  "content": "<extra_id_26>",
707
  "single_word": false,
708
  "lstrip": false,
709
  "rstrip": false,
710
- "normalized": false,
711
- "special": true
712
  },
713
  {
714
  "id": 32074,
 
715
  "content": "<extra_id_25>",
716
  "single_word": false,
717
  "lstrip": false,
718
  "rstrip": false,
719
- "normalized": false,
720
- "special": true
721
  },
722
  {
723
  "id": 32075,
 
724
  "content": "<extra_id_24>",
725
  "single_word": false,
726
  "lstrip": false,
727
  "rstrip": false,
728
- "normalized": false,
729
- "special": true
730
  },
731
  {
732
  "id": 32076,
 
733
  "content": "<extra_id_23>",
734
  "single_word": false,
735
  "lstrip": false,
736
  "rstrip": false,
737
- "normalized": false,
738
- "special": true
739
  },
740
  {
741
  "id": 32077,
 
742
  "content": "<extra_id_22>",
743
  "single_word": false,
744
  "lstrip": false,
745
  "rstrip": false,
746
- "normalized": false,
747
- "special": true
748
  },
749
  {
750
  "id": 32078,
 
751
  "content": "<extra_id_21>",
752
  "single_word": false,
753
  "lstrip": false,
754
  "rstrip": false,
755
- "normalized": false,
756
- "special": true
757
  },
758
  {
759
  "id": 32079,
 
760
  "content": "<extra_id_20>",
761
  "single_word": false,
762
  "lstrip": false,
763
  "rstrip": false,
764
- "normalized": false,
765
- "special": true
766
  },
767
  {
768
  "id": 32080,
 
769
  "content": "<extra_id_19>",
770
  "single_word": false,
771
  "lstrip": false,
772
  "rstrip": false,
773
- "normalized": false,
774
- "special": true
775
  },
776
  {
777
  "id": 32081,
 
778
  "content": "<extra_id_18>",
779
  "single_word": false,
780
  "lstrip": false,
781
  "rstrip": false,
782
- "normalized": false,
783
- "special": true
784
  },
785
  {
786
  "id": 32082,
 
787
  "content": "<extra_id_17>",
788
  "single_word": false,
789
  "lstrip": false,
790
  "rstrip": false,
791
- "normalized": false,
792
- "special": true
793
  },
794
  {
795
  "id": 32083,
 
796
  "content": "<extra_id_16>",
797
  "single_word": false,
798
  "lstrip": false,
799
  "rstrip": false,
800
- "normalized": false,
801
- "special": true
802
  },
803
  {
804
  "id": 32084,
 
805
  "content": "<extra_id_15>",
806
  "single_word": false,
807
  "lstrip": false,
808
  "rstrip": false,
809
- "normalized": false,
810
- "special": true
811
  },
812
  {
813
  "id": 32085,
 
814
  "content": "<extra_id_14>",
815
  "single_word": false,
816
  "lstrip": false,
817
  "rstrip": false,
818
- "normalized": false,
819
- "special": true
820
  },
821
  {
822
  "id": 32086,
 
823
  "content": "<extra_id_13>",
824
  "single_word": false,
825
  "lstrip": false,
826
  "rstrip": false,
827
- "normalized": false,
828
- "special": true
829
  },
830
  {
831
  "id": 32087,
 
832
  "content": "<extra_id_12>",
833
  "single_word": false,
834
  "lstrip": false,
835
  "rstrip": false,
836
- "normalized": false,
837
- "special": true
838
  },
839
  {
840
  "id": 32088,
 
841
  "content": "<extra_id_11>",
842
  "single_word": false,
843
  "lstrip": false,
844
  "rstrip": false,
845
- "normalized": false,
846
- "special": true
847
  },
848
  {
849
  "id": 32089,
 
850
  "content": "<extra_id_10>",
851
  "single_word": false,
852
  "lstrip": false,
853
  "rstrip": false,
854
- "normalized": false,
855
- "special": true
856
  },
857
  {
858
  "id": 32090,
 
859
  "content": "<extra_id_9>",
860
  "single_word": false,
861
  "lstrip": false,
862
  "rstrip": false,
863
- "normalized": false,
864
- "special": true
865
  },
866
  {
867
  "id": 32091,
 
868
  "content": "<extra_id_8>",
869
  "single_word": false,
870
  "lstrip": false,
871
  "rstrip": false,
872
- "normalized": false,
873
- "special": true
874
  },
875
  {
876
  "id": 32092,
 
877
  "content": "<extra_id_7>",
878
  "single_word": false,
879
  "lstrip": false,
880
  "rstrip": false,
881
- "normalized": false,
882
- "special": true
883
  },
884
  {
885
  "id": 32093,
 
886
  "content": "<extra_id_6>",
887
  "single_word": false,
888
  "lstrip": false,
889
  "rstrip": false,
890
- "normalized": false,
891
- "special": true
892
  },
893
  {
894
  "id": 32094,
 
895
  "content": "<extra_id_5>",
896
  "single_word": false,
897
  "lstrip": false,
898
  "rstrip": false,
899
- "normalized": false,
900
- "special": true
901
  },
902
  {
903
  "id": 32095,
 
904
  "content": "<extra_id_4>",
905
  "single_word": false,
906
  "lstrip": false,
907
  "rstrip": false,
908
- "normalized": false,
909
- "special": true
910
  },
911
  {
912
  "id": 32096,
 
913
  "content": "<extra_id_3>",
914
  "single_word": false,
915
  "lstrip": false,
916
  "rstrip": false,
917
- "normalized": false,
918
- "special": true
919
  },
920
  {
921
  "id": 32097,
 
922
  "content": "<extra_id_2>",
923
  "single_word": false,
924
  "lstrip": false,
925
  "rstrip": false,
926
- "normalized": false,
927
- "special": true
928
  },
929
  {
930
  "id": 32098,
 
931
  "content": "<extra_id_1>",
932
  "single_word": false,
933
  "lstrip": false,
934
  "rstrip": false,
935
- "normalized": false,
936
- "special": true
937
  },
938
  {
939
  "id": 32099,
 
940
  "content": "<extra_id_0>",
941
  "single_word": false,
942
  "lstrip": false,
943
  "rstrip": false,
944
- "normalized": false,
945
- "special": true
946
  }
947
  ],
948
  "normalizer": {
 
19
  "added_tokens": [
20
  {
21
  "id": 0,
22
+ "special": true,
23
  "content": "<pad>",
24
  "single_word": false,
25
  "lstrip": false,
26
  "rstrip": false,
27
+ "normalized": false
 
28
  },
29
  {
30
  "id": 1,
31
+ "special": true,
32
  "content": "</s>",
33
  "single_word": false,
34
  "lstrip": false,
35
  "rstrip": false,
36
+ "normalized": false
 
37
  },
38
  {
39
  "id": 2,
40
+ "special": true,
41
  "content": "<unk>",
42
  "single_word": false,
43
  "lstrip": false,
44
  "rstrip": false,
45
+ "normalized": false
 
46
  },
47
  {
48
  "id": 32000,
49
+ "special": true,
50
  "content": "<extra_id_99>",
51
  "single_word": false,
52
  "lstrip": false,
53
  "rstrip": false,
54
+ "normalized": false
 
55
  },
56
  {
57
  "id": 32001,
58
+ "special": true,
59
  "content": "<extra_id_98>",
60
  "single_word": false,
61
  "lstrip": false,
62
  "rstrip": false,
63
+ "normalized": false
 
64
  },
65
  {
66
  "id": 32002,
67
+ "special": true,
68
  "content": "<extra_id_97>",
69
  "single_word": false,
70
  "lstrip": false,
71
  "rstrip": false,
72
+ "normalized": false
 
73
  },
74
  {
75
  "id": 32003,
76
+ "special": true,
77
  "content": "<extra_id_96>",
78
  "single_word": false,
79
  "lstrip": false,
80
  "rstrip": false,
81
+ "normalized": false
 
82
  },
83
  {
84
  "id": 32004,
85
+ "special": true,
86
  "content": "<extra_id_95>",
87
  "single_word": false,
88
  "lstrip": false,
89
  "rstrip": false,
90
+ "normalized": false
 
91
  },
92
  {
93
  "id": 32005,
94
+ "special": true,
95
  "content": "<extra_id_94>",
96
  "single_word": false,
97
  "lstrip": false,
98
  "rstrip": false,
99
+ "normalized": false
 
100
  },
101
  {
102
  "id": 32006,
103
+ "special": true,
104
  "content": "<extra_id_93>",
105
  "single_word": false,
106
  "lstrip": false,
107
  "rstrip": false,
108
+ "normalized": false
 
109
  },
110
  {
111
  "id": 32007,
112
+ "special": true,
113
  "content": "<extra_id_92>",
114
  "single_word": false,
115
  "lstrip": false,
116
  "rstrip": false,
117
+ "normalized": false
 
118
  },
119
  {
120
  "id": 32008,
121
+ "special": true,
122
  "content": "<extra_id_91>",
123
  "single_word": false,
124
  "lstrip": false,
125
  "rstrip": false,
126
+ "normalized": false
 
127
  },
128
  {
129
  "id": 32009,
130
+ "special": true,
131
  "content": "<extra_id_90>",
132
  "single_word": false,
133
  "lstrip": false,
134
  "rstrip": false,
135
+ "normalized": false
 
136
  },
137
  {
138
  "id": 32010,
139
+ "special": true,
140
  "content": "<extra_id_89>",
141
  "single_word": false,
142
  "lstrip": false,
143
  "rstrip": false,
144
+ "normalized": false
 
145
  },
146
  {
147
  "id": 32011,
148
+ "special": true,
149
  "content": "<extra_id_88>",
150
  "single_word": false,
151
  "lstrip": false,
152
  "rstrip": false,
153
+ "normalized": false
 
154
  },
155
  {
156
  "id": 32012,
157
+ "special": true,
158
  "content": "<extra_id_87>",
159
  "single_word": false,
160
  "lstrip": false,
161
  "rstrip": false,
162
+ "normalized": false
 
163
  },
164
  {
165
  "id": 32013,
166
+ "special": true,
167
  "content": "<extra_id_86>",
168
  "single_word": false,
169
  "lstrip": false,
170
  "rstrip": false,
171
+ "normalized": false
 
172
  },
173
  {
174
  "id": 32014,
175
+ "special": true,
176
  "content": "<extra_id_85>",
177
  "single_word": false,
178
  "lstrip": false,
179
  "rstrip": false,
180
+ "normalized": false
 
181
  },
182
  {
183
  "id": 32015,
184
+ "special": true,
185
  "content": "<extra_id_84>",
186
  "single_word": false,
187
  "lstrip": false,
188
  "rstrip": false,
189
+ "normalized": false
 
190
  },
191
  {
192
  "id": 32016,
193
+ "special": true,
194
  "content": "<extra_id_83>",
195
  "single_word": false,
196
  "lstrip": false,
197
  "rstrip": false,
198
+ "normalized": false
 
199
  },
200
  {
201
  "id": 32017,
202
+ "special": true,
203
  "content": "<extra_id_82>",
204
  "single_word": false,
205
  "lstrip": false,
206
  "rstrip": false,
207
+ "normalized": false
 
208
  },
209
  {
210
  "id": 32018,
211
+ "special": true,
212
  "content": "<extra_id_81>",
213
  "single_word": false,
214
  "lstrip": false,
215
  "rstrip": false,
216
+ "normalized": false
 
217
  },
218
  {
219
  "id": 32019,
220
+ "special": true,
221
  "content": "<extra_id_80>",
222
  "single_word": false,
223
  "lstrip": false,
224
  "rstrip": false,
225
+ "normalized": false
 
226
  },
227
  {
228
  "id": 32020,
229
+ "special": true,
230
  "content": "<extra_id_79>",
231
  "single_word": false,
232
  "lstrip": false,
233
  "rstrip": false,
234
+ "normalized": false
 
235
  },
236
  {
237
  "id": 32021,
238
+ "special": true,
239
  "content": "<extra_id_78>",
240
  "single_word": false,
241
  "lstrip": false,
242
  "rstrip": false,
243
+ "normalized": false
 
244
  },
245
  {
246
  "id": 32022,
247
+ "special": true,
248
  "content": "<extra_id_77>",
249
  "single_word": false,
250
  "lstrip": false,
251
  "rstrip": false,
252
+ "normalized": false
 
253
  },
254
  {
255
  "id": 32023,
256
+ "special": true,
257
  "content": "<extra_id_76>",
258
  "single_word": false,
259
  "lstrip": false,
260
  "rstrip": false,
261
+ "normalized": false
 
262
  },
263
  {
264
  "id": 32024,
265
+ "special": true,
266
  "content": "<extra_id_75>",
267
  "single_word": false,
268
  "lstrip": false,
269
  "rstrip": false,
270
+ "normalized": false
 
271
  },
272
  {
273
  "id": 32025,
274
+ "special": true,
275
  "content": "<extra_id_74>",
276
  "single_word": false,
277
  "lstrip": false,
278
  "rstrip": false,
279
+ "normalized": false
 
280
  },
281
  {
282
  "id": 32026,
283
+ "special": true,
284
  "content": "<extra_id_73>",
285
  "single_word": false,
286
  "lstrip": false,
287
  "rstrip": false,
288
+ "normalized": false
 
289
  },
290
  {
291
  "id": 32027,
292
+ "special": true,
293
  "content": "<extra_id_72>",
294
  "single_word": false,
295
  "lstrip": false,
296
  "rstrip": false,
297
+ "normalized": false
 
298
  },
299
  {
300
  "id": 32028,
301
+ "special": true,
302
  "content": "<extra_id_71>",
303
  "single_word": false,
304
  "lstrip": false,
305
  "rstrip": false,
306
+ "normalized": false
 
307
  },
308
  {
309
  "id": 32029,
310
+ "special": true,
311
  "content": "<extra_id_70>",
312
  "single_word": false,
313
  "lstrip": false,
314
  "rstrip": false,
315
+ "normalized": false
 
316
  },
317
  {
318
  "id": 32030,
319
+ "special": true,
320
  "content": "<extra_id_69>",
321
  "single_word": false,
322
  "lstrip": false,
323
  "rstrip": false,
324
+ "normalized": false
 
325
  },
326
  {
327
  "id": 32031,
328
+ "special": true,
329
  "content": "<extra_id_68>",
330
  "single_word": false,
331
  "lstrip": false,
332
  "rstrip": false,
333
+ "normalized": false
 
334
  },
335
  {
336
  "id": 32032,
337
+ "special": true,
338
  "content": "<extra_id_67>",
339
  "single_word": false,
340
  "lstrip": false,
341
  "rstrip": false,
342
+ "normalized": false
 
343
  },
344
  {
345
  "id": 32033,
346
+ "special": true,
347
  "content": "<extra_id_66>",
348
  "single_word": false,
349
  "lstrip": false,
350
  "rstrip": false,
351
+ "normalized": false
 
352
  },
353
  {
354
  "id": 32034,
355
+ "special": true,
356
  "content": "<extra_id_65>",
357
  "single_word": false,
358
  "lstrip": false,
359
  "rstrip": false,
360
+ "normalized": false
 
361
  },
362
  {
363
  "id": 32035,
364
+ "special": true,
365
  "content": "<extra_id_64>",
366
  "single_word": false,
367
  "lstrip": false,
368
  "rstrip": false,
369
+ "normalized": false
 
370
  },
371
  {
372
  "id": 32036,
373
+ "special": true,
374
  "content": "<extra_id_63>",
375
  "single_word": false,
376
  "lstrip": false,
377
  "rstrip": false,
378
+ "normalized": false
 
379
  },
380
  {
381
  "id": 32037,
382
+ "special": true,
383
  "content": "<extra_id_62>",
384
  "single_word": false,
385
  "lstrip": false,
386
  "rstrip": false,
387
+ "normalized": false
 
388
  },
389
  {
390
  "id": 32038,
391
+ "special": true,
392
  "content": "<extra_id_61>",
393
  "single_word": false,
394
  "lstrip": false,
395
  "rstrip": false,
396
+ "normalized": false
 
397
  },
398
  {
399
  "id": 32039,
400
+ "special": true,
401
  "content": "<extra_id_60>",
402
  "single_word": false,
403
  "lstrip": false,
404
  "rstrip": false,
405
+ "normalized": false
 
406
  },
407
  {
408
  "id": 32040,
409
+ "special": true,
410
  "content": "<extra_id_59>",
411
  "single_word": false,
412
  "lstrip": false,
413
  "rstrip": false,
414
+ "normalized": false
 
415
  },
416
  {
417
  "id": 32041,
418
+ "special": true,
419
  "content": "<extra_id_58>",
420
  "single_word": false,
421
  "lstrip": false,
422
  "rstrip": false,
423
+ "normalized": false
 
424
  },
425
  {
426
  "id": 32042,
427
+ "special": true,
428
  "content": "<extra_id_57>",
429
  "single_word": false,
430
  "lstrip": false,
431
  "rstrip": false,
432
+ "normalized": false
 
433
  },
434
  {
435
  "id": 32043,
436
+ "special": true,
437
  "content": "<extra_id_56>",
438
  "single_word": false,
439
  "lstrip": false,
440
  "rstrip": false,
441
+ "normalized": false
 
442
  },
443
  {
444
  "id": 32044,
445
+ "special": true,
446
  "content": "<extra_id_55>",
447
  "single_word": false,
448
  "lstrip": false,
449
  "rstrip": false,
450
+ "normalized": false
 
451
  },
452
  {
453
  "id": 32045,
454
+ "special": true,
455
  "content": "<extra_id_54>",
456
  "single_word": false,
457
  "lstrip": false,
458
  "rstrip": false,
459
+ "normalized": false
 
460
  },
461
  {
462
  "id": 32046,
463
+ "special": true,
464
  "content": "<extra_id_53>",
465
  "single_word": false,
466
  "lstrip": false,
467
  "rstrip": false,
468
+ "normalized": false
 
469
  },
470
  {
471
  "id": 32047,
472
+ "special": true,
473
  "content": "<extra_id_52>",
474
  "single_word": false,
475
  "lstrip": false,
476
  "rstrip": false,
477
+ "normalized": false
 
478
  },
479
  {
480
  "id": 32048,
481
+ "special": true,
482
  "content": "<extra_id_51>",
483
  "single_word": false,
484
  "lstrip": false,
485
  "rstrip": false,
486
+ "normalized": false
 
487
  },
488
  {
489
  "id": 32049,
490
+ "special": true,
491
  "content": "<extra_id_50>",
492
  "single_word": false,
493
  "lstrip": false,
494
  "rstrip": false,
495
+ "normalized": false
 
496
  },
497
  {
498
  "id": 32050,
499
+ "special": true,
500
  "content": "<extra_id_49>",
501
  "single_word": false,
502
  "lstrip": false,
503
  "rstrip": false,
504
+ "normalized": false
 
505
  },
506
  {
507
  "id": 32051,
508
+ "special": true,
509
  "content": "<extra_id_48>",
510
  "single_word": false,
511
  "lstrip": false,
512
  "rstrip": false,
513
+ "normalized": false
 
514
  },
515
  {
516
  "id": 32052,
517
+ "special": true,
518
  "content": "<extra_id_47>",
519
  "single_word": false,
520
  "lstrip": false,
521
  "rstrip": false,
522
+ "normalized": false
 
523
  },
524
  {
525
  "id": 32053,
526
+ "special": true,
527
  "content": "<extra_id_46>",
528
  "single_word": false,
529
  "lstrip": false,
530
  "rstrip": false,
531
+ "normalized": false
 
532
  },
533
  {
534
  "id": 32054,
535
+ "special": true,
536
  "content": "<extra_id_45>",
537
  "single_word": false,
538
  "lstrip": false,
539
  "rstrip": false,
540
+ "normalized": false
 
541
  },
542
  {
543
  "id": 32055,
544
+ "special": true,
545
  "content": "<extra_id_44>",
546
  "single_word": false,
547
  "lstrip": false,
548
  "rstrip": false,
549
+ "normalized": false
 
550
  },
551
  {
552
  "id": 32056,
553
+ "special": true,
554
  "content": "<extra_id_43>",
555
  "single_word": false,
556
  "lstrip": false,
557
  "rstrip": false,
558
+ "normalized": false
 
559
  },
560
  {
561
  "id": 32057,
562
+ "special": true,
563
  "content": "<extra_id_42>",
564
  "single_word": false,
565
  "lstrip": false,
566
  "rstrip": false,
567
+ "normalized": false
 
568
  },
569
  {
570
  "id": 32058,
571
+ "special": true,
572
  "content": "<extra_id_41>",
573
  "single_word": false,
574
  "lstrip": false,
575
  "rstrip": false,
576
+ "normalized": false
 
577
  },
578
  {
579
  "id": 32059,
580
+ "special": true,
581
  "content": "<extra_id_40>",
582
  "single_word": false,
583
  "lstrip": false,
584
  "rstrip": false,
585
+ "normalized": false
 
586
  },
587
  {
588
  "id": 32060,
589
+ "special": true,
590
  "content": "<extra_id_39>",
591
  "single_word": false,
592
  "lstrip": false,
593
  "rstrip": false,
594
+ "normalized": false
 
595
  },
596
  {
597
  "id": 32061,
598
+ "special": true,
599
  "content": "<extra_id_38>",
600
  "single_word": false,
601
  "lstrip": false,
602
  "rstrip": false,
603
+ "normalized": false
 
604
  },
605
  {
606
  "id": 32062,
607
+ "special": true,
608
  "content": "<extra_id_37>",
609
  "single_word": false,
610
  "lstrip": false,
611
  "rstrip": false,
612
+ "normalized": false
 
613
  },
614
  {
615
  "id": 32063,
616
+ "special": true,
617
  "content": "<extra_id_36>",
618
  "single_word": false,
619
  "lstrip": false,
620
  "rstrip": false,
621
+ "normalized": false
 
622
  },
623
  {
624
  "id": 32064,
625
+ "special": true,
626
  "content": "<extra_id_35>",
627
  "single_word": false,
628
  "lstrip": false,
629
  "rstrip": false,
630
+ "normalized": false
 
631
  },
632
  {
633
  "id": 32065,
634
+ "special": true,
635
  "content": "<extra_id_34>",
636
  "single_word": false,
637
  "lstrip": false,
638
  "rstrip": false,
639
+ "normalized": false
 
640
  },
641
  {
642
  "id": 32066,
643
+ "special": true,
644
  "content": "<extra_id_33>",
645
  "single_word": false,
646
  "lstrip": false,
647
  "rstrip": false,
648
+ "normalized": false
 
649
  },
650
  {
651
  "id": 32067,
652
+ "special": true,
653
  "content": "<extra_id_32>",
654
  "single_word": false,
655
  "lstrip": false,
656
  "rstrip": false,
657
+ "normalized": false
 
658
  },
659
  {
660
  "id": 32068,
661
+ "special": true,
662
  "content": "<extra_id_31>",
663
  "single_word": false,
664
  "lstrip": false,
665
  "rstrip": false,
666
+ "normalized": false
 
667
  },
668
  {
669
  "id": 32069,
670
+ "special": true,
671
  "content": "<extra_id_30>",
672
  "single_word": false,
673
  "lstrip": false,
674
  "rstrip": false,
675
+ "normalized": false
 
676
  },
677
  {
678
  "id": 32070,
679
+ "special": true,
680
  "content": "<extra_id_29>",
681
  "single_word": false,
682
  "lstrip": false,
683
  "rstrip": false,
684
+ "normalized": false
 
685
  },
686
  {
687
  "id": 32071,
688
+ "special": true,
689
  "content": "<extra_id_28>",
690
  "single_word": false,
691
  "lstrip": false,
692
  "rstrip": false,
693
+ "normalized": false
 
694
  },
695
  {
696
  "id": 32072,
697
+ "special": true,
698
  "content": "<extra_id_27>",
699
  "single_word": false,
700
  "lstrip": false,
701
  "rstrip": false,
702
+ "normalized": false
 
703
  },
704
  {
705
  "id": 32073,
706
+ "special": true,
707
  "content": "<extra_id_26>",
708
  "single_word": false,
709
  "lstrip": false,
710
  "rstrip": false,
711
+ "normalized": false
 
712
  },
713
  {
714
  "id": 32074,
715
+ "special": true,
716
  "content": "<extra_id_25>",
717
  "single_word": false,
718
  "lstrip": false,
719
  "rstrip": false,
720
+ "normalized": false
 
721
  },
722
  {
723
  "id": 32075,
724
+ "special": true,
725
  "content": "<extra_id_24>",
726
  "single_word": false,
727
  "lstrip": false,
728
  "rstrip": false,
729
+ "normalized": false
 
730
  },
731
  {
732
  "id": 32076,
733
+ "special": true,
734
  "content": "<extra_id_23>",
735
  "single_word": false,
736
  "lstrip": false,
737
  "rstrip": false,
738
+ "normalized": false
 
739
  },
740
  {
741
  "id": 32077,
742
+ "special": true,
743
  "content": "<extra_id_22>",
744
  "single_word": false,
745
  "lstrip": false,
746
  "rstrip": false,
747
+ "normalized": false
 
748
  },
749
  {
750
  "id": 32078,
751
+ "special": true,
752
  "content": "<extra_id_21>",
753
  "single_word": false,
754
  "lstrip": false,
755
  "rstrip": false,
756
+ "normalized": false
 
757
  },
758
  {
759
  "id": 32079,
760
+ "special": true,
761
  "content": "<extra_id_20>",
762
  "single_word": false,
763
  "lstrip": false,
764
  "rstrip": false,
765
+ "normalized": false
 
766
  },
767
  {
768
  "id": 32080,
769
+ "special": true,
770
  "content": "<extra_id_19>",
771
  "single_word": false,
772
  "lstrip": false,
773
  "rstrip": false,
774
+ "normalized": false
 
775
  },
776
  {
777
  "id": 32081,
778
+ "special": true,
779
  "content": "<extra_id_18>",
780
  "single_word": false,
781
  "lstrip": false,
782
  "rstrip": false,
783
+ "normalized": false
 
784
  },
785
  {
786
  "id": 32082,
787
+ "special": true,
788
  "content": "<extra_id_17>",
789
  "single_word": false,
790
  "lstrip": false,
791
  "rstrip": false,
792
+ "normalized": false
 
793
  },
794
  {
795
  "id": 32083,
796
+ "special": true,
797
  "content": "<extra_id_16>",
798
  "single_word": false,
799
  "lstrip": false,
800
  "rstrip": false,
801
+ "normalized": false
 
802
  },
803
  {
804
  "id": 32084,
805
+ "special": true,
806
  "content": "<extra_id_15>",
807
  "single_word": false,
808
  "lstrip": false,
809
  "rstrip": false,
810
+ "normalized": false
 
811
  },
812
  {
813
  "id": 32085,
814
+ "special": true,
815
  "content": "<extra_id_14>",
816
  "single_word": false,
817
  "lstrip": false,
818
  "rstrip": false,
819
+ "normalized": false
 
820
  },
821
  {
822
  "id": 32086,
823
+ "special": true,
824
  "content": "<extra_id_13>",
825
  "single_word": false,
826
  "lstrip": false,
827
  "rstrip": false,
828
+ "normalized": false
 
829
  },
830
  {
831
  "id": 32087,
832
+ "special": true,
833
  "content": "<extra_id_12>",
834
  "single_word": false,
835
  "lstrip": false,
836
  "rstrip": false,
837
+ "normalized": false
 
838
  },
839
  {
840
  "id": 32088,
841
+ "special": true,
842
  "content": "<extra_id_11>",
843
  "single_word": false,
844
  "lstrip": false,
845
  "rstrip": false,
846
+ "normalized": false
 
847
  },
848
  {
849
  "id": 32089,
850
+ "special": true,
851
  "content": "<extra_id_10>",
852
  "single_word": false,
853
  "lstrip": false,
854
  "rstrip": false,
855
+ "normalized": false
 
856
  },
857
  {
858
  "id": 32090,
859
+ "special": true,
860
  "content": "<extra_id_9>",
861
  "single_word": false,
862
  "lstrip": false,
863
  "rstrip": false,
864
+ "normalized": false
 
865
  },
866
  {
867
  "id": 32091,
868
+ "special": true,
869
  "content": "<extra_id_8>",
870
  "single_word": false,
871
  "lstrip": false,
872
  "rstrip": false,
873
+ "normalized": false
 
874
  },
875
  {
876
  "id": 32092,
877
+ "special": true,
878
  "content": "<extra_id_7>",
879
  "single_word": false,
880
  "lstrip": false,
881
  "rstrip": false,
882
+ "normalized": false
 
883
  },
884
  {
885
  "id": 32093,
886
+ "special": true,
887
  "content": "<extra_id_6>",
888
  "single_word": false,
889
  "lstrip": false,
890
  "rstrip": false,
891
+ "normalized": false
 
892
  },
893
  {
894
  "id": 32094,
895
+ "special": true,
896
  "content": "<extra_id_5>",
897
  "single_word": false,
898
  "lstrip": false,
899
  "rstrip": false,
900
+ "normalized": false
 
901
  },
902
  {
903
  "id": 32095,
904
+ "special": true,
905
  "content": "<extra_id_4>",
906
  "single_word": false,
907
  "lstrip": false,
908
  "rstrip": false,
909
+ "normalized": false
 
910
  },
911
  {
912
  "id": 32096,
913
+ "special": true,
914
  "content": "<extra_id_3>",
915
  "single_word": false,
916
  "lstrip": false,
917
  "rstrip": false,
918
+ "normalized": false
 
919
  },
920
  {
921
  "id": 32097,
922
+ "special": true,
923
  "content": "<extra_id_2>",
924
  "single_word": false,
925
  "lstrip": false,
926
  "rstrip": false,
927
+ "normalized": false
 
928
  },
929
  {
930
  "id": 32098,
931
+ "special": true,
932
  "content": "<extra_id_1>",
933
  "single_word": false,
934
  "lstrip": false,
935
  "rstrip": false,
936
+ "normalized": false
 
937
  },
938
  {
939
  "id": 32099,
940
+ "special": true,
941
  "content": "<extra_id_0>",
942
  "single_word": false,
943
  "lstrip": false,
944
  "rstrip": false,
945
+ "normalized": false
 
946
  }
947
  ],
948
  "normalizer": {