lightonai
/

gpt2-tokenizer-0923

Model card Files Files and versions Community

ohallstrom commited on Sep 14, 2023

Commit

0ae7ba1

•

1 Parent(s): f9430da

Add tokenizer files

Browse files

Files changed (3) hide show

special_tokens_map.json +371 -0
tokenizer.json +0 -0
tokenizer_config.json +9 -0

special_tokens_map.json ADDED Viewed

	@@ -0,0 +1,371 @@

+{
+  "additional_special_tokens": [
+    ">>TITLE<<",
+    ">>ABSTRACT<<",
+    ">>INTRODUCTION<<",
+    ">>SUMMARY<<",
+    ">>COMMENT<<",
+    ">>ANSWER<<",
+    ">>QUESTION<<",
+    ">>DOMAIN<<",
+    ">>PREFIX<<",
+    ">>SUFFIX<<",
+    ">>MIDDLE<<",
+    ">>[S]<<",
+    ">>[X]<<",
+    ">>[R]<<",
+    ">>S0<<",
+    ">>S1<<",
+    ">>S2<<",
+    ">>S3<<",
+    ">>S4<<",
+    ">>S5<<",
+    ">>S6<<",
+    ">>S7<<",
+    ">>S8<<",
+    ">>S9<<",
+    ">>S10<<",
+    ">>S11<<",
+    ">>S12<<",
+    ">>S13<<",
+    ">>S14<<",
+    ">>S15<<",
+    ">>S16<<",
+    ">>S17<<",
+    ">>S18<<",
+    ">>S19<<",
+    ">>S20<<",
+    ">>S21<<",
+    ">>S22<<",
+    ">>S23<<",
+    ">>S24<<",
+    ">>S25<<",
+    ">>S26<<",
+    ">>S27<<",
+    ">>S28<<",
+    ">>S29<<",
+    ">>S30<<",
+    ">>S31<<",
+    ">>S32<<",
+    ">>S33<<",
+    ">>S34<<",
+    ">>S35<<",
+    ">>S36<<",
+    ">>S37<<",
+    ">>S38<<",
+    ">>S39<<",
+    ">>S40<<",
+    ">>S41<<",
+    ">>S42<<",
+    ">>S43<<",
+    ">>S44<<",
+    ">>S45<<",
+    ">>S46<<",
+    ">>S47<<",
+    ">>S48<<",
+    ">>S49<<",
+    ">>S50<<",
+    ">>S51<<",
+    ">>S52<<",
+    ">>S53<<",
+    ">>S54<<",
+    ">>S55<<",
+    ">>S56<<",
+    ">>S57<<",
+    ">>S58<<",
+    ">>S59<<",
+    ">>S60<<",
+    ">>S61<<",
+    ">>S62<<",
+    ">>S63<<",
+    ">>S64<<",
+    ">>S65<<",
+    ">>S66<<",
+    ">>S67<<",
+    ">>S68<<",
+    ">>S69<<",
+    ">>S70<<",
+    ">>S71<<",
+    ">>S72<<",
+    ">>S73<<",
+    ">>S74<<",
+    ">>S75<<",
+    ">>S76<<",
+    ">>S77<<",
+    ">>S78<<",
+    ">>S79<<",
+    ">>S80<<",
+    ">>S81<<",
+    ">>S82<<",
+    ">>S83<<",
+    ">>S84<<",
+    ">>S85<<",
+    ">>S86<<",
+    ">>S87<<",
+    ">>S88<<",
+    ">>S89<<",
+    ">>S90<<",
+    ">>S91<<",
+    ">>S92<<",
+    ">>S93<<",
+    ">>S94<<",
+    ">>S95<<",
+    ">>S96<<",
+    ">>S97<<",
+    ">>S98<<",
+    ">>S99<<",
+    ">>S100<<",
+    ">>S101<<",
+    ">>S102<<",
+    ">>S103<<",
+    ">>S104<<",
+    ">>S105<<",
+    ">>S106<<",
+    ">>S107<<",
+    ">>S108<<",
+    ">>S109<<",
+    ">>S110<<",
+    ">>S111<<",
+    ">>S112<<",
+    ">>S113<<",
+    ">>S114<<",
+    ">>S115<<",
+    ">>S116<<",
+    ">>S117<<",
+    ">>S118<<",
+    ">>S119<<",
+    ">>S120<<",
+    ">>S121<<",
+    ">>S122<<",
+    ">>S123<<",
+    ">>S124<<",
+    ">>S125<<",
+    ">>S126<<",
+    ">>S127<<",
+    ">>S128<<",
+    ">>S129<<",
+    ">>S130<<",
+    ">>S131<<",
+    ">>S132<<",
+    ">>S133<<",
+    ">>S134<<",
+    ">>S135<<",
+    ">>S136<<",
+    ">>S137<<",
+    ">>S138<<",
+    ">>S139<<",
+    ">>S140<<",
+    ">>S141<<",
+    ">>S142<<",
+    ">>S143<<",
+    ">>S144<<",
+    ">>S145<<",
+    ">>S146<<",
+    ">>S147<<",
+    ">>S148<<",
+    ">>S149<<",
+    ">>S150<<",
+    ">>S151<<",
+    ">>S152<<",
+    ">>S153<<",
+    ">>S154<<",
+    ">>S155<<",
+    ">>S156<<",
+    ">>S157<<",
+    ">>S158<<",
+    ">>S159<<",
+    ">>S160<<",
+    ">>S161<<",
+    ">>S162<<",
+    ">>S163<<",
+    ">>S164<<",
+    ">>S165<<",
+    ">>S166<<",
+    ">>S167<<",
+    ">>S168<<",
+    ">>S169<<",
+    ">>S170<<",
+    ">>S171<<",
+    ">>S172<<",
+    ">>S173<<",
+    ">>S174<<",
+    ">>S175<<",
+    ">>S176<<",
+    ">>S177<<",
+    ">>S178<<",
+    ">>S179<<",
+    ">>S180<<",
+    ">>S181<<",
+    ">>S182<<",
+    ">>S183<<",
+    ">>S184<<",
+    ">>S185<<",
+    ">>S186<<",
+    ">>S187<<",
+    ">>S188<<",
+    ">>S189<<",
+    ">>S190<<",
+    ">>S191<<",
+    ">>S192<<",
+    ">>S193<<",
+    ">>S194<<",
+    ">>S195<<",
+    ">>S196<<",
+    ">>S197<<",
+    ">>S198<<",
+    ">>S199<<",
+    ">>S200<<",
+    ">>S201<<",
+    ">>S202<<",
+    ">>S203<<",
+    ">>S204<<",
+    ">>S205<<",
+    ">>S206<<",
+    ">>S207<<",
+    ">>S208<<",
+    ">>S209<<",
+    ">>S210<<",
+    ">>S211<<",
+    ">>S212<<",
+    ">>S213<<",
+    ">>S214<<",
+    ">>S215<<",
+    ">>S216<<",
+    ">>S217<<",
+    ">>S218<<",
+    ">>S219<<",
+    ">>S220<<",
+    ">>S221<<",
+    ">>S222<<",
+    ">>S223<<",
+    ">>S224<<",
+    ">>S225<<",
+    ">>S226<<",
+    ">>S227<<",
+    ">>S228<<",
+    ">>S229<<",
+    ">>S230<<",
+    ">>S231<<",
+    ">>S232<<",
+    ">>S233<<",
+    ">>S234<<",
+    ">>S235<<",
+    ">>S236<<",
+    ">>S237<<",
+    ">>S238<<",
+    ">>S239<<",
+    ">>S240<<",
+    ">>S241<<",
+    ">>S242<<",
+    ">>S243<<",
+    ">>S244<<",
+    ">>S245<<",
+    ">>S246<<",
+    ">>S247<<",
+    ">>S248<<",
+    ">>S249<<",
+    ">>S250<<",
+    ">>S251<<",
+    ">>S252<<",
+    ">>S253<<",
+    ">>S254<<",
+    ">>S255<<",
+    ">>S256<<",
+    ">>S257<<",
+    ">>S258<<",
+    ">>S259<<",
+    ">>S260<<",
+    ">>S261<<",
+    ">>S262<<",
+    ">>S263<<",
+    ">>S264<<",
+    ">>S265<<",
+    ">>S266<<",
+    ">>S267<<",
+    ">>S268<<",
+    ">>S269<<",
+    ">>S270<<",
+    ">>S271<<",
+    ">>S272<<",
+    ">>S273<<",
+    ">>S274<<",
+    ">>S275<<",
+    ">>S276<<",
+    ">>S277<<",
+    ">>S278<<",
+    ">>S279<<",
+    ">>S280<<",
+    ">>S281<<",
+    ">>S282<<",
+    ">>S283<<",
+    ">>S284<<",
+    ">>S285<<",
+    ">>S286<<",
+    ">>S287<<",
+    ">>S288<<",
+    ">>S289<<",
+    ">>S290<<",
+    ">>S291<<",
+    ">>S292<<",
+    ">>S293<<",
+    ">>S294<<",
+    ">>S295<<",
+    ">>S296<<",
+    ">>S297<<",
+    ">>S298<<",
+    ">>S299<<",
+    ">>S300<<",
+    ">>S301<<",
+    ">>S302<<",
+    ">>S303<<",
+    ">>S304<<",
+    ">>S305<<",
+    ">>S306<<",
+    ">>S307<<",
+    ">>S308<<",
+    ">>S309<<",
+    ">>S310<<",
+    ">>S311<<",
+    ">>S312<<",
+    ">>S313<<",
+    ">>S314<<",
+    ">>S315<<",
+    ">>S316<<",
+    ">>S317<<",
+    ">>S318<<",
+    ">>S319<<",
+    ">>S320<<",
+    ">>S321<<",
+    ">>S322<<",
+    ">>S323<<",
+    ">>S324<<",
+    ">>S325<<",
+    ">>S326<<",
+    ">>S327<<",
+    ">>S328<<",
+    ">>S329<<",
+    ">>S330<<",
+    ">>S331<<",
+    ">>S332<<",
+    ">>S333<<",
+    ">>S334<<",
+    ">>S335<<",
+    ">>S336<<",
+    ">>S337<<",
+    ">>S338<<",
+    ">>S339<<",
+    ">>S340<<",
+    ">>S341<<",
+    ">>S342<<",
+    ">>S343<<",
+    ">>S344<<",
+    ">>S345<<",
+    "<start_system>",
+    "<start_user>",
+    "<start_assistant>",
+    "<end_message>"
+  ],
+  "bos_token": "<|endoftext|>",
+  "eos_token": "<|endoftext|>",
+  "unk_token": "<|endoftext|>"
+}

tokenizer.json ADDED Viewed

The diff for this file is too large to render. See raw diff

tokenizer_config.json ADDED Viewed

	@@ -0,0 +1,9 @@

+{
+  "add_prefix_space": false,
+  "bos_token": "<|endoftext|>",
+  "eos_token": "<|endoftext|>",
+  "model_max_length": 1024,
+  "special_tokens_map_file": null,
+  "tokenizer_class": "GPT2Tokenizer",
+  "unk_token": "<|endoftext|>"
+}