tiny-random-funnel / tokenizer.json
LysandreJik's picture
v3
ec246a6
{"version":"1.0","truncation":null,"padding":null,"added_tokens":[{"id":0,"special":true,"content":"<pad>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false},{"id":1,"special":true,"content":"<s>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false},{"id":2,"special":true,"content":"</s>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false},{"id":3,"special":true,"content":"<unk>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false},{"id":4,"special":true,"content":"<cls>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false},{"id":5,"special":true,"content":"<sep>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false},{"id":6,"special":true,"content":"<mask>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false}],"normalizer":{"type":"BertNormalizer","clean_text":true,"handle_chinese_chars":true,"strip_accents":null,"lowercase":true},"pre_tokenizer":{"type":"BertPreTokenizer"},"post_processor":{"type":"TemplateProcessing","single":[{"SpecialToken":{"id":"<cls>","type_id":2}},{"Sequence":{"id":"A","type_id":0}},{"SpecialToken":{"id":"<sep>","type_id":0}}],"pair":[{"SpecialToken":{"id":"<cls>","type_id":2}},{"Sequence":{"id":"A","type_id":0}},{"SpecialToken":{"id":"<sep>","type_id":0}},{"Sequence":{"id":"B","type_id":1}},{"SpecialToken":{"id":"<sep>","type_id":1}}],"special_tokens":{"<cls>":{"id":"<cls>","ids":[4],"tokens":["<cls>"]},"<sep>":{"id":"<sep>","ids":[5],"tokens":["<sep>"]}}},"decoder":{"type":"WordPiece","prefix":"##","cleanup":true},"model":{"type":"WordPiece","unk_token":"<unk>","continuing_subword_prefix":"##","max_input_chars_per_word":100,"vocab":{"<pad>":0,"<s>":1,"</s>":2,"<unk>":3,"<cls>":4,"<sep>":5,"<mask>":6,"!":7,"\"":8,"#":9,"$":10,"%":11,"&":12,"'":13,"(":14,")":15,"*":16,"+":17,",":18,"-":19,".":20,"/":21,"0":22,"1":23,"2":24,"3":25,"4":26,"5":27,"6":28,"7":29,"8":30,"9":31,":":32,";":33,"<":34,"=":35,">":36,"?":37,"@":38,"[":39,"\\":40,"]":41,"^":42,"_":43,"`":44,"a":45,"b":46,"c":47,"d":48,"e":49,"f":50,"g":51,"h":52,"i":53,"j":54,"k":55,"l":56,"m":57,"n":58,"o":59,"p":60,"q":61,"r":62,"s":63,"t":64,"u":65,"v":66,"w":67,"x":68,"y":69,"z":70,"|":71,"}":72,"~":73,"¡":74,"¢":75,"£":76,"¥":77,"§":78,"°":79,"±":80,"²":81,"³":82,"´":83,"µ":84,"·":85,"º":86,"½":87,"¿":88,"×":89,"ß":90,"æ":91,"ð":92,"ø":93,"þ":94,"đ":95,"ħ":96,"ı":97,"ł":98,"œ":99,"ɐ":100,"ɑ":101,"ɒ":102,"ɔ":103,"ə":104,"ɛ":105,"ɜ":106,"ɡ":107,"ɢ":108,"ɪ":109,"ɫ":110,"ɳ":111,"ɽ":112,"ɾ":113,"ʁ":114,"ʃ":115,"ʊ":116,"ʋ":117,"ʒ":118,"ʔ":119,"ʕ":120,"ʲ":121,"ʻ":122,"ʼ":123,"ʾ":124,"ʿ":125,"ˈ":126,"ˌ":127,"ː":128,"α":129,"β":130,"γ":131,"δ":132,"ε":133,"η":134,"θ":135,"ι":136,"κ":137,"λ":138,"μ":139,"ν":140,"ξ":141,"ο":142,"π":143,"ρ":144,"ς":145,"σ":146,"τ":147,"υ":148,"φ":149,"χ":150,"ψ":151,"ω":152,"а":153,"б":154,"в":155,"г":156,"д":157,"е":158,"ж":159,"з":160,"и":161,"к":162,"л":163,"м":164,"н":165,"о":166,"п":167,"р":168,"с":169,"т":170,"у":171,"х":172,"ц":173,"ш":174,"ъ":175,"ы":176,"ь":177,"ю":178,"я":179,"є":180,"א":181,"ב":182,"ג":183,"ה":184,"ו":185,"ז":186,"ח":187,"י":188,"ל":189,"ם":190,"מ":191,"ן":192,"נ":193,"ס":194,"ף":195,"פ":196,"צ":197,"ר":198,"ש":199,"ת":200,"ء":201,"ا":202,"ب":203,"ة":204,"ت":205,"ث":206,"ج":207,"ح":208,"خ":209,"د":210,"ذ":211,"ر":212,"س":213,"ش":214,"ص":215,"ع":216,"ف":217,"ق":218,"ك":219,"ل":220,"م":221,"ن":222,"ه":223,"و":224,"ي":225,"ܐ":226,"ܕ":227,"ܗ":228,"ܝ":229,"ܠ":230,"ܢ":231,"ܬ":232,"अ":233,"ई":234,"क":235,"ग":236,"ण":237,"त":238,"द":239,"न":240,"प":241,"ब":242,"म":243,"य":244,"र":245,"ल":246,"व":247,"स":248,"ह":249,"ा":250,"ि":251,"আ":252,"ল":253,"হ":254,"া":255,"ਅ":256,"ਲ":257,"ਹ":258,"ਾ":259,"അ":260,"ള":261,"ഹ":262,"ാ":263,"ก":264,"ค":265,"ง":266,"ช":267,"ซ":268,"ญ":269,"ฐ":270,"ณ":271,"ด":272,"ต":273,"น":274,"บ":275,"ป":276,"พ":277,"ภ":278,"ม":279,"ย":280,"ร":281,"ล":282,"ว":283,"ศ":284,"ษ":285,"ส":286,"ห":287,"อ":288,"ฮ":289,"ะ":290,"า":291,"เ":292,"แ":293,"ไ":294,"ა":295,"ბ":296,"გ":297,"დ":298,"ე":299,"ვ":300,"ზ":301,"თ":302,"ი":303,"კ":304,"ლ":305,"მ":306,"ნ":307,"ო":308,"პ":309,"ჟ":310,"რ":311,"ს":312,"ტ":313,"უ":314,"ფ":315,"ქ":316,"ღ":317,"ყ":318,"შ":319,"ჩ":320,"ც":321,"ძ":322,"წ":323,"ჭ":324,"ხ":325,"ჯ":326,"ჰ":327,"ჱ":328,"ჲ":329,"ჳ":330,"ჴ":331,"ჵ":332,"ჶ":333,"ჷ":334,"ჸ":335,"ჹ":336,"ჺ":337,"჻":338,"ᄃ":339,"ᄅ":340,"ᄇ":341,"ᄋ":342,"ᄌ":343,"ᅡ":344,"ᅢ":345,"ᅦ":346,"ᅧ":347,"ᅩ":348,"ᅮ":349,"ᅵ":350,"ᆨ":351,"ᆫ":352,"ᆯ":353,"ᆸ":354,"ᆼ":355,"ᵻ":356,"‐":357,"‑":358,"–":359,"—":360,"―":361,"‘":362,"’":363,"“":364,"”":365,"„":366,"†":367,"‡":368,"•":369,"…":370,"′":371,"″":372,"⁄":373,"₣":374,"₤":375,"€":376,"₹":377,"⅓":378,"⅔":379,"→":380,"−":381,"≡":382,"≤":383,"①":384,"☉":385,"☫":386,"♀":387,"♭":388,"♯":389,"⚳":390,"ⴀ":391,"ⴂ":392,"ⴃ":393,"ⴈ":394,"ⴌ":395,"ⴕ":396,"ⴟ":397,"〈":398,"〉":399,"〜":400,"あ":401,"い":402,"う":403,"お":404,"か":405,"き":406,"く":407,"け":408,"こ":409,"さ":410,"し":411,"す":412,"せ":413,"た":414,"ち":415,"っ":416,"つ":417,"と":418,"な":419,"に":420,"の":421,"は":422,"ひ":423,"ふ":424,"ほ":425,"ま":426,"み":427,"め":428,"も":429,"ゃ":430,"ゆ":431,"ょ":432,"ら":433,"り":434,"る":435,"れ":436,"わ":437,"を":438,"ん":439,"ァ":440,"ア":441,"ィ":442,"イ":443,"ゥ":444,"ウ":445,"ェ":446,"エ":447,"ォ":448,"オ":449,"カ":450,"キ":451,"ク":452,"ケ":453,"コ":454,"サ":455,"シ":456,"ス":457,"セ":458,"タ":459,"チ":460,"ッ":461,"ツ":462,"テ":463,"ト":464,"ナ":465,"ニ":466,"ネ":467,"ノ":468,"ハ":469,"フ":470,"ヘ":471,"マ":472,"ミ":473,"ム":474,"モ":475,"ャ":476,"ュ":477,"ョ":478,"ラ":479,"リ":480,"ル":481,"レ":482,"ロ":483,"ン":484,"・":485,"ー":486,"一":487,"七":488,"下":489,"世":490,"丙":491,"中":492,"主":493,"乃":494,"之":495,"乙":496,"九":497,"二":498,"云":499,"人":500,"今":501,"付":502,"作":503,"侗":504,"依":505,"信":506,"傳":507,"儚":508,"充":509,"光":510,"全":511,"兵":512,"其":513,"具":514,"円":515,"再":516,"出":517,"判":518,"前":519,"剛":520,"劇":521,"劉":522,"動":523,"化":524,"北":525,"华":526,"厂":527,"去":528,"古":529,"可":530,"台":531,"史":532,"同":533,"名":534,"君":535,"吳":536,"周":537,"命":538,"和":539,"咲":540,"善":541,"四":542,"國":543,"園":544,"圣":545,"在":546,"坂":547,"堤":548,"場":549,"塘":550,"夕":551,"大":552,"天":553,"夫":554,"女":555,"妙":556,"姚":557,"子":558,"孟":559,"守":560,"安":561,"宋":562,"完":563,"宗":564,"宝":565,"宫":566,"寝":567,"寺":568,"小":569,"少":570,"尾":571,"山":572,"岳":573,"川":574,"州":575,"巳":576,"市":577,"師":578,"平":579,"广":580,"庆":581,"府":582,"座":583,"廬":584,"建":585,"式":586,"張":587,"彌":588,"彩":589,"彼":590,"後":591,"御":592,"德":593,"思":594,"愛":595,"憑":596,"憶":597,"應":598,"懷":599,"战":600,"戦":601,"扈":602,"技":603,"拉":604,"拳":605,"挑":606,"揺":607,"攻":608,"放":609,"政":610,"散":611,"斯":612,"方":613,"日":614,"旦":615,"旭":616,"昌":617,"明":618,"星":619,"春":620,"晋":621,"景":622,"曦":623,"月":624,"望":625,"未":626,"本":627,"李":628,"村":629,"杜":630,"束":631,"来":632,"林":633,"桜":634,"梶":635,"棘":636,"椎":637,"楊":638,"楚":639,"榮":640,"橘":641,"機":642,"正":643,"殻":644,"殿":645,"母":646,"水":647,"汉":648,"沂":649,"沙":650,"河":651,"泗":652,"波":653,"泣":654,"洪":655,"淹":656,"清":657,"湯":658,"漢":659,"澄":660,"澤":661,"火":662,"灯":663,"灵":664,"灼":665,"焼":666,"熱":667,"物":668,"狐":669,"狸":670,"玄":671,"王":672,"玩":673,"珂":674,"珙":675,"球":676,"理":677,"琦":678,"琪":679,"瓊":680,"生":681,"田":682,"畢":683,"番":684,"瘡":685,"白":686,"皮":687,"真":688,"砲":689,"礮":690,"祈":691,"神":692,"祠":693,"秋":694,"空":695,"立":696,"精":697,"約":698,"絵":699,"織":700,"義":701,"翠":702,"者":703,"耕":704,"肖":705,"胡":706,"膀":707,"臂":708,"興":709,"良":710,"花":711,"芳":712,"芽":713,"若":714,"英":715,"藕":716,"藥":717,"蘄":718,"蘇":719,"行":720,"裁":721,"規":722,"覺":723,"观":724,"解":725,"記":726,"誓":727,"誡":728,"誰":729,"謎":730,"许":731,"谭":732,"豪":733,"豫":734,"費":735,"贵":736,"赤":737,"趙":738,"足":739,"跡":740,"転":741,"辛":742,"逆":743,"遇":744,"運":745,"過":746,"遠":747,"選":748,"邦":749,"邱":750,"部":751,"郭":752,"都":753,"酈":754,"里":755,"野":756,"金":757,"銃":758,"鋼":759,"錄":760,"錡":761,"鍵":762,"鐵":763,"钱":764,"铁":765,"關":766,"防":767,"阿":768,"陈":769,"陳":770,"陽":771,"隊":772,"階":773,"集":774,"雪":775,"雲":776,"霖":777,"霹":778,"靂":779,"韓":780,"願":781,"顯":782,"颜":783,"马":784,"高":785,"龍":786,"ﷲ":787,"ﻋ":788,"/":789,"3":790,"~":791,"##ן":792,"##r":793,"##a":794,"##c":795,"##t":796,"##i":797,"##l":798,"##y":799,"##o":800,"##n":801,"##d":802,"##e":803,"##u":804,"##m":805,"##p":806,"##w":807,"##k":808,"##h":809,"##b":810,"##s":811,"##g":812,"##v":813,"##z":814,"##3":815,"##2":816,"##9":817,"##f":818,"##j":819,"##x":820,"##5":821,"##8":822,"##0":823,"##6":824,"##4":825,"##7":826,"##1":827,"##q":828,"##თ":829,"##ა":830,"##ვ":831,"##რ":832,"##ი":833,"##ق":834,"##د":835,"##ي":836,"##ش":837,"##و":838,"##ლ":839,"##მ":840,"##წ":841,"##ფ":842,"##ე":843,"##ო":844,"##ხ":845,"##დ":846,"##უ":847,"##ł":848,"##ب":849,"##ت":850,"##ل":851,"##ر":852,"##ح":853,"##ル":854,"##レ":855,"##イ":856,"##ン":857,"##し":858,"##た":859,"##ˈ":860,"##り":861,"##の":862,"##ɪ":863,"##ɑ":864,"##ø":865,"##к":866,"##р":867,"##а":868,"##и":869,"##н":870,"##ャ":871,"##モ":872,"##ל":873,"##ה":874,"##ا":875,"##ه":876,"##め":877,"##ɐ":878,"##æ":879,"##л":880,"##đ":881,"##ə":882,"##ː":883,"##っ":884,"##ള":885,"##ɛ":886,"##ك":887,"##เ":888,"##ป":889,"##ย":890,"##δ":891,"##ω":892,"##ν":893,"##ハ":894,"##ト":895,"##ス":896,"##ᅦ":897,"##ᄃ":898,"##ᅩ":899,"##ᆨ":900,"##ᄅ":901,"##ᅵ":902,"##ᆸ":903,"##ᄇ":904,"##ᅧ":905,"##ᆼ":906,"##ᄋ":907,"##ᆫ":908,"##ᅢ":909,"##ウ":910,"##ァ":911,"##キ":912,"##ュ":913,"##リ":914,"##ア":915,"##3":916,"##λ":917,"##α":918,"##χ":919,"##ク":920,"##タ":921,"##ム":922,"##チ":923,"##ェ":924,"##ー":925,"##つ":926,"##せ":927,"##ん":928,"##о":929,"##в":930,"##е":931,"##т":932,"##с":933,"##я":934,"##μ":935,"##ε":936,"##ι":937,"##β":938,"##τ":939,"##ツ":940,"##ネ":941,"##う":942,"##у":943,"##す":944,"##ɔ":945,"##い":946,"##ნ":947,"##ɾ":948,"##ʲ":949,"##º":950,"##は":951,"##く":952,"##か":953,"##ρ":954,"##ξ":955,"##ま":956,"##θ":957,"##η":958,"##ο":959,"##π":960,"##κ":961,"##ς":962,"##ค":963,"##ร":964,"##ว":965,"##ต":966,"##ı":967,"##ß":968,"##พ":969,"##ف":970,"##ʿ":971,"##ს":972,"##ю":973,"##з":974,"##ʊ":975,"##ו":976,"##れ":977,"##ტ":978,"##お":979,"##ッ":980,"##ラ":981,"##ᅮ":982,"##ʻ":983,"##ჱ":984,"##х":985,"##ʃ":986,"##ะ":987,"##ช":988,"##อ":989,"##น":990,"##ذ":991,"##ن":992,"##ة":993,"##ा":994,"##ण":995,"##ि":996,"##ग":997,"##ロ":998,"##サ":999,"##п":1000,"##г":1001,"##ภ":1002,"##า":1003,"##ด":1004,"##פ":1005,"##ც":1006,"##ы":1007,"##ב":1008,"##ר":1009,"##ם":1010,"##ʔ":1011,"##フ":1012,"##ص":1013,"##ɡ":1014,"##ก":1015,"##ल":1016,"##ゆ":1017,"##ई":1018,"##υ":1019,"##ɫ":1020,"##テ":1021,"##ィ":1022,"##ล":1023,"##ง":1024,"##ไ":1025,"##ณ":1026,"##ม":1027,"##こ":1028,"##न":1029,"##क":1030,"##є":1031,"##ɒ":1032,"##י":1033,"##д":1034,"##ъ":1035,"##ж":1036,"##ث":1037,"##م":1038,"##س":1039,"##נ":1040,"##ᅡ":1041,"##ᆯ":1042,"##خ":1043,"##ォ":1044,"##ء":1045,"##な":1046,"##ع":1047,"##ニ":1048,"##エ":1049,"##き":1050,"##ひ":1051,"##と":1052,"##シ":1053,"##ש":1054,"##מ":1055,"##צ":1056,"##ল":1057,"##ゥ":1058,"##ナ":1059,"##ह":1060,"##א":1061,"##マ":1062,"##া":1063,"##হ":1064,"##ら":1065,"##स":1066,"##ษ":1067,"##ฐ":1068,"##द":1069,"##र":1070,"##る":1071,"##あ":1072,"##ょ":1073,"##に":1074,"##გ":1075,"##ð":1076,"##ˌ":1077,"##ミ":1078,"##ฮ":1079,"##ノ":1080,"##ਲ":1081,"##ਹ":1082,"##ш":1083,"##ц":1084,"##ь":1085,"##ち":1086,"##ゃ":1087,"##を":1088,"##ת":1089,"##ਾ":1090,"##も":1091,"##ʒ":1092,"##ʁ":1093,"##ാ":1094,"##ഹ":1095,"##ף":1096,"##γ":1097,"##แ":1098,"##カ":1099,"##œ":1100,"##み":1101,"##σ":1102,"##ჲ":1103,"##ܕ":1104,"##ܝ":1105,"##ܢ":1106,"##ܬ":1107,"##ᵻ":1108,"##セ":1109,"##オ":1110,"##ɳ":1111,"##ܠ":1112,"##ܗ":1113,"##ܐ":1114,"##м":1115,"##ס":1116,"##ɽ":1117,"##ჳ":1118,"##व":1119,"##ქ":1120,"##ɜ":1121,"##ซ":1122,"##わ":1123,"##ψ":1124,"##ョ":1125}}}