tiny-random-funnel / tokenizer.json
LysandreJik's picture
v2 update
4c9aee7
raw history blame
No virus
13.2 kB
{"version":"1.0","truncation":null,"padding":null,"added_tokens":[{"id":0,"special":true,"content":"<pad>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false},{"id":1,"special":true,"content":"<s>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false},{"id":2,"special":true,"content":"</s>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false},{"id":3,"special":true,"content":"<unk>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false},{"id":4,"special":true,"content":"<cls>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false},{"id":5,"special":true,"content":"<sep>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false},{"id":6,"special":true,"content":"<mask>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false}],"normalizer":{"type":"BertNormalizer","clean_text":true,"handle_chinese_chars":true,"strip_accents":null,"lowercase":true},"pre_tokenizer":{"type":"BertPreTokenizer"},"post_processor":{"type":"TemplateProcessing","single":[{"SpecialToken":{"id":"<cls>","type_id":2}},{"Sequence":{"id":"A","type_id":0}},{"SpecialToken":{"id":"<sep>","type_id":0}}],"pair":[{"SpecialToken":{"id":"<cls>","type_id":2}},{"Sequence":{"id":"A","type_id":0}},{"SpecialToken":{"id":"<sep>","type_id":0}},{"Sequence":{"id":"B","type_id":1}},{"SpecialToken":{"id":"<sep>","type_id":1}}],"special_tokens":{"<cls>":{"id":"<cls>","ids":[4],"tokens":["<cls>"]},"<sep>":{"id":"<sep>","ids":[5],"tokens":["<sep>"]}}},"decoder":{"type":"WordPiece","prefix":"##","cleanup":true},"model":{"type":"WordPiece","unk_token":"<unk>","continuing_subword_prefix":"##","max_input_chars_per_word":100,"vocab":{"<pad>":0,"<s>":1,"</s>":2,"<unk>":3,"<cls>":4,"<sep>":5,"<mask>":6,"!":7,"\"":8,"#":9,"$":10,"%":11,"&":12,"'":13,"(":14,")":15,"*":16,"+":17,",":18,"-":19,".":20,"/":21,"0":22,"1":23,"2":24,"3":25,"4":26,"5":27,"6":28,"7":29,"8":30,"9":31,":":32,";":33,"<":34,"=":35,">":36,"?":37,"@":38,"[":39,"\\":40,"]":41,"^":42,"_":43,"`":44,"a":45,"b":46,"c":47,"d":48,"e":49,"f":50,"g":51,"h":52,"i":53,"j":54,"k":55,"l":56,"m":57,"n":58,"o":59,"p":60,"q":61,"r":62,"s":63,"t":64,"u":65,"v":66,"w":67,"x":68,"y":69,"z":70,"|":71,"}":72,"~":73,"¡":74,"¢":75,"£":76,"¥":77,"§":78,"°":79,"±":80,"²":81,"³":82,"´":83,"µ":84,"·":85,"º":86,"½":87,"¿":88,"×":89,"ß":90,"æ":91,"ð":92,"ø":93,"þ":94,"đ":95,"ħ":96,"ı":97,"ł":98,"œ":99,"ɐ":100,"ɑ":101,"ɒ":102,"ɔ":103,"ə":104,"ɛ":105,"ɜ":106,"ɡ":107,"ɢ":108,"ɪ":109,"ɫ":110,"ɳ":111,"ɽ":112,"ɾ":113,"ʁ":114,"ʃ":115,"ʊ":116,"ʋ":117,"ʒ":118,"ʔ":119,"ʕ":120,"ʲ":121,"ʻ":122,"ʼ":123,"ʾ":124,"ʿ":125,"ˈ":126,"ˌ":127,"ː":128,"α":129,"β":130,"γ":131,"δ":132,"ε":133,"η":134,"θ":135,"ι":136,"κ":137,"λ":138,"μ":139,"ν":140,"ξ":141,"ο":142,"π":143,"ρ":144,"ς":145,"σ":146,"τ":147,"υ":148,"φ":149,"χ":150,"ψ":151,"ω":152,"а":153,"б":154,"в":155,"г":156,"д":157,"е":158,"ж":159,"з":160,"и":161,"к":162,"л":163,"м":164,"н":165,"о":166,"п":167,"р":168,"с":169,"т":170,"у":171,"х":172,"ц":173,"ш":174,"ъ":175,"ы":176,"ь":177,"ю":178,"я":179,"є":180,"א":181,"ב":182,"ג":183,"ה":184,"ו":185,"ז":186,"ח":187,"י":188,"ל":189,"ם":190,"מ":191,"ן":192,"נ":193,"ס":194,"ף":195,"פ":196,"צ":197,"ר":198,"ש":199,"ת":200,"ء":201,"ا":202,"ب":203,"ة":204,"ت":205,"ث":206,"ج":207,"ح":208,"خ":209,"د":210,"ذ":211,"ر":212,"س":213,"ش":214,"ص":215,"ع":216,"ف":217,"ق":218,"ك":219,"ل":220,"م":221,"ن":222,"ه":223,"و":224,"ي":225,"ܐ":226,"ܕ":227,"ܗ":228,"ܝ":229,"ܠ":230,"ܢ":231,"ܬ":232,"अ":233,"ई":234,"क":235,"ग":236,"ण":237,"त":238,"द":239,"न":240,"प":241,"ब":242,"म":243,"य":244,"र":245,"ल":246,"व":247,"स":248,"ह":249,"ा":250,"ि":251,"আ":252,"ল":253,"হ":254,"া":255,"ਅ":256,"ਲ":257,"ਹ":258,"ਾ":259,"അ":260,"ള":261,"ഹ":262,"ാ":263,"ก":264,"ค":265,"ง":266,"ช":267,"ซ":268,"ญ":269,"ฐ":270,"ณ":271,"ด":272,"ต":273,"น":274,"บ":275,"ป":276,"พ":277,"ภ":278,"ม":279,"ย":280,"ร":281,"ล":282,"ว":283,"ศ":284,"ษ":285,"ส":286,"ห":287,"อ":288,"ฮ":289,"ะ":290,"า":291,"เ":292,"แ":293,"ไ":294,"ა":295,"ბ":296,"გ":297,"დ":298,"ე":299,"ვ":300,"ზ":301,"თ":302,"ი":303,"კ":304,"ლ":305,"მ":306,"ნ":307,"ო":308,"პ":309,"ჟ":310,"რ":311,"ს":312,"ტ":313,"უ":314,"ფ":315,"ქ":316,"ღ":317,"ყ":318,"შ":319,"ჩ":320,"ც":321,"ძ":322,"წ":323,"ჭ":324,"ხ":325,"ჯ":326,"ჰ":327,"ჱ":328,"ჲ":329,"ჳ":330,"ჴ":331,"ჵ":332,"ჶ":333,"ჷ":334,"ჸ":335,"ჹ":336,"ჺ":337,"჻":338,"ᄃ":339,"ᄅ":340,"ᄇ":341,"ᄋ":342,"ᄌ":343,"ᅡ":344,"ᅢ":345,"ᅦ":346,"ᅧ":347,"ᅩ":348,"ᅮ":349,"ᅵ":350,"ᆨ":351,"ᆫ":352,"ᆯ":353,"ᆸ":354,"ᆼ":355,"ᵻ":356,"‐":357,"‑":358,"–":359,"—":360,"―":361,"‘":362,"’":363,"“":364,"”":365,"„":366,"†":367,"‡":368,"•":369,"…":370,"′":371,"″":372,"⁄":373,"₣":374,"₤":375,"€":376,"₹":377,"⅓":378,"⅔":379,"→":380,"−":381,"≡":382,"≤":383,"①":384,"☉":385,"☫":386,"♀":387,"♭":388,"♯":389,"⚳":390,"ⴀ":391,"ⴂ":392,"ⴃ":393,"ⴈ":394,"ⴌ":395,"ⴕ":396,"ⴟ":397,"〈":398,"〉":399,"〜":400,"あ":401,"い":402,"う":403,"お":404,"か":405,"き":406,"く":407,"け":408,"こ":409,"さ":410,"し":411,"す":412,"せ":413,"た":414,"ち":415,"っ":416,"つ":417,"と":418,"な":419,"に":420,"の":421,"は":422,"ひ":423,"ふ":424,"ほ":425,"ま":426,"み":427,"め":428,"も":429,"ゃ":430,"ゆ":431,"ょ":432,"ら":433,"り":434,"る":435,"れ":436,"わ":437,"を":438,"ん":439,"ァ":440,"ア":441,"ィ":442,"イ":443,"ゥ":444,"ウ":445,"ェ":446,"エ":447,"ォ":448,"オ":449,"カ":450,"キ":451,"ク":452,"ケ":453,"コ":454,"サ":455,"シ":456,"ス":457,"セ":458,"タ":459,"チ":460,"ッ":461,"ツ":462,"テ":463,"ト":464,"ナ":465,"ニ":466,"ネ":467,"ノ":468,"ハ":469,"フ":470,"ヘ":471,"マ":472,"ミ":473,"ム":474,"モ":475,"ャ":476,"ュ":477,"ョ":478,"ラ":479,"リ":480,"ル":481,"レ":482,"ロ":483,"ン":484,"・":485,"ー":486,"一":487,"七":488,"下":489,"世":490,"丙":491,"中":492,"主":493,"乃":494,"之":495,"乙":496,"九":497,"二":498,"云":499,"人":500,"今":501,"付":502,"作":503,"侗":504,"依":505,"信":506,"傳":507,"儚":508,"充":509,"光":510,"全":511,"兵":512,"其":513,"具":514,"円":515,"再":516,"出":517,"判":518,"前":519,"剛":520,"劇":521,"劉":522,"動":523,"化":524,"北":525,"华":526,"厂":527,"去":528,"古":529,"可":530,"台":531,"史":532,"同":533,"名":534,"君":535,"吳":536,"周":537,"命":538,"和":539,"咲":540,"善":541,"四":542,"國":543,"園":544,"圣":545,"在":546,"坂":547,"堤":548,"場":549,"塘":550,"夕":551,"大":552,"天":553,"夫":554,"女":555,"妙":556,"姚":557,"子":558,"孟":559,"守":560,"安":561,"宋":562,"完":563,"宗":564,"宝":565,"宫":566,"寝":567,"寺":568,"小":569,"少":570,"尾":571,"山":572,"岳":573,"川":574,"州":575,"巳":576,"市":577,"師":578,"平":579,"广":580,"庆":581,"府":582,"座":583,"廬":584,"建":585,"式":586,"張":587,"彌":588,"彩":589,"彼":590,"後":591,"御":592,"德":593,"思":594,"愛":595,"憑":596,"憶":597,"應":598,"懷":599,"战":600,"戦":601,"扈":602,"技":603,"拉":604,"拳":605,"挑":606,"揺":607,"攻":608,"放":609,"政":610,"散":611,"斯":612,"方":613,"日":614,"旦":615,"旭":616,"昌":617,"明":618,"星":619,"春":620,"晋":621,"景":622,"曦":623,"月":624,"望":625,"未":626,"本":627,"李":628,"村":629,"杜":630,"束":631,"来":632,"林":633,"桜":634,"梶":635,"棘":636,"椎":637,"楊":638,"楚":639,"榮":640,"橘":641,"機":642,"正":643,"殻":644,"殿":645,"母":646,"水":647,"汉":648,"沂":649,"沙":650,"河":651,"泗":652,"波":653,"泣":654,"洪":655,"淹":656,"清":657,"湯":658,"漢":659,"澄":660,"澤":661,"火":662,"灯":663,"灵":664,"灼":665,"焼":666,"熱":667,"物":668,"狐":669,"狸":670,"玄":671,"王":672,"玩":673,"珂":674,"珙":675,"球":676,"理":677,"琦":678,"琪":679,"瓊":680,"生":681,"田":682,"畢":683,"番":684,"瘡":685,"白":686,"皮":687,"真":688,"砲":689,"礮":690,"祈":691,"神":692,"祠":693,"秋":694,"空":695,"立":696,"精":697,"約":698,"絵":699,"織":700,"義":701,"翠":702,"者":703,"耕":704,"肖":705,"胡":706,"膀":707,"臂":708,"興":709,"良":710,"花":711,"芳":712,"芽":713,"若":714,"英":715,"藕":716,"藥":717,"蘄":718,"蘇":719,"行":720,"裁":721,"規":722,"覺":723,"观":724,"解":725,"記":726,"誓":727,"誡":728,"誰":729,"謎":730,"许":731,"谭":732,"豪":733,"豫":734,"費":735,"贵":736,"赤":737,"趙":738,"足":739,"跡":740,"転":741,"辛":742,"逆":743,"遇":744,"運":745,"過":746,"遠":747,"選":748,"邦":749,"邱":750,"部":751,"郭":752,"都":753,"酈":754,"里":755,"野":756,"金":757,"銃":758,"鋼":759,"錄":760,"錡":761,"鍵":762,"鐵":763,"钱":764,"铁":765,"關":766,"防":767,"阿":768,"陈":769,"陳":770,"陽":771,"隊":772,"階":773,"集":774,"雪":775,"雲":776,"霖":777,"霹":778,"靂":779,"韓":780,"願":781,"顯":782,"颜":783,"马":784,"高":785,"龍":786,"ﷲ":787,"ﻋ":788,"/":789,"3":790,"~":791,"##i":792,"##l":793,"##s":794,"##e":795,"##g":796,"##r":797,"##o":798,"##7":799,"##4":800,"##0":801,"##n":802,"##d":803,"##c":804,"##t":805,"##3":806,"##w":807,"##p":808,"##a":809,"##h":810,"##y":811,"##u":812,"##f":813,"##b":814,"##v":815,"##x":816,"##1":817,"##m":818,"##8":819,"##k":820,"##j":821,"##z":822,"##q":823,"##6":824,"##2":825,"##ʻ":826,"##9":827,"##פ":828,"##ה":829,"##ː":830,"##5":831,"##а":832,"##т":833,"##е":834,"##д":835,"##р":836,"##л":837,"##ო":838,"##ი":839,"##ø":840,"##ܠ":841,"##ܗ":842,"##ܐ":843,"##đ":844,"##ל":845,"##א":846,"##ł":847,"##は":848,"##め":849,"##ツ":850,"##ネ":851,"##უ":852,"##ც":853,"##ე":854,"##ს":855,"##ʿ":856,"##ß":857,"##მ":858,"##ლ":859,"##ɑ":860,"##た":861,"##の":862,"##π":863,"##α":864,"##τ":865,"##ι":866,"##κ":867,"##ο":868,"##ς":869,"##な":870,"##ら":871,"##æ":872,"##ল":873,"##س":874,"##ت":875,"##ا":876,"##ن":877,"##δ":878,"##ω":879,"##ν":880,"##フ":881,"##м":882,"##п":883,"##и":884,"##н":885,"##о":886,"##г":887,"##り":888,"##ン":889,"##カ":890,"##ラ":891,"##お":892,"##か":893,"##ɪ":894,"##ə":895,"##ل":896,"##ب":897,"##μ":898,"##ρ":899,"##ッ":900,"##ク":901,"##マ":902,"##ו":903,"##ा":904,"##ण":905,"##ई":906,"##ı":907,"##し":908,"##す":909,"##き":910,"##ひ":911,"##と":912,"##ε":913,"##ノ":914,"##ー":915,"##ォ":916,"##ル":917,"##テ":918,"##ش":919,"##م":920,"##و":921,"##व":922,"##ह":923,"##ि":924,"##क":925,"##λ":926,"##η":927,"##χ":928,"##ˈ":929,"##υ":930,"##ю":931,"##з":932,"##х":933,"##ロ":934,"##イ":935,"##ト":936,"##キ":937,"##ɽ":938,"##ხ":939,"##დ":940,"##ა":941,"##რ":942,"##გ":943,"##ვ":944,"##ნ":945,"##こ":946,"##い":947,"##ち":948,"##ゃ":949,"##ん":950,"##つ":951,"##を":952,"##ล":953,"##ว":954,"##ง":955,"##พ":956,"##ไ":957,"##ช":958,"##ย":959,"##า":960,"##ร":961,"##ق":962,"##د":963,"##ي":964,"##ש":965,"##מ":966,"##צ":967,"##β":968,"##ャ":969,"##モ":970,"##ð":971,"##ψ":972,"##ィ":973,"##ر":974,"##ح":975,"##ɒ":976,"##к":977,"##ェ":978,"##ス":979,"##ハ":980,"##в":981,"##с":982,"##レ":983,"##ف":984,"##ɛ":985,"##ʊ":986,"##ш":987,"##თ":988,"##く":989,"##み":990,"##ܕ":991,"##ܝ":992,"##ܢ":993,"##ܬ":994,"##ภ":995,"##ด":996,"##ɔ":997,"##ア":998,"##タ":999,"##ɾ":1000,"##ʲ":1001,"##ן":1002,"##リ":1003,"##ナ":1004,"##ᅡ":1005,"##ᆯ":1006,"##ᄅ":1007,"##া":1008,"##হ":1009,"##ъ":1010,"##う":1011,"##っ":1012,"##ょ":1013,"##に":1014,"##द":1015,"##ュ":1016,"##ミ":1017,"##อ":1018,"##ก":1019,"##ษ":1020,"##ฐ":1021,"##ค":1022,"##ต":1023,"##я":1024,"##წ":1025,"##ფ":1026,"##ه":1027,"##ɫ":1028,"##न":1029,"##あ":1030,"##ᅦ":1031,"##ᄃ":1032,"##ᅩ":1033,"##ᆨ":1034,"##ᅵ":1035,"##ᆸ":1036,"##ᄇ":1037,"##ᅧ":1038,"##ᆼ":1039,"##ᄋ":1040,"##ᆫ":1041,"##ᅢ":1042,"##स":1043,"##र":1044,"##ы":1045,"##ウ":1046,"##ァ":1047,"##も":1048,"##γ":1049,"##เ":1050,"##ป":1051,"##ɡ":1052,"##ث":1053,"##י":1054,"##ם":1055,"##サ":1056,"##خ":1057,"##ס":1058,"##せ":1059,"##ك":1060,"##є":1061,"##3":1062,"##ฮ":1063,"##セ":1064,"##オ":1065,"##ع":1066,"##ʃ":1067,"##ц":1068,"##ь":1069,"##ɜ":1070,"##ף":1071,"##ะ":1072,"##น":1073,"##נ":1074,"##ณ":1075,"##ม":1076,"##ص":1077,"##れ":1078,"##ტ":1079,"##ჲ":1080,"##ذ":1081,"##ة":1082,"##ニ":1083,"##ਲ":1084,"##ਹ":1085,"##ซ":1086,"##シ":1087,"##ョ":1088,"##わ":1089,"##θ":1090,"##ჱ":1091,"##ɐ":1092,"##у":1093,"##œ":1094,"##ב":1095,"##ר":1096,"##σ":1097,"##チ":1098,"##ま":1099,"##ാ":1100,"##ഹ":1101,"##る":1102,"##ल":1103,"##ᵻ":1104,"##แ":1105,"##ჳ":1106,"##ɳ":1107,"##ת":1108,"##ゥ":1109,"##エ":1110,"##ʁ":1111,"##ᅮ":1112,"##ء":1113,"##ള":1114,"##º":1115,"##ʔ":1116,"##ˌ":1117,"##ʒ":1118,"##ж":1119,"##ξ":1120,"##ゆ":1121,"##ਾ":1122,"##ग":1123,"##ქ":1124,"##ム":1125}}}