tiny-random-mobilebert / tokenizer.json
LysandreJik's picture
v3
1f919a6
{"version":"1.0","truncation":null,"padding":null,"added_tokens":[{"id":0,"special":true,"content":"[PAD]","single_word":false,"lstrip":false,"rstrip":false,"normalized":false},{"id":1,"special":true,"content":"[UNK]","single_word":false,"lstrip":false,"rstrip":false,"normalized":false},{"id":2,"special":true,"content":"[CLS]","single_word":false,"lstrip":false,"rstrip":false,"normalized":false},{"id":3,"special":true,"content":"[SEP]","single_word":false,"lstrip":false,"rstrip":false,"normalized":false},{"id":4,"special":true,"content":"[MASK]","single_word":false,"lstrip":false,"rstrip":false,"normalized":false}],"normalizer":{"type":"BertNormalizer","clean_text":true,"handle_chinese_chars":true,"strip_accents":null,"lowercase":true},"pre_tokenizer":{"type":"BertPreTokenizer"},"post_processor":{"type":"TemplateProcessing","single":[{"SpecialToken":{"id":"[CLS]","type_id":0}},{"Sequence":{"id":"A","type_id":0}},{"SpecialToken":{"id":"[SEP]","type_id":0}}],"pair":[{"SpecialToken":{"id":"[CLS]","type_id":0}},{"Sequence":{"id":"A","type_id":0}},{"SpecialToken":{"id":"[SEP]","type_id":0}},{"Sequence":{"id":"B","type_id":1}},{"SpecialToken":{"id":"[SEP]","type_id":1}}],"special_tokens":{"[CLS]":{"id":"[CLS]","ids":[2],"tokens":["[CLS]"]},"[SEP]":{"id":"[SEP]","ids":[3],"tokens":["[SEP]"]}}},"decoder":{"type":"WordPiece","prefix":"##","cleanup":true},"model":{"type":"WordPiece","unk_token":"[UNK]","continuing_subword_prefix":"##","max_input_chars_per_word":100,"vocab":{"[PAD]":0,"[UNK]":1,"[CLS]":2,"[SEP]":3,"[MASK]":4,"!":5,"\"":6,"#":7,"$":8,"%":9,"&":10,"'":11,"(":12,")":13,"*":14,"+":15,",":16,"-":17,".":18,"/":19,"0":20,"1":21,"2":22,"3":23,"4":24,"5":25,"6":26,"7":27,"8":28,"9":29,":":30,";":31,"<":32,"=":33,">":34,"?":35,"@":36,"[":37,"\\":38,"]":39,"^":40,"_":41,"`":42,"a":43,"b":44,"c":45,"d":46,"e":47,"f":48,"g":49,"h":50,"i":51,"j":52,"k":53,"l":54,"m":55,"n":56,"o":57,"p":58,"q":59,"r":60,"s":61,"t":62,"u":63,"v":64,"w":65,"x":66,"y":67,"z":68,"|":69,"}":70,"~":71,"¡":72,"¢":73,"£":74,"¥":75,"§":76,"°":77,"±":78,"²":79,"³":80,"´":81,"µ":82,"·":83,"º":84,"½":85,"¿":86,"×":87,"ß":88,"æ":89,"ð":90,"ø":91,"þ":92,"đ":93,"ħ":94,"ı":95,"ł":96,"œ":97,"ɐ":98,"ɑ":99,"ɒ":100,"ɔ":101,"ə":102,"ɛ":103,"ɜ":104,"ɡ":105,"ɢ":106,"ɪ":107,"ɫ":108,"ɳ":109,"ɽ":110,"ɾ":111,"ʁ":112,"ʃ":113,"ʊ":114,"ʋ":115,"ʒ":116,"ʔ":117,"ʕ":118,"ʲ":119,"ʻ":120,"ʼ":121,"ʾ":122,"ʿ":123,"ˈ":124,"ˌ":125,"ː":126,"α":127,"β":128,"γ":129,"δ":130,"ε":131,"η":132,"θ":133,"ι":134,"κ":135,"λ":136,"μ":137,"ν":138,"ξ":139,"ο":140,"π":141,"ρ":142,"ς":143,"σ":144,"τ":145,"υ":146,"φ":147,"χ":148,"ψ":149,"ω":150,"а":151,"б":152,"в":153,"г":154,"д":155,"е":156,"ж":157,"з":158,"и":159,"к":160,"л":161,"м":162,"н":163,"о":164,"п":165,"р":166,"с":167,"т":168,"у":169,"х":170,"ц":171,"ш":172,"ъ":173,"ы":174,"ь":175,"ю":176,"я":177,"є":178,"א":179,"ב":180,"ג":181,"ה":182,"ו":183,"ז":184,"ח":185,"י":186,"ל":187,"ם":188,"מ":189,"ן":190,"נ":191,"ס":192,"ף":193,"פ":194,"צ":195,"ר":196,"ש":197,"ת":198,"ء":199,"ا":200,"ب":201,"ة":202,"ت":203,"ث":204,"ج":205,"ح":206,"خ":207,"د":208,"ذ":209,"ر":210,"س":211,"ش":212,"ص":213,"ع":214,"ف":215,"ق":216,"ك":217,"ل":218,"م":219,"ن":220,"ه":221,"و":222,"ي":223,"ܐ":224,"ܕ":225,"ܗ":226,"ܝ":227,"ܠ":228,"ܢ":229,"ܬ":230,"अ":231,"ई":232,"क":233,"ग":234,"ण":235,"त":236,"द":237,"न":238,"प":239,"ब":240,"म":241,"य":242,"र":243,"ल":244,"व":245,"स":246,"ह":247,"ा":248,"ि":249,"আ":250,"ল":251,"হ":252,"া":253,"ਅ":254,"ਲ":255,"ਹ":256,"ਾ":257,"അ":258,"ള":259,"ഹ":260,"ാ":261,"ก":262,"ค":263,"ง":264,"ช":265,"ซ":266,"ญ":267,"ฐ":268,"ณ":269,"ด":270,"ต":271,"น":272,"บ":273,"ป":274,"พ":275,"ภ":276,"ม":277,"ย":278,"ร":279,"ล":280,"ว":281,"ศ":282,"ษ":283,"ส":284,"ห":285,"อ":286,"ฮ":287,"ะ":288,"า":289,"เ":290,"แ":291,"ไ":292,"ა":293,"ბ":294,"გ":295,"დ":296,"ე":297,"ვ":298,"ზ":299,"თ":300,"ი":301,"კ":302,"ლ":303,"მ":304,"ნ":305,"ო":306,"პ":307,"ჟ":308,"რ":309,"ს":310,"ტ":311,"უ":312,"ფ":313,"ქ":314,"ღ":315,"ყ":316,"შ":317,"ჩ":318,"ც":319,"ძ":320,"წ":321,"ჭ":322,"ხ":323,"ჯ":324,"ჰ":325,"ჱ":326,"ჲ":327,"ჳ":328,"ჴ":329,"ჵ":330,"ჶ":331,"ჷ":332,"ჸ":333,"ჹ":334,"ჺ":335,"჻":336,"ᄃ":337,"ᄅ":338,"ᄇ":339,"ᄋ":340,"ᄌ":341,"ᅡ":342,"ᅢ":343,"ᅦ":344,"ᅧ":345,"ᅩ":346,"ᅮ":347,"ᅵ":348,"ᆨ":349,"ᆫ":350,"ᆯ":351,"ᆸ":352,"ᆼ":353,"ᵻ":354,"‐":355,"‑":356,"–":357,"—":358,"―":359,"‘":360,"’":361,"“":362,"”":363,"„":364,"†":365,"‡":366,"•":367,"…":368,"′":369,"″":370,"⁄":371,"₣":372,"₤":373,"€":374,"₹":375,"⅓":376,"⅔":377,"→":378,"−":379,"≡":380,"≤":381,"①":382,"☉":383,"☫":384,"♀":385,"♭":386,"♯":387,"⚳":388,"ⴀ":389,"ⴂ":390,"ⴃ":391,"ⴈ":392,"ⴌ":393,"ⴕ":394,"ⴟ":395,"〈":396,"〉":397,"〜":398,"あ":399,"い":400,"う":401,"お":402,"か":403,"き":404,"く":405,"け":406,"こ":407,"さ":408,"し":409,"す":410,"せ":411,"た":412,"ち":413,"っ":414,"つ":415,"と":416,"な":417,"に":418,"の":419,"は":420,"ひ":421,"ふ":422,"ほ":423,"ま":424,"み":425,"め":426,"も":427,"ゃ":428,"ゆ":429,"ょ":430,"ら":431,"り":432,"る":433,"れ":434,"わ":435,"を":436,"ん":437,"ァ":438,"ア":439,"ィ":440,"イ":441,"ゥ":442,"ウ":443,"ェ":444,"エ":445,"ォ":446,"オ":447,"カ":448,"キ":449,"ク":450,"ケ":451,"コ":452,"サ":453,"シ":454,"ス":455,"セ":456,"タ":457,"チ":458,"ッ":459,"ツ":460,"テ":461,"ト":462,"ナ":463,"ニ":464,"ネ":465,"ノ":466,"ハ":467,"フ":468,"ヘ":469,"マ":470,"ミ":471,"ム":472,"モ":473,"ャ":474,"ュ":475,"ョ":476,"ラ":477,"リ":478,"ル":479,"レ":480,"ロ":481,"ン":482,"・":483,"ー":484,"一":485,"七":486,"下":487,"世":488,"丙":489,"中":490,"主":491,"乃":492,"之":493,"乙":494,"九":495,"二":496,"云":497,"人":498,"今":499,"付":500,"作":501,"侗":502,"依":503,"信":504,"傳":505,"儚":506,"充":507,"光":508,"全":509,"兵":510,"其":511,"具":512,"円":513,"再":514,"出":515,"判":516,"前":517,"剛":518,"劇":519,"劉":520,"動":521,"化":522,"北":523,"华":524,"厂":525,"去":526,"古":527,"可":528,"台":529,"史":530,"同":531,"名":532,"君":533,"吳":534,"周":535,"命":536,"和":537,"咲":538,"善":539,"四":540,"國":541,"園":542,"圣":543,"在":544,"坂":545,"堤":546,"場":547,"塘":548,"夕":549,"大":550,"天":551,"夫":552,"女":553,"妙":554,"姚":555,"子":556,"孟":557,"守":558,"安":559,"宋":560,"完":561,"宗":562,"宝":563,"宫":564,"寝":565,"寺":566,"小":567,"少":568,"尾":569,"山":570,"岳":571,"川":572,"州":573,"巳":574,"市":575,"師":576,"平":577,"广":578,"庆":579,"府":580,"座":581,"廬":582,"建":583,"式":584,"張":585,"彌":586,"彩":587,"彼":588,"後":589,"御":590,"德":591,"思":592,"愛":593,"憑":594,"憶":595,"應":596,"懷":597,"战":598,"戦":599,"扈":600,"技":601,"拉":602,"拳":603,"挑":604,"揺":605,"攻":606,"放":607,"政":608,"散":609,"斯":610,"方":611,"日":612,"旦":613,"旭":614,"昌":615,"明":616,"星":617,"春":618,"晋":619,"景":620,"曦":621,"月":622,"望":623,"未":624,"本":625,"李":626,"村":627,"杜":628,"束":629,"来":630,"林":631,"桜":632,"梶":633,"棘":634,"椎":635,"楊":636,"楚":637,"榮":638,"橘":639,"機":640,"正":641,"殻":642,"殿":643,"母":644,"水":645,"汉":646,"沂":647,"沙":648,"河":649,"泗":650,"波":651,"泣":652,"洪":653,"淹":654,"清":655,"湯":656,"漢":657,"澄":658,"澤":659,"火":660,"灯":661,"灵":662,"灼":663,"焼":664,"熱":665,"物":666,"狐":667,"狸":668,"玄":669,"王":670,"玩":671,"珂":672,"珙":673,"球":674,"理":675,"琦":676,"琪":677,"瓊":678,"生":679,"田":680,"畢":681,"番":682,"瘡":683,"白":684,"皮":685,"真":686,"砲":687,"礮":688,"祈":689,"神":690,"祠":691,"秋":692,"空":693,"立":694,"精":695,"約":696,"絵":697,"織":698,"義":699,"翠":700,"者":701,"耕":702,"肖":703,"胡":704,"膀":705,"臂":706,"興":707,"良":708,"花":709,"芳":710,"芽":711,"若":712,"英":713,"藕":714,"藥":715,"蘄":716,"蘇":717,"行":718,"裁":719,"規":720,"覺":721,"观":722,"解":723,"記":724,"誓":725,"誡":726,"誰":727,"謎":728,"许":729,"谭":730,"豪":731,"豫":732,"費":733,"贵":734,"赤":735,"趙":736,"足":737,"跡":738,"転":739,"辛":740,"逆":741,"遇":742,"運":743,"過":744,"遠":745,"選":746,"邦":747,"邱":748,"部":749,"郭":750,"都":751,"酈":752,"里":753,"野":754,"金":755,"銃":756,"鋼":757,"錄":758,"錡":759,"鍵":760,"鐵":761,"钱":762,"铁":763,"關":764,"防":765,"阿":766,"陈":767,"陳":768,"陽":769,"隊":770,"階":771,"集":772,"雪":773,"雲":774,"霖":775,"霹":776,"靂":777,"韓":778,"願":779,"顯":780,"颜":781,"马":782,"高":783,"龍":784,"ﷲ":785,"ﻋ":786,"/":787,"3":788,"~":789,"##o":790,"##u":791,"##p":792,"##l":793,"##e":794,"##a":795,"##c":796,"##h":797,"##s":798,"##i":799,"##d":800,"##t":801,"##ラ":802,"##ー":803,"##n":804,"##r":805,"##j":806,"##v":807,"##w":808,"##k":809,"##z":810,"##y":811,"##m":812,"##g":813,"##っ":814,"##こ":815,"##ん":816,"##f":817,"##6":818,"##4":819,"##2":820,"##9":821,"##7":822,"##0":823,"##x":824,"##b":825,"##3":826,"##5":827,"##1":828,"##8":829,"##ʻ":830,"##ß":831,"##ɔ":832,"##ː":833,"##ɡ":834,"##ə":835,"##ˈ":836,"##ʃ":837,"##ɪ":838,"##q":839,"##ł":840,"##か":841,"##イ":842,"##レ":843,"##ツ":844,"##ו":845,"##פ":846,"##ה":847,"##ش":848,"##م":849,"##و":850,"##ن":851,"##æ":852,"##χ":853,"##ネ":854,"##ッ":855,"##ト":856,"##フ":857,"##ォ":858,"##ン":859,"##ア":860,"##ク":861,"##タ":862,"##ा":863,"##ह":864,"##р":865,"##и":866,"##с":867,"##т":868,"##о":869,"##マ":870,"##た":871,"##し":872,"##の":873,"##す":874,"##き":875,"##な":876,"##ひ":877,"##と":878,"##ま":879,"##ノ":880,"##ψ":881,"##ι":882,"##ς":883,"##е":884,"##л":885,"##у":886,"##я":887,"##र":888,"##ν":889,"##τ":890,"##μ":891,"##ο":892,"##უ":893,"##ც":894,"##ე":895,"##ს":896,"##ი":897,"##ي":898,"##ف":899,"##ا":900,"##ল":901,"##つ":902,"##う":903,"##α":904,"##ܕ":905,"##ܝ":906,"##ܢ":907,"##ܬ":908,"##を":909,"##ل":910,"##ب":911,"##ø":912,"##ε":913,"##ρ":914,"##ภ":915,"##า":916,"##ด":917,"##ण":918,"##ई":919,"##ル":920,"##セ":921,"##リ":922,"##オ":923,"##ス":924,"##π":925,"##ص":926,"##ɛ":927,"##ɒ":928,"##ق":929,"##ع":930,"##د":931,"##ı":932,"##ל":933,"##ɑ":934,"##ᵻ":935,"##ʊ":936,"##đ":937,"##რ":938,"##ป":939,"##ย":940,"##ო":941,"##ാ":942,"##ഹ":943,"##ʿ":944,"##д":945,"##ъ":946,"##ж":947,"##а":948,"##β":949,"##λ":950,"##ウ":951,"##ァ":952,"##キ":953,"##ュ":954,"##י":955,"##ि":956,"##ग":957,"##り":958,"##س":959,"##θ":960,"##η":961,"##ल":962,"##ه":963,"##れ":964,"##い":965,"##ю":966,"##з":967,"##ᅦ":968,"##ᄃ":969,"##ᅩ":970,"##ᆨ":971,"##ᄅ":972,"##ᅵ":973,"##ᆸ":974,"##ᄇ":975,"##ᅧ":976,"##ᆼ":977,"##ᄋ":978,"##ᆫ":979,"##ᅢ":980,"##ב":981,"##ר":982,"##ם":983,"##ტ":984,"##ค":985,"##ร":986,"##ว":987,"##ต":988,"##स":989,"##न":990,"##х":991,"##ら":992,"##ล":993,"##ง":994,"##พ":995,"##ไ":996,"##ช":997,"##เ":998,"##व":999,"##द":1000,"##া":1001,"##হ":1002,"##テ":1003,"##ィ":1004,"##ɳ":1005,"##ゥ":1006,"##ナ":1007,"##は":1008,"##ث":1009,"##ы":1010,"##н":1011,"##ω":1012,"##ر":1013,"##ð":1014,"##エ":1015,"##º":1016,"##ょ":1017,"##に":1018,"##ェ":1019,"##ハ":1020,"##ニ":1021,"##る":1022,"##ლ":1023,"##მ":1024,"##წ":1025,"##ფ":1026,"##γ":1027,"##в":1028,"##к":1029,"##อ":1030,"##ก":1031,"##3":1032,"##נ":1033,"##ჱ":1034,"##แ":1035,"##ᅡ":1036,"##ᆯ":1037,"##ซ":1038,"##め":1039,"##ロ":1040,"##お":1041,"##ס":1042,"##ء":1043,"##δ":1044,"##ャ":1045,"##モ":1046,"##く":1047,"##あ":1048,"##ხ":1049,"##დ":1050,"##ა":1051,"##ʒ":1052,"##ت":1053,"##ჳ":1054,"##ة":1055,"##ʁ":1056,"##ף":1057,"##ɜ":1058,"##œ":1059,"##ᅮ":1060,"##ณ":1061,"##ม":1062,"##ك":1063,"##ש":1064,"##מ":1065,"##צ":1066,"##ษ":1067,"##ฐ":1068,"##σ":1069,"##υ":1070,"##み":1071,"##カ":1072,"##क":1073,"##ਲ":1074,"##ਹ":1075,"##п":1076,"##ゆ":1077,"##シ":1078,"##ョ":1079,"##ʔ":1080,"##ვ":1081,"##ɐ":1082,"##ذ":1083,"##チ":1084,"##ш":1085,"##ξ":1086,"##も":1087,"##خ":1088,"##ح":1089,"##κ":1090,"##ן":1091,"##א":1092,"##ц":1093,"##ь":1094,"##ɾ":1095,"##ʲ":1096,"##ミ":1097,"##ള":1098,"##ქ":1099,"##ნ":1100,"##わ":1101,"##є":1102,"##ჲ":1103,"##ɫ":1104,"##თ":1105,"##ฮ":1106,"##ת":1107,"##ˌ":1108,"##ち":1109,"##ゃ":1110,"##გ":1111,"##ム":1112,"##せ":1113,"##ɽ":1114,"##ะ":1115,"##น":1116,"##м":1117,"##ਾ":1118,"##г":1119,"##サ":1120,"##ܠ":1121,"##ܗ":1122,"##ܐ":1123}}}