tiny-random-dpr / tokenizer.json
1 {"version":"1.0","truncation":null,"padding":null,"added_tokens":[{"id":0,"special":true,"content":"[PAD]","single_word":false,"lstrip":false,"rstrip":false,"normalized":false},{"id":1,"special":true,"content":"[UNK]","single_word":false,"lstrip":false,"rstrip":false,"normalized":false},{"id":2,"special":true,"content":"[CLS]","single_word":false,"lstrip":false,"rstrip":false,"normalized":false},{"id":3,"special":true,"content":"[SEP]","single_word":false,"lstrip":false,"rstrip":false,"normalized":false},{"id":4,"special":true,"content":"[MASK]","single_word":false,"lstrip":false,"rstrip":false,"normalized":false}],"normalizer":{"type":"BertNormalizer","clean_text":true,"handle_chinese_chars":true,"strip_accents":null,"lowercase":true},"pre_tokenizer":{"type":"BertPreTokenizer"},"post_processor":{"type":"TemplateProcessing","single":[{"SpecialToken":{"id":"[CLS]","type_id":0}},{"Sequence":{"id":"A","type_id":0}},{"SpecialToken":{"id":"[SEP]","type_id":0}}],"pair":[{"SpecialToken":{"id":"[CLS]","type_id":0}},{"Sequence":{"id":"A","type_id":0}},{"SpecialToken":{"id":"[SEP]","type_id":0}},{"Sequence":{"id":"B","type_id":1}},{"SpecialToken":{"id":"[SEP]","type_id":1}}],"special_tokens":{"[CLS]":{"id":"[CLS]","ids":[2],"tokens":["[CLS]"]},"[SEP]":{"id":"[SEP]","ids":[3],"tokens":["[SEP]"]}}},"decoder":{"type":"WordPiece","prefix":"##","cleanup":true},"model":{"type":"WordPiece","unk_token":"[UNK]","continuing_subword_prefix":"##","max_input_chars_per_word":100,"vocab":{"[PAD]":0,"[UNK]":1,"[CLS]":2,"[SEP]":3,"[MASK]":4,"!":5,"\"":6,"#":7,"$":8,"%":9,"&":10,"'":11,"(":12,")":13,"*":14,"+":15,",":16,"-":17,".":18,"/":19,"0":20,"1":21,"2":22,"3":23,"4":24,"5":25,"6":26,"7":27,"8":28,"9":29,":":30,";":31,"<":32,"=":33,">":34,"?":35,"@":36,"[":37,"\\":38,"]":39,"^":40,"_":41,"`":42,"a":43,"b":44,"c":45,"d":46,"e":47,"f":48,"g":49,"h":50,"i":51,"j":52,"k":53,"l":54,"m":55,"n":56,"o":57,"p":58,"q":59,"r":60,"s":61,"t":62,"u":63,"v":64,"w":65,"x":66,"y":67,"z":68,"|":69,"}":70,"~":71,"¡":72,"¢":73,"£":74,"¥":75,"§":76,"°":77,"±":78,"²":79,"³":80,"´":81,"µ":82,"·":83,"º":84,"½":85,"¿":86,"×":87,"ß":88,"æ":89,"ð":90,"ø":91,"þ":92,"đ":93,"ħ":94,"ı":95,"ł":96,"œ":97,"ɐ":98,"ɑ":99,"ɒ":100,"ɔ":101,"ə":102,"ɛ":103,"ɜ":104,"ɡ":105,"ɢ":106,"ɪ":107,"ɫ":108,"ɳ":109,"ɽ":110,"ɾ":111,"ʁ":112,"ʃ":113,"ʊ":114,"ʋ":115,"ʒ":116,"ʔ":117,"ʕ":118,"ʲ":119,"ʻ":120,"ʼ":121,"ʾ":122,"ʿ":123,"ˈ":124,"ˌ":125,"ː":126,"α":127,"β":128,"γ":129,"δ":130,"ε":131,"η":132,"θ":133,"ι":134,"κ":135,"λ":136,"μ":137,"ν":138,"ξ":139,"ο":140,"π":141,"ρ":142,"ς":143,"σ":144,"τ":145,"υ":146,"φ":147,"χ":148,"ψ":149,"ω":150,"а":151,"б":152,"в":153,"г":154,"д":155,"е":156,"ж":157,"з":158,"и":159,"к":160,"л":161,"м":162,"н":163,"о":164,"п":165,"р":166,"с":167,"т":168,"у":169,"х":170,"ц":171,"ш":172,"ъ":173,"ы":174,"ь":175,"ю":176,"я":177,"є":178,"א":179,"ב":180,"ג":181,"ה":182,"ו":183,"ז":184,"ח":185,"י":186,"ל":187,"ם":188,"מ":189,"ן":190,"נ":191,"ס":192,"ף":193,"פ":194,"צ":195,"ר":196,"ש":197,"ת":198,"ء":199,"ا":200,"ب":201,"ة":202,"ت":203,"ث":204,"ج":205,"ح":206,"خ":207,"د":208,"ذ":209,"ر":210,"س":211,"ش":212,"ص":213,"ع":214,"ف":215,"ق":216,"ك":217,"ل":218,"م":219,"ن":220,"ه":221,"و":222,"ي":223,"ܐ":224,"ܕ":225,"ܗ":226,"ܝ":227,"ܠ":228,"ܢ":229,"ܬ":230,"अ":231,"ई":232,"क":233,"ग":234,"ण":235,"त":236,"द":237,"न":238,"प":239,"ब":240,"म":241,"य":242,"र":243,"ल":244,"व":245,"स":246,"ह":247,"ा":248,"ि":249,"আ":250,"ল":251,"হ":252,"া":253,"ਅ":254,"ਲ":255,"ਹ":256,"ਾ":257,"അ":258,"ള":259,"ഹ":260,"ാ":261,"ก":262,"ค":263,"ง":264,"ช":265,"ซ":266,"ญ":267,"ฐ":268,"ณ":269,"ด":270,"ต":271,"น":272,"บ":273,"ป":274,"พ":275,"ภ":276,"ม":277,"ย":278,"ร":279,"ล":280,"ว":281,"ศ":282,"ษ":283,"ส":284,"ห":285,"อ":286,"ฮ":287,"ะ":288,"า":289,"เ":290,"แ":291,"ไ":292,"ა":293,"ბ":294,"გ":295,"დ":296,"ე":297,"ვ":298,"ზ":299,"თ":300,"ი":301,"კ":302,"ლ":303,"მ":304,"ნ":305,"ო":306,"პ":307,"ჟ":308,"რ":309,"ს":310,"ტ":311,"უ":312,"ფ":313,"ქ":314,"ღ":315,"ყ":316,"შ":317,"ჩ":318,"ც":319,"ძ":320,"წ":321,"ჭ":322,"ხ":323,"ჯ":324,"ჰ":325,"ჱ":326,"ჲ":327,"ჳ":328,"ჴ":329,"ჵ":330,"ჶ":331,"ჷ":332,"ჸ":333,"ჹ":334,"ჺ":335,"჻":336,"ᄃ":337,"ᄅ":338,"ᄇ":339,"ᄋ":340,"ᄌ":341,"ᅡ":342,"ᅢ":343,"ᅦ":344,"ᅧ":345,"ᅩ":346,"ᅮ":347,"ᅵ":348,"ᆨ":349,"ᆫ":350,"ᆯ":351,"ᆸ":352,"ᆼ":353,"ᵻ":354,"‐":355,"‑":356,"–":357,"—":358,"―":359,"‘":360,"’":361,"“":362,"”":363,"„":364,"†":365,"‡":366,"•":367,"…":368,"′":369,"″":370,"⁄":371,"₣":372,"₤":373,"€":374,"₹":375,"⅓":376,"⅔":377,"→":378,"−":379,"≡":380,"≤":381,"①":382,"☉":383,"☫":384,"♀":385,"♭":386,"♯":387,"⚳":388,"ⴀ":389,"ⴂ":390,"ⴃ":391,"ⴈ":392,"ⴌ":393,"ⴕ":394,"ⴟ":395,"〈":396,"〉":397,"〜":398,"あ":399,"い":400,"う":401,"お":402,"か":403,"き":404,"く":405,"け":406,"こ":407,"さ":408,"し":409,"す":410,"せ":411,"た":412,"ち":413,"っ":414,"つ":415,"と":416,"な":417,"に":418,"の":419,"は":420,"ひ":421,"ふ":422,"ほ":423,"ま":424,"み":425,"め":426,"も":427,"ゃ":428,"ゆ":429,"ょ":430,"ら":431,"り":432,"る":433,"れ":434,"わ":435,"を":436,"ん":437,"ァ":438,"ア":439,"ィ":440,"イ":441,"ゥ":442,"ウ":443,"ェ":444,"エ":445,"ォ":446,"オ":447,"カ":448,"キ":449,"ク":450,"ケ":451,"コ":452,"サ":453,"シ":454,"ス":455,"セ":456,"タ":457,"チ":458,"ッ":459,"ツ":460,"テ":461,"ト":462,"ナ":463,"ニ":464,"ネ":465,"ノ":466,"ハ":467,"フ":468,"ヘ":469,"マ":470,"ミ":471,"ム":472,"モ":473,"ャ":474,"ュ":475,"ョ":476,"ラ":477,"リ":478,"ル":479,"レ":480,"ロ":481,"ン":482,"・":483,"ー":484,"一":485,"七":486,"下":487,"世":488,"丙":489,"中":490,"主":491,"乃":492,"之":493,"乙":494,"九":495,"二":496,"云":497,"人":498,"今":499,"付":500,"作":501,"侗":502,"依":503,"信":504,"傳":505,"儚":506,"充":507,"光":508,"全":509,"兵":510,"其":511,"具":512,"円":513,"再":514,"出":515,"判":516,"前":517,"剛":518,"劇":519,"劉":520,"動":521,"化":522,"北":523,"华":524,"厂":525,"去":526,"古":527,"可":528,"台":529,"史":530,"同":531,"名":532,"君":533,"吳":534,"周":535,"命":536,"和":537,"咲":538,"善":539,"四":540,"國":541,"園":542,"圣":543,"在":544,"坂":545,"堤":546,"場":547,"塘":548,"夕":549,"大":550,"天":551,"夫":552,"女":553,"妙":554,"姚":555,"子":556,"孟":557,"守":558,"安":559,"宋":560,"完":561,"宗":562,"宝":563,"宫":564,"寝":565,"寺":566,"小":567,"少":568,"尾":569,"山":570,"岳":571,"川":572,"州":573,"巳":574,"市":575,"師":576,"平":577,"广":578,"庆":579,"府":580,"座":581,"廬":582,"建":583,"式":584,"張":585,"彌":586,"彩":587,"彼":588,"後":589,"御":590,"德":591,"思":592,"愛":593,"憑":594,"憶":595,"應":596,"懷":597,"战":598,"戦":599,"扈":600,"技":601,"拉":602,"拳":603,"挑":604,"揺":605,"攻":606,"放":607,"政":608,"散":609,"斯":610,"方":611,"日":612,"旦":613,"旭":614,"昌":615,"明":616,"星":617,"春":618,"晋":619,"景":620,"曦":621,"月":622,"望":623,"未":624,"本":625,"李":626,"村":627,"杜":628,"束":629,"来":630,"林":631,"桜":632,"梶":633,"棘":634,"椎":635,"楊":636,"楚":637,"榮":638,"橘":639,"機":640,"正":641,"殻":642,"殿":643,"母":644,"水":645,"汉":646,"沂":647,"沙":648,"河":649,"泗":650,"波":651,"泣":652,"洪":653,"淹":654,"清":655,"湯":656,"漢":657,"澄":658,"澤":659,"火":660,"灯":661,"灵":662,"灼":663,"焼":664,"熱":665,"物":666,"狐":667,"狸":668,"玄":669,"王":670,"玩":671,"珂":672,"珙":673,"球":674,"理":675,"琦":676,"琪":677,"瓊":678,"生":679,"田":680,"畢":681,"番":682,"瘡":683,"白":684,"皮":685,"真":686,"砲":687,"礮":688,"祈":689,"神":690,"祠":691,"秋":692,"空":693,"立":694,"精":695,"約":696,"絵":697,"織":698,"義":699,"翠":700,"者":701,"耕":702,"肖":703,"胡":704,"膀":705,"臂":706,"興":707,"良":708,"花":709,"芳":710,"芽":711,"若":712,"英":713,"藕":714,"藥":715,"蘄":716,"蘇":717,"行":718,"裁":719,"規":720,"覺":721,"观":722,"解":723,"記":724,"誓":725,"誡":726,"誰":727,"謎":728,"许":729,"谭":730,"豪":731,"豫":732,"費":733,"贵":734,"赤":735,"趙":736,"足":737,"跡":738,"転":739,"辛":740,"逆":741,"遇":742,"運":743,"過":744,"遠":745,"選":746,"邦":747,"邱":748,"部":749,"郭":750,"都":751,"酈":752,"里":753,"野":754,"金":755,"銃":756,"鋼":757,"錄":758,"錡":759,"鍵":760,"鐵":761,"钱":762,"铁":763,"關":764,"防":765,"阿":766,"陈":767,"陳":768,"陽":769,"隊":770,"階":771,"集":772,"雪":773,"雲":774,"霖":775,"霹":776,"靂":777,"韓":778,"願":779,"顯":780,"颜":781,"马":782,"高":783,"龍":784,"ﷲ":785,"ﻋ":786,"/":787,"3":788,"~":789,"##a":790,"##r":791,"##i":792,"##n":793,"##e":794,"##g":795,"##v":796,"##u":797,"##l":798,"##s":799,"##m":800,"##p":801,"##f":802,"##c":803,"##t":804,"##o":805,"##x":806,"##h":807,"##b":808,"##k":809,"##2":810,"##9":811,"##y":812,"##d":813,"##3":814,"##1":815,"##ø":816,"##w":817,"##j":818,"##q":819,"##แ":820,"##ก":821,"##z":822,"##7":823,"##6":824,"##4":825,"##0":826,"##5":827,"##γ":828,"##8":829,"##ხ":830,"##ე":831,"##დ":832,"##ა":833,"##რ":834,"##ი":835,"##ł":836,"##и":837,"##ш":838,"##н":839,"##е":840,"##в":841,"##ц":842,"##ь":843,"##к":844,"##י":845,"##פ":846,"##ה":847,"##о":848,"##л":849,"##с":850,"##а":851,"##т":852,"##ы":853,"##ツ":854,"##ネ":855,"##æ":856,"##う":857,"##き":858,"##な":859,"##ロ":860,"##イ":861,"##ト":862,"##キ":863,"##ウ":864,"##ァ":865,"##ル":866,"##ュ":867,"##リ":868,"##ア":869,"##3":870,"##ʻ":871,"##ρ":872,"##ε":873,"##ξ":874,"##れ":875,"##い":876,"##ذ":877,"##ي":878,"##ن":879,"##ة":880,"##ɛ":881,"##ə":882,"##ˈ":883,"##ʊ":884,"##π":885,"##α":886,"##τ":887,"##ι":888,"##κ":889,"##ο":890,"##ς":891,"##ク":892,"##サ":893,"##ス":894,"##か":895,"##た":896,"##η":897,"##μ":898,"##ാ":899,"##ഹ":900,"##া":901,"##হ":902,"##ɒ":903,"##υ":904,"##θ":905,"##ß":906,"##く":907,"##º":908,"##ლ":909,"##მ":910,"##წ":911,"##ფ":912,"##ा":913,"##ई":914,"##ン":915,"##タ":916,"##ム":917,"##チ":918,"##ェ":919,"##ー":920,"##λ":921,"##χ":922,"##ქ":923,"##ს":924,"##ნ":925,"##ː":926,"##ν":927,"##ω":928,"##س":929,"##ت":930,"##ا":931,"##ッ":932,"##ラ":933,"##თ":934,"##ვ":935,"##つ":936,"##უ":937,"##ც":938,"##ล":939,"##ย":940,"##า":941,"##ณ":942,"##ม":943,"##ต":944,"##ร":945,"##я":946,"##め":947,"##の":948,"##ら":949,"##ı":950,"##ب":951,"##テ":952,"##ィ":953,"##ハ":954,"##р":955,"##ו":956,"##є":957,"##ო":958,"##ɪ":959,"##ق":960,"##د":961,"##ش":962,"##و":963,"##ء":964,"##đ":965,"##ɽ":966,"##ʔ":967,"##ナ":968,"##わ":969,"##お":970,"##ܠ":971,"##ܗ":972,"##ܐ":973,"##ɾ":974,"##ʲ":975,"##レ":976,"##ɑ":977,"##ɔ":978,"##ɡ":979,"##ʃ":980,"##っ":981,"##σ":982,"##п":983,"##д":984,"##ਲ":985,"##ਹ":986,"##ʿ":987,"##カ":988,"##ण":989,"##स":990,"##न":991,"##す":992,"##ჳ":993,"##ニ":994,"##ל":995,"##ם":996,"##β":997,"##マ":998,"##δ":999,"##क":1000,"##ð":1001,"##こ":1002,"##と":1003,"##あ":1004,"##ん":1005,"##ტ":1006,"##ჲ":1007,"##シ":1008,"##ョ":1009,"##ع":1010,"##อ":1011,"##ว":1012,"##र":1013,"##द":1014,"##ซ":1015,"##у":1016,"##フ":1017,"##व":1018,"##ह":1019,"##ि":1020,"##ჱ":1021,"##ฮ":1022,"##พ":1023,"##ᵻ":1024,"##м":1025,"##ل":1026,"##ر":1027,"##ษ":1028,"##ฐ":1029,"##ノ":1030,"##ב":1031,"##ת":1032,"##א":1033,"##ค":1034,"##خ":1035,"##は":1036,"##り":1037,"##ه":1038,"##ภ":1039,"##ด":1040,"##ち":1041,"##ゃ":1042,"##ح":1043,"##ป":1044,"##ف":1045,"##ᅡ":1046,"##ᆯ":1047,"##ᄅ":1048,"##ᄇ":1049,"##ᅮ":1050,"##ᄃ":1051,"##ᅢ":1052,"##し":1053,"##ъ":1054,"##з":1055,"##ث":1056,"##م":1057,"##も":1058,"##ャ":1059,"##モ":1060,"##გ":1061,"##ɐ":1062,"##ʁ":1063,"##נ":1064,"##る":1065,"##ま":1066,"##ょ":1067,"##に":1068,"##г":1069,"##ɳ":1070,"##ɫ":1071,"##х":1072,"##ォ":1073,"##ك":1074,"##ף":1075,"##ミ":1076,"##ש":1077,"##מ":1078,"##צ":1079,"##を":1080,"##ง":1081,"##ไ":1082,"##ช":1083,"##せ":1084,"##ゆ":1085,"##ю":1086,"##ะ":1087,"##น":1088,"##ग":1089,"##œ":1090,"##ひ":1091,"##ˌ":1092,"##み":1093,"##ص":1094,"##ס":1095,"##ള":1096,"##ʒ":1097,"##セ":1098,"##オ":1099,"##ܕ":1100,"##ܝ":1101,"##ܢ":1102,"##ܬ":1103,"##ल":1104,"##ɜ":1105,"##ゥ":1106,"##ψ":1107,"##ן":1108,"##ר":1109,"##ਾ":1110,"##เ":1111,"##ল":1112,"##ж":1113,"##ᅦ":1114,"##ᅩ":1115,"##ᆨ":1116,"##ᅵ":1117,"##ᆸ":1118,"##ᅧ":1119,"##ᆼ":1120,"##ᄋ":1121,"##ᆫ":1122,"##エ":1123}}}