bert24_32k_tok_llama2 / added_tokens.json
bclavie's picture
Upload tokenizer
c612297 verified
raw
history blame
18.3 kB
{
"[CLS]": 32001,
"[MASK]": 32004,
"[PAD]": 32003,
"[SEP]": 32002,
"[UNK]": 32000,
"[unused0]": 32005,
"[unused100]": 32105,
"[unused101]": 32106,
"[unused102]": 32107,
"[unused103]": 32108,
"[unused104]": 32109,
"[unused105]": 32110,
"[unused106]": 32111,
"[unused107]": 32112,
"[unused108]": 32113,
"[unused109]": 32114,
"[unused10]": 32015,
"[unused110]": 32115,
"[unused111]": 32116,
"[unused112]": 32117,
"[unused113]": 32118,
"[unused114]": 32119,
"[unused115]": 32120,
"[unused116]": 32121,
"[unused117]": 32122,
"[unused118]": 32123,
"[unused119]": 32124,
"[unused11]": 32016,
"[unused120]": 32125,
"[unused121]": 32126,
"[unused122]": 32127,
"[unused123]": 32128,
"[unused124]": 32129,
"[unused125]": 32130,
"[unused126]": 32131,
"[unused127]": 32132,
"[unused128]": 32133,
"[unused129]": 32134,
"[unused12]": 32017,
"[unused130]": 32135,
"[unused131]": 32136,
"[unused132]": 32137,
"[unused133]": 32138,
"[unused134]": 32139,
"[unused135]": 32140,
"[unused136]": 32141,
"[unused137]": 32142,
"[unused138]": 32143,
"[unused139]": 32144,
"[unused13]": 32018,
"[unused140]": 32145,
"[unused141]": 32146,
"[unused142]": 32147,
"[unused143]": 32148,
"[unused144]": 32149,
"[unused145]": 32150,
"[unused146]": 32151,
"[unused147]": 32152,
"[unused148]": 32153,
"[unused149]": 32154,
"[unused14]": 32019,
"[unused150]": 32155,
"[unused151]": 32156,
"[unused152]": 32157,
"[unused153]": 32158,
"[unused154]": 32159,
"[unused155]": 32160,
"[unused156]": 32161,
"[unused157]": 32162,
"[unused158]": 32163,
"[unused159]": 32164,
"[unused15]": 32020,
"[unused160]": 32165,
"[unused161]": 32166,
"[unused162]": 32167,
"[unused163]": 32168,
"[unused164]": 32169,
"[unused165]": 32170,
"[unused166]": 32171,
"[unused167]": 32172,
"[unused168]": 32173,
"[unused169]": 32174,
"[unused16]": 32021,
"[unused170]": 32175,
"[unused171]": 32176,
"[unused172]": 32177,
"[unused173]": 32178,
"[unused174]": 32179,
"[unused175]": 32180,
"[unused176]": 32181,
"[unused177]": 32182,
"[unused178]": 32183,
"[unused179]": 32184,
"[unused17]": 32022,
"[unused180]": 32185,
"[unused181]": 32186,
"[unused182]": 32187,
"[unused183]": 32188,
"[unused184]": 32189,
"[unused185]": 32190,
"[unused186]": 32191,
"[unused187]": 32192,
"[unused188]": 32193,
"[unused189]": 32194,
"[unused18]": 32023,
"[unused190]": 32195,
"[unused191]": 32196,
"[unused192]": 32197,
"[unused193]": 32198,
"[unused194]": 32199,
"[unused195]": 32200,
"[unused196]": 32201,
"[unused197]": 32202,
"[unused198]": 32203,
"[unused199]": 32204,
"[unused19]": 32024,
"[unused1]": 32006,
"[unused200]": 32205,
"[unused201]": 32206,
"[unused202]": 32207,
"[unused203]": 32208,
"[unused204]": 32209,
"[unused205]": 32210,
"[unused206]": 32211,
"[unused207]": 32212,
"[unused208]": 32213,
"[unused209]": 32214,
"[unused20]": 32025,
"[unused210]": 32215,
"[unused211]": 32216,
"[unused212]": 32217,
"[unused213]": 32218,
"[unused214]": 32219,
"[unused215]": 32220,
"[unused216]": 32221,
"[unused217]": 32222,
"[unused218]": 32223,
"[unused219]": 32224,
"[unused21]": 32026,
"[unused220]": 32225,
"[unused221]": 32226,
"[unused222]": 32227,
"[unused223]": 32228,
"[unused224]": 32229,
"[unused225]": 32230,
"[unused226]": 32231,
"[unused227]": 32232,
"[unused228]": 32233,
"[unused229]": 32234,
"[unused22]": 32027,
"[unused230]": 32235,
"[unused231]": 32236,
"[unused232]": 32237,
"[unused233]": 32238,
"[unused234]": 32239,
"[unused235]": 32240,
"[unused236]": 32241,
"[unused237]": 32242,
"[unused238]": 32243,
"[unused239]": 32244,
"[unused23]": 32028,
"[unused240]": 32245,
"[unused241]": 32246,
"[unused242]": 32247,
"[unused243]": 32248,
"[unused244]": 32249,
"[unused245]": 32250,
"[unused246]": 32251,
"[unused247]": 32252,
"[unused248]": 32253,
"[unused249]": 32254,
"[unused24]": 32029,
"[unused250]": 32255,
"[unused251]": 32256,
"[unused252]": 32257,
"[unused253]": 32258,
"[unused254]": 32259,
"[unused255]": 32260,
"[unused256]": 32261,
"[unused257]": 32262,
"[unused258]": 32263,
"[unused259]": 32264,
"[unused25]": 32030,
"[unused260]": 32265,
"[unused261]": 32266,
"[unused262]": 32267,
"[unused263]": 32268,
"[unused264]": 32269,
"[unused265]": 32270,
"[unused266]": 32271,
"[unused267]": 32272,
"[unused268]": 32273,
"[unused269]": 32274,
"[unused26]": 32031,
"[unused270]": 32275,
"[unused271]": 32276,
"[unused272]": 32277,
"[unused273]": 32278,
"[unused274]": 32279,
"[unused275]": 32280,
"[unused276]": 32281,
"[unused277]": 32282,
"[unused278]": 32283,
"[unused279]": 32284,
"[unused27]": 32032,
"[unused280]": 32285,
"[unused281]": 32286,
"[unused282]": 32287,
"[unused283]": 32288,
"[unused284]": 32289,
"[unused285]": 32290,
"[unused286]": 32291,
"[unused287]": 32292,
"[unused288]": 32293,
"[unused289]": 32294,
"[unused28]": 32033,
"[unused290]": 32295,
"[unused291]": 32296,
"[unused292]": 32297,
"[unused293]": 32298,
"[unused294]": 32299,
"[unused295]": 32300,
"[unused296]": 32301,
"[unused297]": 32302,
"[unused298]": 32303,
"[unused299]": 32304,
"[unused29]": 32034,
"[unused2]": 32007,
"[unused300]": 32305,
"[unused301]": 32306,
"[unused302]": 32307,
"[unused303]": 32308,
"[unused304]": 32309,
"[unused305]": 32310,
"[unused306]": 32311,
"[unused307]": 32312,
"[unused308]": 32313,
"[unused309]": 32314,
"[unused30]": 32035,
"[unused310]": 32315,
"[unused311]": 32316,
"[unused312]": 32317,
"[unused313]": 32318,
"[unused314]": 32319,
"[unused315]": 32320,
"[unused316]": 32321,
"[unused317]": 32322,
"[unused318]": 32323,
"[unused319]": 32324,
"[unused31]": 32036,
"[unused320]": 32325,
"[unused321]": 32326,
"[unused322]": 32327,
"[unused323]": 32328,
"[unused324]": 32329,
"[unused325]": 32330,
"[unused326]": 32331,
"[unused327]": 32332,
"[unused328]": 32333,
"[unused329]": 32334,
"[unused32]": 32037,
"[unused330]": 32335,
"[unused331]": 32336,
"[unused332]": 32337,
"[unused333]": 32338,
"[unused334]": 32339,
"[unused335]": 32340,
"[unused336]": 32341,
"[unused337]": 32342,
"[unused338]": 32343,
"[unused339]": 32344,
"[unused33]": 32038,
"[unused340]": 32345,
"[unused341]": 32346,
"[unused342]": 32347,
"[unused343]": 32348,
"[unused344]": 32349,
"[unused345]": 32350,
"[unused346]": 32351,
"[unused347]": 32352,
"[unused348]": 32353,
"[unused349]": 32354,
"[unused34]": 32039,
"[unused350]": 32355,
"[unused351]": 32356,
"[unused352]": 32357,
"[unused353]": 32358,
"[unused354]": 32359,
"[unused355]": 32360,
"[unused356]": 32361,
"[unused357]": 32362,
"[unused358]": 32363,
"[unused359]": 32364,
"[unused35]": 32040,
"[unused360]": 32365,
"[unused361]": 32366,
"[unused362]": 32367,
"[unused363]": 32368,
"[unused364]": 32369,
"[unused365]": 32370,
"[unused366]": 32371,
"[unused367]": 32372,
"[unused368]": 32373,
"[unused369]": 32374,
"[unused36]": 32041,
"[unused370]": 32375,
"[unused371]": 32376,
"[unused372]": 32377,
"[unused373]": 32378,
"[unused374]": 32379,
"[unused375]": 32380,
"[unused376]": 32381,
"[unused377]": 32382,
"[unused378]": 32383,
"[unused379]": 32384,
"[unused37]": 32042,
"[unused380]": 32385,
"[unused381]": 32386,
"[unused382]": 32387,
"[unused383]": 32388,
"[unused384]": 32389,
"[unused385]": 32390,
"[unused386]": 32391,
"[unused387]": 32392,
"[unused388]": 32393,
"[unused389]": 32394,
"[unused38]": 32043,
"[unused390]": 32395,
"[unused391]": 32396,
"[unused392]": 32397,
"[unused393]": 32398,
"[unused394]": 32399,
"[unused395]": 32400,
"[unused396]": 32401,
"[unused397]": 32402,
"[unused398]": 32403,
"[unused399]": 32404,
"[unused39]": 32044,
"[unused3]": 32008,
"[unused400]": 32405,
"[unused401]": 32406,
"[unused402]": 32407,
"[unused403]": 32408,
"[unused404]": 32409,
"[unused405]": 32410,
"[unused406]": 32411,
"[unused407]": 32412,
"[unused408]": 32413,
"[unused409]": 32414,
"[unused40]": 32045,
"[unused410]": 32415,
"[unused411]": 32416,
"[unused412]": 32417,
"[unused413]": 32418,
"[unused414]": 32419,
"[unused415]": 32420,
"[unused416]": 32421,
"[unused417]": 32422,
"[unused418]": 32423,
"[unused419]": 32424,
"[unused41]": 32046,
"[unused420]": 32425,
"[unused421]": 32426,
"[unused422]": 32427,
"[unused423]": 32428,
"[unused424]": 32429,
"[unused425]": 32430,
"[unused426]": 32431,
"[unused427]": 32432,
"[unused428]": 32433,
"[unused429]": 32434,
"[unused42]": 32047,
"[unused430]": 32435,
"[unused431]": 32436,
"[unused432]": 32437,
"[unused433]": 32438,
"[unused434]": 32439,
"[unused435]": 32440,
"[unused436]": 32441,
"[unused437]": 32442,
"[unused438]": 32443,
"[unused439]": 32444,
"[unused43]": 32048,
"[unused440]": 32445,
"[unused441]": 32446,
"[unused442]": 32447,
"[unused443]": 32448,
"[unused444]": 32449,
"[unused445]": 32450,
"[unused446]": 32451,
"[unused447]": 32452,
"[unused448]": 32453,
"[unused449]": 32454,
"[unused44]": 32049,
"[unused450]": 32455,
"[unused451]": 32456,
"[unused452]": 32457,
"[unused453]": 32458,
"[unused454]": 32459,
"[unused455]": 32460,
"[unused456]": 32461,
"[unused457]": 32462,
"[unused458]": 32463,
"[unused459]": 32464,
"[unused45]": 32050,
"[unused460]": 32465,
"[unused461]": 32466,
"[unused462]": 32467,
"[unused463]": 32468,
"[unused464]": 32469,
"[unused465]": 32470,
"[unused466]": 32471,
"[unused467]": 32472,
"[unused468]": 32473,
"[unused469]": 32474,
"[unused46]": 32051,
"[unused470]": 32475,
"[unused471]": 32476,
"[unused472]": 32477,
"[unused473]": 32478,
"[unused474]": 32479,
"[unused475]": 32480,
"[unused476]": 32481,
"[unused477]": 32482,
"[unused478]": 32483,
"[unused479]": 32484,
"[unused47]": 32052,
"[unused480]": 32485,
"[unused481]": 32486,
"[unused482]": 32487,
"[unused483]": 32488,
"[unused484]": 32489,
"[unused485]": 32490,
"[unused486]": 32491,
"[unused487]": 32492,
"[unused488]": 32493,
"[unused489]": 32494,
"[unused48]": 32053,
"[unused490]": 32495,
"[unused491]": 32496,
"[unused492]": 32497,
"[unused493]": 32498,
"[unused494]": 32499,
"[unused495]": 32500,
"[unused496]": 32501,
"[unused497]": 32502,
"[unused498]": 32503,
"[unused499]": 32504,
"[unused49]": 32054,
"[unused4]": 32009,
"[unused500]": 32505,
"[unused501]": 32506,
"[unused502]": 32507,
"[unused503]": 32508,
"[unused504]": 32509,
"[unused505]": 32510,
"[unused506]": 32511,
"[unused507]": 32512,
"[unused508]": 32513,
"[unused509]": 32514,
"[unused50]": 32055,
"[unused510]": 32515,
"[unused511]": 32516,
"[unused512]": 32517,
"[unused513]": 32518,
"[unused514]": 32519,
"[unused515]": 32520,
"[unused516]": 32521,
"[unused517]": 32522,
"[unused518]": 32523,
"[unused519]": 32524,
"[unused51]": 32056,
"[unused520]": 32525,
"[unused521]": 32526,
"[unused522]": 32527,
"[unused523]": 32528,
"[unused524]": 32529,
"[unused525]": 32530,
"[unused526]": 32531,
"[unused527]": 32532,
"[unused528]": 32533,
"[unused529]": 32534,
"[unused52]": 32057,
"[unused530]": 32535,
"[unused531]": 32536,
"[unused532]": 32537,
"[unused533]": 32538,
"[unused534]": 32539,
"[unused535]": 32540,
"[unused536]": 32541,
"[unused537]": 32542,
"[unused538]": 32543,
"[unused539]": 32544,
"[unused53]": 32058,
"[unused540]": 32545,
"[unused541]": 32546,
"[unused542]": 32547,
"[unused543]": 32548,
"[unused544]": 32549,
"[unused545]": 32550,
"[unused546]": 32551,
"[unused547]": 32552,
"[unused548]": 32553,
"[unused549]": 32554,
"[unused54]": 32059,
"[unused550]": 32555,
"[unused551]": 32556,
"[unused552]": 32557,
"[unused553]": 32558,
"[unused554]": 32559,
"[unused555]": 32560,
"[unused556]": 32561,
"[unused557]": 32562,
"[unused558]": 32563,
"[unused559]": 32564,
"[unused55]": 32060,
"[unused560]": 32565,
"[unused561]": 32566,
"[unused562]": 32567,
"[unused563]": 32568,
"[unused564]": 32569,
"[unused565]": 32570,
"[unused566]": 32571,
"[unused567]": 32572,
"[unused568]": 32573,
"[unused569]": 32574,
"[unused56]": 32061,
"[unused570]": 32575,
"[unused571]": 32576,
"[unused572]": 32577,
"[unused573]": 32578,
"[unused574]": 32579,
"[unused575]": 32580,
"[unused576]": 32581,
"[unused577]": 32582,
"[unused578]": 32583,
"[unused579]": 32584,
"[unused57]": 32062,
"[unused580]": 32585,
"[unused581]": 32586,
"[unused582]": 32587,
"[unused583]": 32588,
"[unused584]": 32589,
"[unused585]": 32590,
"[unused586]": 32591,
"[unused587]": 32592,
"[unused588]": 32593,
"[unused589]": 32594,
"[unused58]": 32063,
"[unused590]": 32595,
"[unused591]": 32596,
"[unused592]": 32597,
"[unused593]": 32598,
"[unused594]": 32599,
"[unused595]": 32600,
"[unused596]": 32601,
"[unused597]": 32602,
"[unused598]": 32603,
"[unused599]": 32604,
"[unused59]": 32064,
"[unused5]": 32010,
"[unused600]": 32605,
"[unused601]": 32606,
"[unused602]": 32607,
"[unused603]": 32608,
"[unused604]": 32609,
"[unused605]": 32610,
"[unused606]": 32611,
"[unused607]": 32612,
"[unused608]": 32613,
"[unused609]": 32614,
"[unused60]": 32065,
"[unused610]": 32615,
"[unused611]": 32616,
"[unused612]": 32617,
"[unused613]": 32618,
"[unused614]": 32619,
"[unused615]": 32620,
"[unused616]": 32621,
"[unused617]": 32622,
"[unused618]": 32623,
"[unused619]": 32624,
"[unused61]": 32066,
"[unused620]": 32625,
"[unused621]": 32626,
"[unused622]": 32627,
"[unused623]": 32628,
"[unused624]": 32629,
"[unused625]": 32630,
"[unused626]": 32631,
"[unused627]": 32632,
"[unused628]": 32633,
"[unused629]": 32634,
"[unused62]": 32067,
"[unused630]": 32635,
"[unused631]": 32636,
"[unused632]": 32637,
"[unused633]": 32638,
"[unused634]": 32639,
"[unused635]": 32640,
"[unused636]": 32641,
"[unused637]": 32642,
"[unused638]": 32643,
"[unused639]": 32644,
"[unused63]": 32068,
"[unused640]": 32645,
"[unused641]": 32646,
"[unused642]": 32647,
"[unused643]": 32648,
"[unused644]": 32649,
"[unused645]": 32650,
"[unused646]": 32651,
"[unused647]": 32652,
"[unused648]": 32653,
"[unused649]": 32654,
"[unused64]": 32069,
"[unused650]": 32655,
"[unused651]": 32656,
"[unused652]": 32657,
"[unused653]": 32658,
"[unused654]": 32659,
"[unused655]": 32660,
"[unused656]": 32661,
"[unused657]": 32662,
"[unused658]": 32663,
"[unused659]": 32664,
"[unused65]": 32070,
"[unused660]": 32665,
"[unused661]": 32666,
"[unused662]": 32667,
"[unused663]": 32668,
"[unused664]": 32669,
"[unused665]": 32670,
"[unused666]": 32671,
"[unused667]": 32672,
"[unused668]": 32673,
"[unused669]": 32674,
"[unused66]": 32071,
"[unused670]": 32675,
"[unused671]": 32676,
"[unused672]": 32677,
"[unused673]": 32678,
"[unused674]": 32679,
"[unused675]": 32680,
"[unused676]": 32681,
"[unused677]": 32682,
"[unused678]": 32683,
"[unused679]": 32684,
"[unused67]": 32072,
"[unused680]": 32685,
"[unused681]": 32686,
"[unused682]": 32687,
"[unused683]": 32688,
"[unused684]": 32689,
"[unused685]": 32690,
"[unused686]": 32691,
"[unused687]": 32692,
"[unused688]": 32693,
"[unused689]": 32694,
"[unused68]": 32073,
"[unused690]": 32695,
"[unused691]": 32696,
"[unused692]": 32697,
"[unused693]": 32698,
"[unused694]": 32699,
"[unused695]": 32700,
"[unused696]": 32701,
"[unused697]": 32702,
"[unused698]": 32703,
"[unused699]": 32704,
"[unused69]": 32074,
"[unused6]": 32011,
"[unused700]": 32705,
"[unused701]": 32706,
"[unused702]": 32707,
"[unused703]": 32708,
"[unused704]": 32709,
"[unused705]": 32710,
"[unused706]": 32711,
"[unused707]": 32712,
"[unused708]": 32713,
"[unused709]": 32714,
"[unused70]": 32075,
"[unused710]": 32715,
"[unused711]": 32716,
"[unused712]": 32717,
"[unused713]": 32718,
"[unused714]": 32719,
"[unused715]": 32720,
"[unused716]": 32721,
"[unused717]": 32722,
"[unused718]": 32723,
"[unused719]": 32724,
"[unused71]": 32076,
"[unused720]": 32725,
"[unused721]": 32726,
"[unused722]": 32727,
"[unused723]": 32728,
"[unused724]": 32729,
"[unused725]": 32730,
"[unused726]": 32731,
"[unused727]": 32732,
"[unused728]": 32733,
"[unused729]": 32734,
"[unused72]": 32077,
"[unused730]": 32735,
"[unused731]": 32736,
"[unused732]": 32737,
"[unused733]": 32738,
"[unused734]": 32739,
"[unused735]": 32740,
"[unused736]": 32741,
"[unused737]": 32742,
"[unused738]": 32743,
"[unused739]": 32744,
"[unused73]": 32078,
"[unused740]": 32745,
"[unused741]": 32746,
"[unused742]": 32747,
"[unused743]": 32748,
"[unused744]": 32749,
"[unused745]": 32750,
"[unused746]": 32751,
"[unused747]": 32752,
"[unused748]": 32753,
"[unused749]": 32754,
"[unused74]": 32079,
"[unused750]": 32755,
"[unused751]": 32756,
"[unused752]": 32757,
"[unused753]": 32758,
"[unused754]": 32759,
"[unused755]": 32760,
"[unused756]": 32761,
"[unused757]": 32762,
"[unused758]": 32763,
"[unused759]": 32764,
"[unused75]": 32080,
"[unused760]": 32765,
"[unused761]": 32766,
"[unused762]": 32767,
"[unused76]": 32081,
"[unused77]": 32082,
"[unused78]": 32083,
"[unused79]": 32084,
"[unused7]": 32012,
"[unused80]": 32085,
"[unused81]": 32086,
"[unused82]": 32087,
"[unused83]": 32088,
"[unused84]": 32089,
"[unused85]": 32090,
"[unused86]": 32091,
"[unused87]": 32092,
"[unused88]": 32093,
"[unused89]": 32094,
"[unused8]": 32013,
"[unused90]": 32095,
"[unused91]": 32096,
"[unused92]": 32097,
"[unused93]": 32098,
"[unused94]": 32099,
"[unused95]": 32100,
"[unused96]": 32101,
"[unused97]": 32102,
"[unused98]": 32103,
"[unused99]": 32104,
"[unused9]": 32014
}