brackets-nested / tokenizer.json
Mlxa's picture
Upload tokenizer
17baa0f
{
"version": "1.0",
"truncation": null,
"padding": null,
"added_tokens": [
{
"id": 500,
"content": "[UNK]",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 501,
"content": "[PAD]",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
}
],
"normalizer": null,
"pre_tokenizer": {
"type": "WhitespaceSplit"
},
"post_processor": null,
"decoder": null,
"model": {
"type": "WordLevel",
"vocab": {
"<1": 0,
"1>": 1,
"<2": 2,
"2>": 3,
"<3": 4,
"3>": 5,
"<4": 6,
"4>": 7,
"<5": 8,
"5>": 9,
"<6": 10,
"6>": 11,
"<7": 12,
"7>": 13,
"<8": 14,
"8>": 15,
"<9": 16,
"9>": 17,
"<10": 18,
"10>": 19,
"<11": 20,
"11>": 21,
"<12": 22,
"12>": 23,
"<13": 24,
"13>": 25,
"<14": 26,
"14>": 27,
"<15": 28,
"15>": 29,
"<16": 30,
"16>": 31,
"<17": 32,
"17>": 33,
"<18": 34,
"18>": 35,
"<19": 36,
"19>": 37,
"<20": 38,
"20>": 39,
"<21": 40,
"21>": 41,
"<22": 42,
"22>": 43,
"<23": 44,
"23>": 45,
"<24": 46,
"24>": 47,
"<25": 48,
"25>": 49,
"<26": 50,
"26>": 51,
"<27": 52,
"27>": 53,
"<28": 54,
"28>": 55,
"<29": 56,
"29>": 57,
"<30": 58,
"30>": 59,
"<31": 60,
"31>": 61,
"<32": 62,
"32>": 63,
"<33": 64,
"33>": 65,
"<34": 66,
"34>": 67,
"<35": 68,
"35>": 69,
"<36": 70,
"36>": 71,
"<37": 72,
"37>": 73,
"<38": 74,
"38>": 75,
"<39": 76,
"39>": 77,
"<40": 78,
"40>": 79,
"<41": 80,
"41>": 81,
"<42": 82,
"42>": 83,
"<43": 84,
"43>": 85,
"<44": 86,
"44>": 87,
"<45": 88,
"45>": 89,
"<46": 90,
"46>": 91,
"<47": 92,
"47>": 93,
"<48": 94,
"48>": 95,
"<49": 96,
"49>": 97,
"<50": 98,
"50>": 99,
"<51": 100,
"51>": 101,
"<52": 102,
"52>": 103,
"<53": 104,
"53>": 105,
"<54": 106,
"54>": 107,
"<55": 108,
"55>": 109,
"<56": 110,
"56>": 111,
"<57": 112,
"57>": 113,
"<58": 114,
"58>": 115,
"<59": 116,
"59>": 117,
"<60": 118,
"60>": 119,
"<61": 120,
"61>": 121,
"<62": 122,
"62>": 123,
"<63": 124,
"63>": 125,
"<64": 126,
"64>": 127,
"<65": 128,
"65>": 129,
"<66": 130,
"66>": 131,
"<67": 132,
"67>": 133,
"<68": 134,
"68>": 135,
"<69": 136,
"69>": 137,
"<70": 138,
"70>": 139,
"<71": 140,
"71>": 141,
"<72": 142,
"72>": 143,
"<73": 144,
"73>": 145,
"<74": 146,
"74>": 147,
"<75": 148,
"75>": 149,
"<76": 150,
"76>": 151,
"<77": 152,
"77>": 153,
"<78": 154,
"78>": 155,
"<79": 156,
"79>": 157,
"<80": 158,
"80>": 159,
"<81": 160,
"81>": 161,
"<82": 162,
"82>": 163,
"<83": 164,
"83>": 165,
"<84": 166,
"84>": 167,
"<85": 168,
"85>": 169,
"<86": 170,
"86>": 171,
"<87": 172,
"87>": 173,
"<88": 174,
"88>": 175,
"<89": 176,
"89>": 177,
"<90": 178,
"90>": 179,
"<91": 180,
"91>": 181,
"<92": 182,
"92>": 183,
"<93": 184,
"93>": 185,
"<94": 186,
"94>": 187,
"<95": 188,
"95>": 189,
"<96": 190,
"96>": 191,
"<97": 192,
"97>": 193,
"<98": 194,
"98>": 195,
"<99": 196,
"99>": 197,
"<100": 198,
"100>": 199,
"<101": 200,
"101>": 201,
"<102": 202,
"102>": 203,
"<103": 204,
"103>": 205,
"<104": 206,
"104>": 207,
"<105": 208,
"105>": 209,
"<106": 210,
"106>": 211,
"<107": 212,
"107>": 213,
"<108": 214,
"108>": 215,
"<109": 216,
"109>": 217,
"<110": 218,
"110>": 219,
"<111": 220,
"111>": 221,
"<112": 222,
"112>": 223,
"<113": 224,
"113>": 225,
"<114": 226,
"114>": 227,
"<115": 228,
"115>": 229,
"<116": 230,
"116>": 231,
"<117": 232,
"117>": 233,
"<118": 234,
"118>": 235,
"<119": 236,
"119>": 237,
"<120": 238,
"120>": 239,
"<121": 240,
"121>": 241,
"<122": 242,
"122>": 243,
"<123": 244,
"123>": 245,
"<124": 246,
"124>": 247,
"<125": 248,
"125>": 249,
"<126": 250,
"126>": 251,
"<127": 252,
"127>": 253,
"<128": 254,
"128>": 255,
"<129": 256,
"129>": 257,
"<130": 258,
"130>": 259,
"<131": 260,
"131>": 261,
"<132": 262,
"132>": 263,
"<133": 264,
"133>": 265,
"<134": 266,
"134>": 267,
"<135": 268,
"135>": 269,
"<136": 270,
"136>": 271,
"<137": 272,
"137>": 273,
"<138": 274,
"138>": 275,
"<139": 276,
"139>": 277,
"<140": 278,
"140>": 279,
"<141": 280,
"141>": 281,
"<142": 282,
"142>": 283,
"<143": 284,
"143>": 285,
"<144": 286,
"144>": 287,
"<145": 288,
"145>": 289,
"<146": 290,
"146>": 291,
"<147": 292,
"147>": 293,
"<148": 294,
"148>": 295,
"<149": 296,
"149>": 297,
"<150": 298,
"150>": 299,
"<151": 300,
"151>": 301,
"<152": 302,
"152>": 303,
"<153": 304,
"153>": 305,
"<154": 306,
"154>": 307,
"<155": 308,
"155>": 309,
"<156": 310,
"156>": 311,
"<157": 312,
"157>": 313,
"<158": 314,
"158>": 315,
"<159": 316,
"159>": 317,
"<160": 318,
"160>": 319,
"<161": 320,
"161>": 321,
"<162": 322,
"162>": 323,
"<163": 324,
"163>": 325,
"<164": 326,
"164>": 327,
"<165": 328,
"165>": 329,
"<166": 330,
"166>": 331,
"<167": 332,
"167>": 333,
"<168": 334,
"168>": 335,
"<169": 336,
"169>": 337,
"<170": 338,
"170>": 339,
"<171": 340,
"171>": 341,
"<172": 342,
"172>": 343,
"<173": 344,
"173>": 345,
"<174": 346,
"174>": 347,
"<175": 348,
"175>": 349,
"<176": 350,
"176>": 351,
"<177": 352,
"177>": 353,
"<178": 354,
"178>": 355,
"<179": 356,
"179>": 357,
"<180": 358,
"180>": 359,
"<181": 360,
"181>": 361,
"<182": 362,
"182>": 363,
"<183": 364,
"183>": 365,
"<184": 366,
"184>": 367,
"<185": 368,
"185>": 369,
"<186": 370,
"186>": 371,
"<187": 372,
"187>": 373,
"<188": 374,
"188>": 375,
"<189": 376,
"189>": 377,
"<190": 378,
"190>": 379,
"<191": 380,
"191>": 381,
"<192": 382,
"192>": 383,
"<193": 384,
"193>": 385,
"<194": 386,
"194>": 387,
"<195": 388,
"195>": 389,
"<196": 390,
"196>": 391,
"<197": 392,
"197>": 393,
"<198": 394,
"198>": 395,
"<199": 396,
"199>": 397,
"<200": 398,
"200>": 399,
"<201": 400,
"201>": 401,
"<202": 402,
"202>": 403,
"<203": 404,
"203>": 405,
"<204": 406,
"204>": 407,
"<205": 408,
"205>": 409,
"<206": 410,
"206>": 411,
"<207": 412,
"207>": 413,
"<208": 414,
"208>": 415,
"<209": 416,
"209>": 417,
"<210": 418,
"210>": 419,
"<211": 420,
"211>": 421,
"<212": 422,
"212>": 423,
"<213": 424,
"213>": 425,
"<214": 426,
"214>": 427,
"<215": 428,
"215>": 429,
"<216": 430,
"216>": 431,
"<217": 432,
"217>": 433,
"<218": 434,
"218>": 435,
"<219": 436,
"219>": 437,
"<220": 438,
"220>": 439,
"<221": 440,
"221>": 441,
"<222": 442,
"222>": 443,
"<223": 444,
"223>": 445,
"<224": 446,
"224>": 447,
"<225": 448,
"225>": 449,
"<226": 450,
"226>": 451,
"<227": 452,
"227>": 453,
"<228": 454,
"228>": 455,
"<229": 456,
"229>": 457,
"<230": 458,
"230>": 459,
"<231": 460,
"231>": 461,
"<232": 462,
"232>": 463,
"<233": 464,
"233>": 465,
"<234": 466,
"234>": 467,
"<235": 468,
"235>": 469,
"<236": 470,
"236>": 471,
"<237": 472,
"237>": 473,
"<238": 474,
"238>": 475,
"<239": 476,
"239>": 477,
"<240": 478,
"240>": 479,
"<241": 480,
"241>": 481,
"<242": 482,
"242>": 483,
"<243": 484,
"243>": 485,
"<244": 486,
"244>": 487,
"<245": 488,
"245>": 489,
"<246": 490,
"246>": 491,
"<247": 492,
"247>": 493,
"<248": 494,
"248>": 495,
"<249": 496,
"249>": 497,
"<250": 498,
"250>": 499,
"[UNK]": 500,
"[PAD]": 501
},
"unk_token": "[UNK]"
}
}