imgremlin's picture
Upload tokenizer
216dedb
{
"<bos>": 0,
"<pad>": 1,
"<eos>": 2,
"<unk>": 3,
".0": 4,
".1": 5,
".2": 6,
".3": 7,
".4": 8,
".5": 9,
".6": 10,
".7": 11,
".8": 12,
".888888": 13,
".9": 14,
"100": 15,
"1000": 16,
"1001": 17,
"1002": 18,
"1003": 19,
"1004": 20,
"1005": 21,
"1006": 22,
"1007": 23,
"1008": 24,
"1009": 25,
"101": 26,
"1011": 27,
"1012": 28,
"1013": 29,
"1014": 30,
"1015": 31,
"1016": 32,
"1017": 33,
"1018": 34,
"1019": 35,
"102": 36,
"1020": 37,
"1021": 38,
"1022": 39,
"1023": 40,
"1024": 41,
"1026": 42,
"1027": 43,
"1028": 44,
"1029": 45,
"103": 46,
"1030": 47,
"1031": 48,
"1032": 49,
"1033": 50,
"1034": 51,
"1035": 52,
"1036": 53,
"1037": 54,
"1038": 55,
"1039": 56,
"104": 57,
"1040": 58,
"1041": 59,
"1043": 60,
"1044": 61,
"1045": 62,
"1046": 63,
"1047": 64,
"1048": 65,
"1049": 66,
"105": 67,
"1050": 68,
"1051": 69,
"1052": 70,
"1053": 71,
"1054": 72,
"1055": 73,
"1057": 74,
"1058": 75,
"1059": 76,
"106": 77,
"1060": 78,
"1061": 79,
"1062": 80,
"1063": 81,
"1064": 82,
"1065": 83,
"1067": 84,
"1068": 85,
"1069": 86,
"107": 87,
"1070": 88,
"1071": 89,
"1072": 90,
"1073": 91,
"1074": 92,
"1075": 93,
"1076": 94,
"1077": 95,
"1078": 96,
"1079": 97,
"108": 98,
"1080": 99,
"1083": 100,
"1084": 101,
"1085": 102,
"1086": 103,
"1087": 104,
"1088": 105,
"1089": 106,
"109": 107,
"1090": 108,
"1091": 109,
"1092": 110,
"1093": 111,
"1094": 112,
"1095": 113,
"1096": 114,
"1098": 115,
"1099": 116,
"110": 117,
"1100": 118,
"1101": 119,
"1102": 120,
"1103": 121,
"1104": 122,
"1105": 123,
"1106": 124,
"1107": 125,
"1109": 126,
"111": 127,
"1111": 128,
"1112": 129,
"1113": 130,
"1114": 131,
"1115": 132,
"1117": 133,
"1118": 134,
"1119": 135,
"112": 136,
"1120": 137,
"1121": 138,
"1122": 139,
"1123": 140,
"1124": 141,
"1125": 142,
"1126": 143,
"1127": 144,
"1128": 145,
"113": 146,
"1130": 147,
"1131": 148,
"1134": 149,
"1135": 150,
"1136": 151,
"1137": 152,
"1139": 153,
"114": 154,
"1140": 155,
"1141": 156,
"1142": 157,
"1143": 158,
"1144": 159,
"1145": 160,
"1147": 161,
"1149": 162,
"115": 163,
"1150": 164,
"1151": 165,
"1152": 166,
"1153": 167,
"1154": 168,
"1155": 169,
"1156": 170,
"1159": 171,
"116": 172,
"1161": 173,
"1162": 174,
"1163": 175,
"1164": 176,
"1165": 177,
"1166": 178,
"1167": 179,
"117": 180,
"1173": 181,
"1175": 182,
"1176": 183,
"1177": 184,
"1178": 185,
"1179": 186,
"118": 187,
"1180": 188,
"1181": 189,
"1182": 190,
"1183": 191,
"1184": 192,
"1185": 193,
"1186": 194,
"1189": 195,
"119": 196,
"1191": 197,
"1192": 198,
"1193": 199,
"1198": 200,
"120": 201,
"1201": 202,
"1203": 203,
"1205": 204,
"1206": 205,
"1207": 206,
"1208": 207,
"1209": 208,
"121": 209,
"1210": 210,
"1211": 211,
"1212": 212,
"1215": 213,
"1216": 214,
"1217": 215,
"1218": 216,
"1219": 217,
"122": 218,
"1220": 219,
"1221": 220,
"1222": 221,
"1223": 222,
"1224": 223,
"1225": 224,
"1226": 225,
"1227": 226,
"1228": 227,
"123": 228,
"1230": 229,
"1231": 230,
"1232": 231,
"1233": 232,
"1234": 233,
"1236": 234,
"1237": 235,
"1238": 236,
"1239": 237,
"124": 238,
"1240": 239,
"1241": 240,
"1242": 241,
"1243": 242,
"1245": 243,
"1246": 244,
"1249": 245,
"125": 246,
"1250": 247,
"1251": 248,
"1252": 249,
"1255": 250,
"1257": 251,
"1258": 252,
"1259": 253,
"126": 254,
"1260": 255,
"1263": 256,
"1265": 257,
"1266": 258,
"1267": 259,
"1268": 260,
"1269": 261,
"127": 262,
"1270": 263,
"1271": 264,
"1275": 265,
"1276": 266,
"1277": 267,
"1278": 268,
"1279": 269,
"128": 270,
"1280": 271,
"1281": 272,
"1282": 273,
"1283": 274,
"1287": 275,
"1288": 276,
"1289": 277,
"129": 278,
"1290": 279,
"1293": 280,
"1294": 281,
"1296": 282,
"1297": 283,
"1298": 284,
"1299": 285,
"130": 286,
"1303": 287,
"1305": 288,
"1306": 289,
"1307": 290,
"1308": 291,
"1309": 292,
"131": 293,
"1310": 294,
"1311": 295,
"1314": 296,
"1315": 297,
"1317": 298,
"1318": 299,
"132": 300,
"1321": 301,
"1323": 302,
"1324": 303,
"1325": 304,
"1327": 305,
"133": 306,
"1332": 307,
"1333": 308,
"1339": 309,
"134": 310,
"1340": 311,
"1341": 312,
"1345": 313,
"1347": 314,
"135": 315,
"1353": 316,
"1356": 317,
"1357": 318,
"1358": 319,
"1359": 320,
"136": 321,
"1360": 322,
"1362": 323,
"1363": 324,
"1365": 325,
"1366": 326,
"1367": 327,
"1369": 328,
"137": 329,
"1370": 330,
"1371": 331,
"1372": 332,
"1377": 333,
"1378": 334,
"138": 335,
"1381": 336,
"1382": 337,
"1384": 338,
"1385": 339,
"1389": 340,
"139": 341,
"1390": 342,
"1391": 343,
"1393": 344,
"1394": 345,
"1395": 346,
"1396": 347,
"1399": 348,
"140": 349,
"1403": 350,
"1404": 351,
"1407": 352,
"1408": 353,
"1409": 354,
"141": 355,
"1411": 356,
"1412": 357,
"1413": 358,
"1419": 359,
"142": 360,
"1421": 361,
"1422": 362,
"1425": 363,
"1426": 364,
"1427": 365,
"1429": 366,
"143": 367,
"1430": 368,
"1431": 369,
"1432": 370,
"1437": 371,
"1438": 372,
"1439": 373,
"144": 374,
"1441": 375,
"1443": 376,
"1444": 377,
"1449": 378,
"145": 379,
"1450": 380,
"1455": 381,
"146": 382,
"1461": 383,
"1466": 384,
"1467": 385,
"147": 386,
"1471": 387,
"1472": 388,
"1479": 389,
"148": 390,
"1480": 391,
"1489": 392,
"149": 393,
"1490": 394,
"1491": 395,
"1492": 396,
"1496": 397,
"1497": 398,
"1498": 399,
"150": 400,
"151": 401,
"152": 402,
"1527": 403,
"1528": 404,
"153": 405,
"154": 406,
"1543": 407,
"1546": 408,
"155": 409,
"1553": 410,
"1559": 411,
"156": 412,
"1561": 413,
"1569": 414,
"157": 415,
"1574": 416,
"158": 417,
"159": 418,
"160": 419,
"161": 420,
"1611": 421,
"162": 422,
"1620": 423,
"1621": 424,
"163": 425,
"1633": 426,
"164": 427,
"165": 428,
"1653": 429,
"1659": 430,
"166": 431,
"167": 432,
"168": 433,
"169": 434,
"170": 435,
"171": 436,
"172": 437,
"173": 438,
"174": 439,
"175": 440,
"176": 441,
"177": 442,
"178": 443,
"179": 444,
"180": 445,
"181": 446,
"182": 447,
"183": 448,
"184": 449,
"185": 450,
"186": 451,
"187": 452,
"188": 453,
"189": 454,
"190": 455,
"191": 456,
"192": 457,
"193": 458,
"194": 459,
"195": 460,
"196": 461,
"197": 462,
"198": 463,
"199": 464,
"200": 465,
"201": 466,
"202": 467,
"203": 468,
"204": 469,
"205": 470,
"206": 471,
"207": 472,
"208": 473,
"209": 474,
"210": 475,
"211": 476,
"212": 477,
"213": 478,
"214": 479,
"215": 480,
"216": 481,
"217": 482,
"218": 483,
"219": 484,
"220": 485,
"221": 486,
"222": 487,
"223": 488,
"224": 489,
"225": 490,
"226": 491,
"227": 492,
"228": 493,
"229": 494,
"230": 495,
"231": 496,
"232": 497,
"233": 498,
"234": 499,
"235": 500,
"236": 501,
"237": 502,
"238": 503,
"239": 504,
"240": 505,
"241": 506,
"242": 507,
"243": 508,
"244": 509,
"245": 510,
"246": 511,
"247": 512,
"248": 513,
"249": 514,
"250": 515,
"251": 516,
"252": 517,
"253": 518,
"254": 519,
"255": 520,
"256": 521,
"257": 522,
"258": 523,
"259": 524,
"260": 525,
"261": 526,
"262": 527,
"263": 528,
"264": 529,
"265": 530,
"266": 531,
"267": 532,
"268": 533,
"269": 534,
"270": 535,
"271": 536,
"272": 537,
"273": 538,
"274": 539,
"275": 540,
"276": 541,
"277": 542,
"278": 543,
"279": 544,
"280": 545,
"281": 546,
"282": 547,
"283": 548,
"284": 549,
"285": 550,
"286": 551,
"287": 552,
"288": 553,
"289": 554,
"290": 555,
"291": 556,
"292": 557,
"293": 558,
"294": 559,
"295": 560,
"296": 561,
"297": 562,
"298": 563,
"299": 564,
"300": 565,
"301": 566,
"302": 567,
"303": 568,
"304": 569,
"305": 570,
"306": 571,
"307": 572,
"308": 573,
"309": 574,
"310": 575,
"311": 576,
"312": 577,
"313": 578,
"314": 579,
"315": 580,
"316": 581,
"317": 582,
"318": 583,
"319": 584,
"320": 585,
"321": 586,
"322": 587,
"323": 588,
"324": 589,
"325": 590,
"326": 591,
"327": 592,
"328": 593,
"329": 594,
"330": 595,
"331": 596,
"332": 597,
"333": 598,
"334": 599,
"335": 600,
"336": 601,
"337": 602,
"338": 603,
"339": 604,
"340": 605,
"341": 606,
"342": 607,
"343": 608,
"344": 609,
"345": 610,
"346": 611,
"347": 612,
"348": 613,
"349": 614,
"350": 615,
"351": 616,
"352": 617,
"353": 618,
"354": 619,
"355": 620,
"356": 621,
"357": 622,
"358": 623,
"359": 624,
"360": 625,
"361": 626,
"362": 627,
"363": 628,
"364": 629,
"365": 630,
"366": 631,
"367": 632,
"368": 633,
"369": 634,
"370": 635,
"371": 636,
"372": 637,
"373": 638,
"374": 639,
"375": 640,
"376": 641,
"377": 642,
"378": 643,
"379": 644,
"380": 645,
"381": 646,
"382": 647,
"383": 648,
"384": 649,
"385": 650,
"386": 651,
"387": 652,
"388": 653,
"389": 654,
"390": 655,
"391": 656,
"392": 657,
"393": 658,
"394": 659,
"395": 660,
"396": 661,
"397": 662,
"398": 663,
"399": 664,
"400": 665,
"401": 666,
"402": 667,
"403": 668,
"404": 669,
"405": 670,
"406": 671,
"407": 672,
"408": 673,
"409": 674,
"410": 675,
"411": 676,
"412": 677,
"413": 678,
"414": 679,
"415": 680,
"416": 681,
"417": 682,
"418": 683,
"419": 684,
"420": 685,
"421": 686,
"422": 687,
"423": 688,
"424": 689,
"425": 690,
"426": 691,
"427": 692,
"428": 693,
"429": 694,
"430": 695,
"431": 696,
"432": 697,
"433": 698,
"434": 699,
"435": 700,
"436": 701,
"437": 702,
"438": 703,
"439": 704,
"440": 705,
"441": 706,
"442": 707,
"443": 708,
"444": 709,
"445": 710,
"446": 711,
"447": 712,
"448": 713,
"449": 714,
"450": 715,
"451": 716,
"452": 717,
"453": 718,
"454": 719,
"455": 720,
"456": 721,
"457": 722,
"458": 723,
"459": 724,
"460": 725,
"461": 726,
"462": 727,
"463": 728,
"464": 729,
"465": 730,
"466": 731,
"467": 732,
"468": 733,
"469": 734,
"470": 735,
"471": 736,
"472": 737,
"473": 738,
"474": 739,
"475": 740,
"476": 741,
"477": 742,
"478": 743,
"479": 744,
"480": 745,
"481": 746,
"482": 747,
"483": 748,
"484": 749,
"485": 750,
"486": 751,
"487": 752,
"488": 753,
"489": 754,
"490": 755,
"491": 756,
"492": 757,
"493": 758,
"494": 759,
"495": 760,
"496": 761,
"497": 762,
"498": 763,
"499": 764,
"50": 765,
"500": 766,
"501": 767,
"502": 768,
"503": 769,
"504": 770,
"505": 771,
"506": 772,
"507": 773,
"508": 774,
"509": 775,
"51": 776,
"510": 777,
"511": 778,
"512": 779,
"513": 780,
"514": 781,
"515": 782,
"516": 783,
"517": 784,
"518": 785,
"519": 786,
"52": 787,
"520": 788,
"521": 789,
"522": 790,
"523": 791,
"524": 792,
"525": 793,
"526": 794,
"527": 795,
"528": 796,
"529": 797,
"53": 798,
"530": 799,
"531": 800,
"532": 801,
"533": 802,
"534": 803,
"535": 804,
"536": 805,
"537": 806,
"538": 807,
"539": 808,
"54": 809,
"540": 810,
"541": 811,
"542": 812,
"543": 813,
"544": 814,
"545": 815,
"546": 816,
"547": 817,
"548": 818,
"549": 819,
"55": 820,
"550": 821,
"551": 822,
"552": 823,
"553": 824,
"554": 825,
"555": 826,
"556": 827,
"557": 828,
"558": 829,
"559": 830,
"56": 831,
"560": 832,
"561": 833,
"562": 834,
"563": 835,
"564": 836,
"565": 837,
"566": 838,
"567": 839,
"568": 840,
"569": 841,
"57": 842,
"570": 843,
"571": 844,
"572": 845,
"573": 846,
"574": 847,
"575": 848,
"576": 849,
"577": 850,
"578": 851,
"579": 852,
"58": 853,
"580": 854,
"581": 855,
"582": 856,
"583": 857,
"584": 858,
"585": 859,
"586": 860,
"587": 861,
"588": 862,
"589": 863,
"59": 864,
"590": 865,
"591": 866,
"592": 867,
"593": 868,
"594": 869,
"595": 870,
"596": 871,
"597": 872,
"598": 873,
"599": 874,
"60": 875,
"600": 876,
"601": 877,
"602": 878,
"603": 879,
"604": 880,
"605": 881,
"606": 882,
"607": 883,
"608": 884,
"609": 885,
"61": 886,
"610": 887,
"611": 888,
"612": 889,
"613": 890,
"614": 891,
"615": 892,
"616": 893,
"617": 894,
"618": 895,
"619": 896,
"62": 897,
"620": 898,
"621": 899,
"622": 900,
"623": 901,
"624": 902,
"625": 903,
"626": 904,
"627": 905,
"628": 906,
"629": 907,
"63": 908,
"630": 909,
"631": 910,
"632": 911,
"633": 912,
"634": 913,
"635": 914,
"636": 915,
"637": 916,
"638": 917,
"639": 918,
"64": 919,
"640": 920,
"641": 921,
"642": 922,
"643": 923,
"644": 924,
"645": 925,
"646": 926,
"647": 927,
"648": 928,
"649": 929,
"65": 930,
"650": 931,
"651": 932,
"652": 933,
"653": 934,
"654": 935,
"655": 936,
"656": 937,
"657": 938,
"658": 939,
"659": 940,
"66": 941,
"660": 942,
"661": 943,
"662": 944,
"663": 945,
"664": 946,
"665": 947,
"666": 948,
"667": 949,
"668": 950,
"669": 951,
"67": 952,
"670": 953,
"671": 954,
"673": 955,
"674": 956,
"675": 957,
"676": 958,
"677": 959,
"678": 960,
"679": 961,
"68": 962,
"680": 963,
"681": 964,
"682": 965,
"683": 966,
"684": 967,
"685": 968,
"686": 969,
"687": 970,
"688": 971,
"689": 972,
"69": 973,
"690": 974,
"691": 975,
"692": 976,
"693": 977,
"694": 978,
"695": 979,
"696": 980,
"697": 981,
"698": 982,
"699": 983,
"70": 984,
"700": 985,
"701": 986,
"702": 987,
"703": 988,
"704": 989,
"705": 990,
"706": 991,
"707": 992,
"708": 993,
"709": 994,
"71": 995,
"710": 996,
"711": 997,
"712": 998,
"713": 999,
"714": 1000,
"715": 1001,
"716": 1002,
"717": 1003,
"718": 1004,
"719": 1005,
"72": 1006,
"720": 1007,
"721": 1008,
"722": 1009,
"723": 1010,
"724": 1011,
"725": 1012,
"726": 1013,
"727": 1014,
"728": 1015,
"729": 1016,
"73": 1017,
"730": 1018,
"731": 1019,
"732": 1020,
"733": 1021,
"734": 1022,
"735": 1023,
"736": 1024,
"737": 1025,
"738": 1026,
"739": 1027,
"74": 1028,
"740": 1029,
"741": 1030,
"742": 1031,
"743": 1032,
"744": 1033,
"745": 1034,
"746": 1035,
"747": 1036,
"748": 1037,
"749": 1038,
"75": 1039,
"750": 1040,
"751": 1041,
"752": 1042,
"753": 1043,
"754": 1044,
"755": 1045,
"756": 1046,
"757": 1047,
"758": 1048,
"759": 1049,
"76": 1050,
"760": 1051,
"761": 1052,
"762": 1053,
"763": 1054,
"764": 1055,
"765": 1056,
"766": 1057,
"767": 1058,
"768": 1059,
"769": 1060,
"77": 1061,
"770": 1062,
"771": 1063,
"772": 1064,
"773": 1065,
"774": 1066,
"775": 1067,
"776": 1068,
"777": 1069,
"778": 1070,
"779": 1071,
"78": 1072,
"780": 1073,
"781": 1074,
"782": 1075,
"783": 1076,
"784": 1077,
"785": 1078,
"786": 1079,
"787": 1080,
"788": 1081,
"789": 1082,
"79": 1083,
"790": 1084,
"791": 1085,
"792": 1086,
"793": 1087,
"794": 1088,
"795": 1089,
"796": 1090,
"797": 1091,
"798": 1092,
"799": 1093,
"80": 1094,
"800": 1095,
"801": 1096,
"803": 1097,
"804": 1098,
"805": 1099,
"806": 1100,
"807": 1101,
"808": 1102,
"809": 1103,
"81": 1104,
"810": 1105,
"811": 1106,
"812": 1107,
"813": 1108,
"814": 1109,
"815": 1110,
"816": 1111,
"817": 1112,
"818": 1113,
"819": 1114,
"82": 1115,
"820": 1116,
"821": 1117,
"822": 1118,
"823": 1119,
"824": 1120,
"825": 1121,
"826": 1122,
"827": 1123,
"828": 1124,
"829": 1125,
"83": 1126,
"830": 1127,
"831": 1128,
"832": 1129,
"833": 1130,
"834": 1131,
"835": 1132,
"836": 1133,
"837": 1134,
"838": 1135,
"839": 1136,
"84": 1137,
"840": 1138,
"841": 1139,
"842": 1140,
"843": 1141,
"844": 1142,
"845": 1143,
"846": 1144,
"847": 1145,
"848": 1146,
"849": 1147,
"85": 1148,
"850": 1149,
"851": 1150,
"852": 1151,
"853": 1152,
"854": 1153,
"855": 1154,
"856": 1155,
"857": 1156,
"858": 1157,
"859": 1158,
"86": 1159,
"860": 1160,
"861": 1161,
"862": 1162,
"863": 1163,
"864": 1164,
"865": 1165,
"866": 1166,
"867": 1167,
"868": 1168,
"869": 1169,
"87": 1170,
"870": 1171,
"871": 1172,
"872": 1173,
"873": 1174,
"875": 1175,
"876": 1176,
"877": 1177,
"878": 1178,
"879": 1179,
"88": 1180,
"880": 1181,
"881": 1182,
"882": 1183,
"883": 1184,
"884": 1185,
"885": 1186,
"886": 1187,
"887": 1188,
"888": 1189,
"889": 1190,
"89": 1191,
"890": 1192,
"891": 1193,
"892": 1194,
"893": 1195,
"894": 1196,
"895": 1197,
"896": 1198,
"897": 1199,
"898": 1200,
"899": 1201,
"90": 1202,
"900": 1203,
"901": 1204,
"902": 1205,
"903": 1206,
"905": 1207,
"906": 1208,
"907": 1209,
"908": 1210,
"909": 1211,
"91": 1212,
"910": 1213,
"911": 1214,
"912": 1215,
"913": 1216,
"914": 1217,
"917": 1218,
"918": 1219,
"919": 1220,
"92": 1221,
"921": 1222,
"923": 1223,
"924": 1224,
"925": 1225,
"926": 1226,
"927": 1227,
"928": 1228,
"929": 1229,
"93": 1230,
"930": 1231,
"931": 1232,
"932": 1233,
"935": 1234,
"937": 1235,
"938": 1236,
"939": 1237,
"94": 1238,
"940": 1239,
"941": 1240,
"942": 1241,
"943": 1242,
"944": 1243,
"945": 1244,
"946": 1245,
"947": 1246,
"948": 1247,
"949": 1248,
"95": 1249,
"950": 1250,
"951": 1251,
"952": 1252,
"953": 1253,
"954": 1254,
"955": 1255,
"956": 1256,
"957": 1257,
"958": 1258,
"959": 1259,
"96": 1260,
"960": 1261,
"961": 1262,
"962": 1263,
"963": 1264,
"965": 1265,
"967": 1266,
"968": 1267,
"969": 1268,
"97": 1269,
"970": 1270,
"971": 1271,
"972": 1272,
"973": 1273,
"974": 1274,
"975": 1275,
"977": 1276,
"978": 1277,
"979": 1278,
"98": 1279,
"980": 1280,
"981": 1281,
"982": 1282,
"983": 1283,
"985": 1284,
"986": 1285,
"987": 1286,
"988": 1287,
"989": 1288,
"99": 1289,
"990": 1290,
"991": 1291,
"992": 1292,
"993": 1293,
"994": 1294,
"995": 1295,
"996": 1296,
"997": 1297,
"998": 1298,
"999": 1299,
"999999": 1300
}