syp1229's picture
Upload tokenizer
1220f7b
raw
history blame
17.3 kB
{
"(": 1,
")": 2,
"[PAD]": 1225,
"[UNK]": 1224,
"n": 3,
"p": 4,
"s": 5,
"|": 0,
"": 6,
"’": 7,
"가": 8,
"각": 9,
"간": 10,
"갈": 11,
"감": 12,
"갑": 13,
"값": 14,
"갓": 15,
"갔": 16,
"강": 17,
"갖": 18,
"같": 19,
"개": 20,
"객": 21,
"갠": 22,
"갯": 23,
"갱": 24,
"갸": 25,
"걀": 26,
"걔": 27,
"거": 28,
"걱": 29,
"건": 30,
"걷": 31,
"걸": 32,
"검": 33,
"겁": 34,
"것": 35,
"겉": 36,
"겊": 37,
"게": 38,
"겐": 39,
"겟": 40,
"겠": 41,
"겨": 42,
"격": 43,
"겪": 44,
"견": 45,
"결": 46,
"겸": 47,
"겹": 48,
"겼": 49,
"경": 50,
"곁": 51,
"계": 52,
"고": 53,
"곡": 54,
"곤": 55,
"곧": 56,
"골": 57,
"곰": 58,
"곱": 59,
"곳": 60,
"공": 61,
"곶": 62,
"과": 63,
"관": 64,
"광": 65,
"괜": 66,
"괭": 67,
"괴": 68,
"굉": 69,
"교": 70,
"구": 71,
"국": 72,
"군": 73,
"굳": 74,
"굴": 75,
"굵": 76,
"굶": 77,
"굽": 78,
"굿": 79,
"궁": 80,
"궂": 81,
"궈": 82,
"권": 83,
"귀": 84,
"귄": 85,
"규": 86,
"균": 87,
"귤": 88,
"그": 89,
"극": 90,
"근": 91,
"글": 92,
"긁": 93,
"금": 94,
"급": 95,
"긋": 96,
"긍": 97,
"기": 98,
"긴": 99,
"길": 100,
"김": 101,
"깁": 102,
"깃": 103,
"깄": 104,
"깅": 105,
"깊": 106,
"까": 107,
"깍": 108,
"깎": 109,
"깐": 110,
"깔": 111,
"깜": 112,
"깝": 113,
"깥": 114,
"깨": 115,
"깬": 116,
"깽": 117,
"꺼": 118,
"꺽": 119,
"껀": 120,
"껌": 121,
"껍": 122,
"껏": 123,
"껑": 124,
"께": 125,
"껴": 126,
"꼈": 127,
"꼬": 128,
"꼭": 129,
"꼰": 130,
"꼼": 131,
"꼽": 132,
"꽁": 133,
"꽂": 134,
"꽃": 135,
"꽝": 136,
"꽤": 137,
"꾀": 138,
"꾸": 139,
"꾹": 140,
"꾼": 141,
"꿀": 142,
"꿈": 143,
"꿉": 144,
"꿍": 145,
"꿔": 146,
"꿨": 147,
"뀌": 148,
"뀐": 149,
"뀔": 150,
"끄": 151,
"끈": 152,
"끊": 153,
"끌": 154,
"끓": 155,
"끔": 156,
"끗": 157,
"끝": 158,
"끼": 159,
"낀": 160,
"낄": 161,
"낌": 162,
"나": 163,
"낙": 164,
"난": 165,
"날": 166,
"남": 167,
"납": 168,
"낫": 169,
"났": 170,
"낭": 171,
"낮": 172,
"낯": 173,
"낳": 174,
"내": 175,
"낸": 176,
"낼": 177,
"냄": 178,
"냈": 179,
"냉": 180,
"냐": 181,
"냥": 182,
"너": 183,
"넉": 184,
"넌": 185,
"널": 186,
"넓": 187,
"넘": 188,
"넣": 189,
"네": 190,
"넷": 191,
"녀": 192,
"녁": 193,
"년": 194,
"념": 195,
"녔": 196,
"녕": 197,
"노": 198,
"녹": 199,
"논": 200,
"놀": 201,
"놈": 202,
"농": 203,
"높": 204,
"놓": 205,
"놔": 206,
"놨": 207,
"뇌": 208,
"뇨": 209,
"누": 210,
"눅": 211,
"눈": 212,
"눌": 213,
"눕": 214,
"눗": 215,
"눠": 216,
"눴": 217,
"뉘": 218,
"뉴": 219,
"늄": 220,
"느": 221,
"는": 222,
"늘": 223,
"늙": 224,
"능": 225,
"늦": 226,
"늪": 227,
"늬": 228,
"니": 229,
"닌": 230,
"닐": 231,
"님": 232,
"닙": 233,
"닝": 234,
"다": 235,
"닥": 236,
"닦": 237,
"단": 238,
"닫": 239,
"달": 240,
"닭": 241,
"닮": 242,
"담": 243,
"답": 244,
"닷": 245,
"당": 246,
"닿": 247,
"대": 248,
"댁": 249,
"댄": 250,
"댓": 251,
"댔": 252,
"댕": 253,
"더": 254,
"덕": 255,
"던": 256,
"덜": 257,
"덟": 258,
"덥": 259,
"덧": 260,
"덩": 261,
"덮": 262,
"데": 263,
"덴": 264,
"델": 265,
"도": 266,
"독": 267,
"돈": 268,
"돋": 269,
"돌": 270,
"돕": 271,
"돗": 272,
"동": 273,
"돼": 274,
"됐": 275,
"되": 276,
"된": 277,
"될": 278,
"됨": 279,
"됩": 280,
"두": 281,
"둑": 282,
"둔": 283,
"둘": 284,
"둠": 285,
"둥": 286,
"둬": 287,
"뒀": 288,
"뒤": 289,
"뒷": 290,
"뒹": 291,
"드": 292,
"득": 293,
"든": 294,
"듣": 295,
"들": 296,
"듬": 297,
"듯": 298,
"등": 299,
"듸": 300,
"디": 301,
"딘": 302,
"딜": 303,
"딤": 304,
"딨": 305,
"딩": 306,
"딪": 307,
"따": 308,
"딱": 309,
"딴": 310,
"딸": 311,
"땀": 312,
"땅": 313,
"때": 314,
"땐": 315,
"땜": 316,
"땠": 317,
"땡": 318,
"떠": 319,
"떡": 320,
"떤": 321,
"떨": 322,
"떴": 323,
"떻": 324,
"떼": 325,
"또": 326,
"똑": 327,
"똘": 328,
"똥": 329,
"뚜": 330,
"뚝": 331,
"뚫": 332,
"뚱": 333,
"뛰": 334,
"뜀": 335,
"뜨": 336,
"뜩": 337,
"뜬": 338,
"뜯": 339,
"뜰": 340,
"뜸": 341,
"뜻": 342,
"띄": 343,
"띕": 344,
"띠": 345,
"띵": 346,
"라": 347,
"락": 348,
"란": 349,
"랄": 350,
"람": 351,
"랍": 352,
"랐": 353,
"랑": 354,
"랗": 355,
"래": 356,
"랙": 357,
"랜": 358,
"램": 359,
"랩": 360,
"랫": 361,
"랬": 362,
"랭": 363,
"랴": 364,
"략": 365,
"량": 366,
"러": 367,
"럭": 368,
"런": 369,
"럴": 370,
"럼": 371,
"럽": 372,
"럿": 373,
"렀": 374,
"렁": 375,
"렇": 376,
"레": 377,
"렉": 378,
"렌": 379,
"렘": 380,
"려": 381,
"력": 382,
"련": 383,
"렬": 384,
"렴": 385,
"렵": 386,
"렷": 387,
"렸": 388,
"령": 389,
"례": 390,
"로": 391,
"록": 392,
"론": 393,
"롤": 394,
"롭": 395,
"롯": 396,
"롱": 397,
"뢰": 398,
"료": 399,
"루": 400,
"룩": 401,
"룰": 402,
"룸": 403,
"룹": 404,
"뤄": 405,
"뤘": 406,
"류": 407,
"륙": 408,
"륜": 409,
"률": 410,
"륨": 411,
"륭": 412,
"르": 413,
"른": 414,
"를": 415,
"름": 416,
"릅": 417,
"릇": 418,
"릉": 419,
"릎": 420,
"리": 421,
"릭": 422,
"린": 423,
"릴": 424,
"림": 425,
"립": 426,
"릿": 427,
"링": 428,
"마": 429,
"막": 430,
"만": 431,
"많": 432,
"말": 433,
"맑": 434,
"맘": 435,
"맙": 436,
"맛": 437,
"망": 438,
"맞": 439,
"맡": 440,
"매": 441,
"맥": 442,
"맨": 443,
"맵": 444,
"맷": 445,
"맹": 446,
"머": 447,
"먹": 448,
"먼": 449,
"멀": 450,
"멈": 451,
"멋": 452,
"멍": 453,
"메": 454,
"멘": 455,
"멜": 456,
"멤": 457,
"멧": 458,
"멩": 459,
"며": 460,
"면": 461,
"멸": 462,
"명": 463,
"몇": 464,
"모": 465,
"목": 466,
"몬": 467,
"몰": 468,
"몸": 469,
"몹": 470,
"못": 471,
"몽": 472,
"묘": 473,
"무": 474,
"묵": 475,
"묶": 476,
"문": 477,
"묻": 478,
"물": 479,
"묽": 480,
"뭉": 481,
"뭐": 482,
"뭔": 483,
"뭘": 484,
"뮤": 485,
"므": 486,
"미": 487,
"믹": 488,
"민": 489,
"믿": 490,
"밀": 491,
"밉": 492,
"밌": 493,
"밍": 494,
"밑": 495,
"바": 496,
"박": 497,
"밖": 498,
"반": 499,
"받": 500,
"발": 501,
"밝": 502,
"밟": 503,
"밤": 504,
"밥": 505,
"방": 506,
"밭": 507,
"배": 508,
"백": 509,
"밴": 510,
"밸": 511,
"뱀": 512,
"뱃": 513,
"버": 514,
"벅": 515,
"번": 516,
"벌": 517,
"범": 518,
"법": 519,
"벗": 520,
"베": 521,
"벤": 522,
"벨": 523,
"벳": 524,
"벼": 525,
"벽": 526,
"변": 527,
"별": 528,
"볍": 529,
"병": 530,
"볕": 531,
"보": 532,
"복": 533,
"볶": 534,
"본": 535,
"볼": 536,
"봄": 537,
"봅": 538,
"봇": 539,
"봉": 540,
"봐": 541,
"봤": 542,
"봬": 543,
"뵈": 544,
"부": 545,
"북": 546,
"분": 547,
"불": 548,
"붉": 549,
"붐": 550,
"붓": 551,
"붕": 552,
"붙": 553,
"뷔": 554,
"뷰": 555,
"브": 556,
"블": 557,
"비": 558,
"빅": 559,
"빈": 560,
"빌": 561,
"빔": 562,
"빕": 563,
"빗": 564,
"빙": 565,
"빛": 566,
"빠": 567,
"빡": 568,
"빨": 569,
"빴": 570,
"빵": 571,
"빼": 572,
"빽": 573,
"뺀": 574,
"뺄": 575,
"뺏": 576,
"뺐": 577,
"뺨": 578,
"뻐": 579,
"뻔": 580,
"뻗": 581,
"뻥": 582,
"뼈": 583,
"뼛": 584,
"뽀": 585,
"뽑": 586,
"뽕": 587,
"뾰": 588,
"뿌": 589,
"뿍": 590,
"뿐": 591,
"뿜": 592,
"쁘": 593,
"쁜": 594,
"쁠": 595,
"삐": 596,
"사": 597,
"삭": 598,
"산": 599,
"살": 600,
"삶": 601,
"삼": 602,
"샀": 603,
"상": 604,
"새": 605,
"색": 606,
"샐": 607,
"샘": 608,
"샛": 609,
"생": 610,
"샤": 611,
"샴": 612,
"샵": 613,
"샷": 614,
"섀": 615,
"서": 616,
"석": 617,
"섞": 618,
"선": 619,
"설": 620,
"섬": 621,
"섭": 622,
"섯": 623,
"섰": 624,
"성": 625,
"세": 626,
"센": 627,
"셀": 628,
"셈": 629,
"셋": 630,
"셔": 631,
"션": 632,
"셜": 633,
"셨": 634,
"셰": 635,
"소": 636,
"속": 637,
"손": 638,
"솔": 639,
"솜": 640,
"송": 641,
"쇄": 642,
"쇠": 643,
"쇼": 644,
"숄": 645,
"숍": 646,
"수": 647,
"숙": 648,
"순": 649,
"숟": 650,
"술": 651,
"숨": 652,
"숫": 653,
"숭": 654,
"숯": 655,
"숱": 656,
"쉐": 657,
"쉬": 658,
"쉴": 659,
"쉽": 660,
"슈": 661,
"슐": 662,
"슘": 663,
"스": 664,
"슥": 665,
"슨": 666,
"슬": 667,
"슴": 668,
"습": 669,
"슷": 670,
"승": 671,
"시": 672,
"식": 673,
"신": 674,
"싣": 675,
"실": 676,
"싫": 677,
"심": 678,
"십": 679,
"싱": 680,
"싶": 681,
"싸": 682,
"싹": 683,
"싼": 684,
"쌀": 685,
"쌈": 686,
"쌌": 687,
"쌓": 688,
"쌤": 689,
"쌩": 690,
"써": 691,
"썩": 692,
"썬": 693,
"썰": 694,
"썼": 695,
"쎄": 696,
"쏘": 697,
"쏙": 698,
"쏟": 699,
"쏠": 700,
"쐬": 701,
"쑤": 702,
"쑥": 703,
"쓰": 704,
"쓴": 705,
"쓸": 706,
"씀": 707,
"씁": 708,
"씨": 709,
"씩": 710,
"씬": 711,
"씹": 712,
"씻": 713,
"아": 714,
"악": 715,
"안": 716,
"앉": 717,
"않": 718,
"알": 719,
"앓": 720,
"암": 721,
"압": 722,
"앗": 723,
"았": 724,
"앙": 725,
"앞": 726,
"애": 727,
"액": 728,
"앤": 729,
"앨": 730,
"앱": 731,
"앵": 732,
"야": 733,
"약": 734,
"얀": 735,
"얄": 736,
"얇": 737,
"얌": 738,
"양": 739,
"얘": 740,
"어": 741,
"억": 742,
"언": 743,
"얹": 744,
"얻": 745,
"얼": 746,
"엄": 747,
"업": 748,
"없": 749,
"엇": 750,
"었": 751,
"엉": 752,
"엊": 753,
"엌": 754,
"엎": 755,
"에": 756,
"엑": 757,
"엔": 758,
"엘": 759,
"엠": 760,
"엣": 761,
"엥": 762,
"여": 763,
"역": 764,
"연": 765,
"열": 766,
"염": 767,
"엽": 768,
"엿": 769,
"였": 770,
"영": 771,
"옆": 772,
"예": 773,
"옛": 774,
"오": 775,
"옥": 776,
"온": 777,
"올": 778,
"옮": 779,
"옵": 780,
"옷": 781,
"옹": 782,
"와": 783,
"왁": 784,
"완": 785,
"왔": 786,
"왕": 787,
"왜": 788,
"왠": 789,
"외": 790,
"왼": 791,
"요": 792,
"욕": 793,
"용": 794,
"우": 795,
"욱": 796,
"운": 797,
"울": 798,
"움": 799,
"웃": 800,
"웅": 801,
"워": 802,
"웍": 803,
"원": 804,
"월": 805,
"웠": 806,
"웨": 807,
"웬": 808,
"웰": 809,
"웹": 810,
"위": 811,
"윗": 812,
"유": 813,
"육": 814,
"윤": 815,
"율": 816,
"윳": 817,
"융": 818,
"으": 819,
"은": 820,
"을": 821,
"읊": 822,
"음": 823,
"읍": 824,
"응": 825,
"의": 826,
"이": 827,
"익": 828,
"인": 829,
"일": 830,
"읽": 831,
"잃": 832,
"임": 833,
"입": 834,
"잇": 835,
"있": 836,
"잉": 837,
"잊": 838,
"잌": 839,
"잎": 840,
"자": 841,
"작": 842,
"잔": 843,
"잖": 844,
"잘": 845,
"잠": 846,
"잡": 847,
"잣": 848,
"잤": 849,
"장": 850,
"재": 851,
"잼": 852,
"쟁": 853,
"쟤": 854,
"저": 855,
"적": 856,
"전": 857,
"절": 858,
"젊": 859,
"점": 860,
"접": 861,
"젓": 862,
"정": 863,
"젖": 864,
"제": 865,
"젝": 866,
"젠": 867,
"져": 868,
"졌": 869,
"조": 870,
"족": 871,
"존": 872,
"졸": 873,
"좀": 874,
"좁": 875,
"종": 876,
"좋": 877,
"좌": 878,
"죄": 879,
"죠": 880,
"주": 881,
"죽": 882,
"준": 883,
"줄": 884,
"줌": 885,
"중": 886,
"줘": 887,
"줬": 888,
"쥐": 889,
"쥬": 890,
"즈": 891,
"즉": 892,
"즌": 893,
"즐": 894,
"즘": 895,
"즙": 896,
"증": 897,
"지": 898,
"직": 899,
"진": 900,
"질": 901,
"짐": 902,
"집": 903,
"짓": 904,
"징": 905,
"짚": 906,
"짜": 907,
"짝": 908,
"짧": 909,
"짬": 910,
"짰": 911,
"짱": 912,
"째": 913,
"쨌": 914,
"쨍": 915,
"쩌": 916,
"쩍": 917,
"쩐": 918,
"쩔": 919,
"쩜": 920,
"쩡": 921,
"쪄": 922,
"쪘": 923,
"쪼": 924,
"쪽": 925,
"쫀": 926,
"쫄": 927,
"쫌": 928,
"쫓": 929,
"쫙": 930,
"쬐": 931,
"쭈": 932,
"쭉": 933,
"쭤": 934,
"쯤": 935,
"찌": 936,
"찍": 937,
"찐": 938,
"찔": 939,
"찜": 940,
"찝": 941,
"찢": 942,
"차": 943,
"착": 944,
"찬": 945,
"찮": 946,
"찰": 947,
"참": 948,
"찹": 949,
"창": 950,
"찾": 951,
"채": 952,
"책": 953,
"챙": 954,
"챠": 955,
"처": 956,
"척": 957,
"천": 958,
"철": 959,
"첨": 960,
"첫": 961,
"청": 962,
"체": 963,
"쳇": 964,
"쳐": 965,
"쳤": 966,
"초": 967,
"촉": 968,
"촌": 969,
"촐": 970,
"총": 971,
"촬": 972,
"최": 973,
"추": 974,
"축": 975,
"춘": 976,
"출": 977,
"춤": 978,
"춥": 979,
"춧": 980,
"충": 981,
"춰": 982,
"췄": 983,
"취": 984,
"츄": 985,
"츠": 986,
"측": 987,
"층": 988,
"치": 989,
"칙": 990,
"친": 991,
"칠": 992,
"칡": 993,
"침": 994,
"칫": 995,
"칭": 996,
"카": 997,
"칵": 998,
"칸": 999,
"칼": 1000,
"캉": 1001,
"캐": 1002,
"캔": 1003,
"캠": 1004,
"캡": 1005,
"캬": 1006,
"커": 1007,
"컥": 1008,
"컨": 1009,
"컬": 1010,
"컴": 1011,
"컵": 1012,
"컷": 1013,
"컸": 1014,
"케": 1015,
"켓": 1016,
"켜": 1017,
"켰": 1018,
"코": 1019,
"콕": 1020,
"콘": 1021,
"콜": 1022,
"콤": 1023,
"콧": 1024,
"콩": 1025,
"쾌": 1026,
"쿄": 1027,
"쿠": 1028,
"쿡": 1029,
"쿨": 1030,
"쿵": 1031,
"쿽": 1032,
"퀴": 1033,
"큐": 1034,
"큘": 1035,
"크": 1036,
"큰": 1037,
"클": 1038,
"큼": 1039,
"키": 1040,
"킨": 1041,
"킬": 1042,
"킹": 1043,
"타": 1044,
"탁": 1045,
"탄": 1046,
"탈": 1047,
"탐": 1048,
"탑": 1049,
"탓": 1050,
"탔": 1051,
"탕": 1052,
"태": 1053,
"택": 1054,
"탭": 1055,
"탱": 1056,
"터": 1057,
"턱": 1058,
"턴": 1059,
"털": 1060,
"텀": 1061,
"텁": 1062,
"텃": 1063,
"텅": 1064,
"테": 1065,
"텍": 1066,
"텐": 1067,
"텔": 1068,
"템": 1069,
"텨": 1070,
"토": 1071,
"톡": 1072,
"톤": 1073,
"톨": 1074,
"톱": 1075,
"통": 1076,
"퇴": 1077,
"투": 1078,
"툰": 1079,
"툴": 1080,
"퉁": 1081,
"튀": 1082,
"튈": 1083,
"튜": 1084,
"트": 1085,
"특": 1086,
"튼": 1087,
"틀": 1088,
"틈": 1089,
"틔": 1090,
"티": 1091,
"틱": 1092,
"틴": 1093,
"틸": 1094,
"팀": 1095,
"팁": 1096,
"팅": 1097,
"파": 1098,
"팍": 1099,
"팎": 1100,
"판": 1101,
"팔": 1102,
"팟": 1103,
"팠": 1104,
"팡": 1105,
"패": 1106,
"팩": 1107,
"팬": 1108,
"팰": 1109,
"팸": 1110,
"팽": 1111,
"퍼": 1112,
"펄": 1113,
"펍": 1114,
"페": 1115,
"펙": 1116,
"펜": 1117,
"펭": 1118,
"펴": 1119,
"편": 1120,
"펼": 1121,
"폈": 1122,
"평": 1123,
"폐": 1124,
"포": 1125,
"폭": 1126,
"폰": 1127,
"폴": 1128,
"폼": 1129,
"표": 1130,
"푸": 1131,
"푹": 1132,
"푼": 1133,
"풀": 1134,
"품": 1135,
"풍": 1136,
"퓨": 1137,
"프": 1138,
"픈": 1139,
"플": 1140,
"픔": 1141,
"피": 1142,
"픽": 1143,
"핀": 1144,
"필": 1145,
"핏": 1146,
"핑": 1147,
"하": 1148,
"학": 1149,
"한": 1150,
"할": 1151,
"함": 1152,
"합": 1153,
"핫": 1154,
"항": 1155,
"해": 1156,
"핵": 1157,
"핸": 1158,
"햄": 1159,
"햇": 1160,
"했": 1161,
"행": 1162,
"향": 1163,
"허": 1164,
"헌": 1165,
"헐": 1166,
"험": 1167,
"헛": 1168,
"헝": 1169,
"헤": 1170,
"헬": 1171,
"헹": 1172,
"혀": 1173,
"혁": 1174,
"현": 1175,
"혈": 1176,
"협": 1177,
"혔": 1178,
"형": 1179,
"혜": 1180,
"호": 1181,
"혹": 1182,
"혼": 1183,
"홀": 1184,
"홈": 1185,
"홉": 1186,
"홍": 1187,
"화": 1188,
"확": 1189,
"환": 1190,
"활": 1191,
"황": 1192,
"회": 1193,
"획": 1194,
"횟": 1195,
"횡": 1196,
"효": 1197,
"후": 1198,
"훅": 1199,
"훈": 1200,
"훌": 1201,
"훑": 1202,
"훤": 1203,
"훨": 1204,
"훼": 1205,
"휘": 1206,
"휠": 1207,
"휩": 1208,
"휴": 1209,
"흐": 1210,
"흑": 1211,
"흔": 1212,
"흘": 1213,
"흙": 1214,
"흠": 1215,
"흡": 1216,
"흥": 1217,
"희": 1218,
"흰": 1219,
"히": 1220,
"힌": 1221,
"힐": 1222,
"힘": 1223
}