jaeyeon's picture
add tokenizer
b280fc8
{
"[PAD]": 529,
"[UNK]": 528,
"|": 0,
"๊ฐ€": 1,
"๊ฐ": 2,
"๊ฐ„": 3,
"๊ฐˆ": 4,
"๊ฐ": 5,
"๊ฐ‘": 6,
"๊ฐ’": 7,
"๊ฐ•": 8,
"๊ฐ–": 9,
"๊ฐ™": 10,
"๊ฐœ": 11,
"๊ฑฐ": 12,
"๊ฑฑ": 13,
"๊ฑด": 14,
"๊ฑธ": 15,
"๊ฒ": 16,
"๊ฒƒ": 17,
"๊ฒŒ": 18,
"๊ฒ ": 19,
"๊ฒจ": 20,
"๊ฒฉ": 21,
"๊ฒฐ": 22,
"๊ฒน": 23,
"๊ฒฝ": 24,
"๊ณ„": 25,
"๊ณ ": 26,
"๊ณก": 27,
"๊ณฑ": 28,
"๊ณต": 29,
"๊ณผ": 30,
"๊ด€": 31,
"๊ด„": 32,
"๊ดœ": 33,
"๊ต‰": 34,
"๊ต": 35,
"๊ตฌ": 36,
"๊ตญ": 37,
"๊ถค": 38,
"๊ทœ": 39,
"๊ทธ": 40,
"๊ทน": 41,
"๊ทผ": 42,
"๊ธ€": 43,
"๊ธˆ": 44,
"๊ธฐ": 45,
"๊ธด": 46,
"๊ธธ": 47,
"๊น€": 48,
"๊น": 49,
"๊นŠ": 50,
"๊นŒ": 51,
"๊น": 52,
"๊ผญ": 53,
"๊ผด": 54,
"๊พธ": 55,
"๊ฟฐ": 56,
"๋€Œ": 57,
"๋€": 58,
"๋ˆ": 59,
"๋Š": 60,
"๋": 61,
"๋ผ": 62,
"๋‚Œ": 63,
"๋‚˜": 64,
"๋‚™": 65,
"๋‚œ": 66,
"๋‚ ": 67,
"๋‚ฉ": 68,
"๋‚ฌ": 69,
"๋‚ด": 70,
"๋ƒˆ": 71,
"๋ƒ": 72,
"๋ƒฅ": 73,
"๋„ˆ": 74,
"๋„“": 75,
"๋„˜": 76,
"๋„ฃ": 77,
"๋„ค": 78,
"๋…„": 79,
"๋…": 80,
"๋…ธ": 81,
"๋…ผ": 82,
"๋†’": 83,
"๋†“": 84,
"๋†จ": 85,
"๋ˆ„": 86,
"๋ˆˆ": 87,
"๋ˆ ": 88,
"๋‰œ": 89,
"๋‰ด": 90,
"๋Š": 91,
"๋Š”": 92,
"๋Š˜": 93,
"๋Šฅ": 94,
"๋‹ˆ": 95,
"๋‹Œ": 96,
"๋‹": 97,
"๋‹˜": 98,
"๋‹™": 99,
"๋‹ค": 100,
"๋‹จ": 101,
"๋‹ฌ": 102,
"๋‹ฎ": 103,
"๋‹ด": 104,
"๋‹ต": 105,
"๋‹น": 106,
"๋Œ€": 107,
"๋Œˆ": 108,
"๋Œ“": 109,
"๋”": 110,
"๋˜": 111,
"๋Ÿ": 112,
"๋ฐ": 113,
"๋„": 114,
"๋…": 115,
"๋Œ": 116,
"๋™": 117,
"๋ผ": 118,
"๋": 119,
"๋˜": 120,
"๋œ": 121,
"๋ ": 122,
"๋ฉ": 123,
"๋‘": 124,
"๋‘˜": 125,
"๋‘ฅ": 126,
"๋’ค": 127,
"๋“œ": 128,
"๋“ ": 129,
"๋“ฃ": 130,
"๋“ค": 131,
"๋“ฌ": 132,
"๋“ญ": 133,
"๋“ฑ": 134,
"๋””": 135,
"๋”ฐ": 136,
"๋”ฑ": 137,
"๋•Œ": 138,
"๋– ": 139,
"๋–ค": 140,
"๋–จ": 141,
"๋–ป": 142,
"๋˜": 143,
"๋˜‘": 144,
"๋˜ฅ": 145,
"๋šฑ": 146,
"๋œป": 147,
"๋ผ": 148,
"๋ž€": 149,
"๋ž„": 150,
"๋žŒ": 151,
"๋ž": 152,
"๋ž‘": 153,
"๋ž˜": 154,
"๋žœ": 155,
"๋žต": 156,
"๋Ÿ‰": 157,
"๋Ÿฌ": 158,
"๋Ÿฐ": 159,
"๋Ÿด": 160,
"๋Ÿผ": 161,
"๋Ÿฝ": 162,
"๋ ‡": 163,
"๋ ˆ": 164,
"๋ ค": 165,
"๋ ฅ": 166,
"๋ จ": 167,
"๋ ต": 168,
"๋ ธ": 169,
"๋ น": 170,
"๋ก€": 171,
"๋กœ": 172,
"๋ก ": 173,
"๋กญ": 174,
"๋ฃŒ": 175,
"๋ฃจ": 176,
"๋ฅ˜": 177,
"๋ฅ ": 178,
"๋ฅจ": 179,
"๋ฅญ": 180,
"๋ฅด": 181,
"๋ฅต": 182,
"๋ฅธ": 183,
"๋ฅผ": 184,
"๋ฆ„": 185,
"๋ฆ…": 186,
"๋ฆฌ": 187,
"๋ฆญ": 188,
"๋ฆฐ": 189,
"๋ฆผ": 190,
"๋ฆฝ": 191,
"๋งˆ": 192,
"๋ง‰": 193,
"๋งŒ": 194,
"๋งŽ": 195,
"๋ง": 196,
"๋ง˜": 197,
"๋งž": 198,
"๋งฃ": 199,
"๋งฅ": 200,
"๋จธ": 201,
"๋จน": 202,
"๋จผ": 203,
"๋ฉˆ": 204,
"๋ฉ”": 205,
"๋ฉ˜": 206,
"๋ฉฉ": 207,
"๋ฉด": 208,
"๋ช…": 209,
"๋ช‡": 210,
"๋ชจ": 211,
"๋ชฉ": 212,
"๋ชฐ": 213,
"๋ชป": 214,
"๋ฌด": 215,
"๋ฌธ": 216,
"๋ฌผ": 217,
"๋ญ‰": 218,
"๋ญ": 219,
"๋ญ˜": 220,
"๋ญก": 221,
"๋ฏธ": 222,
"๋ฏผ": 223,
"๋ฐ€": 224,
"๋ฐ”": 225,
"๋ฐ–": 226,
"๋ฐ˜": 227,
"๋ฐ›": 228,
"๋ฐœ": 229,
"๋ฐฉ": 230,
"๋ฐฐ": 231,
"๋ฐฑ": 232,
"๋ฒ„": 233,
"๋ฒˆ": 234,
"๋ฒŒ": 235,
"๋ฒ”": 236,
"๋ฒก": 237,
"๋ฒฝ": 238,
"๋ณ€": 239,
"๋ณ„": 240,
"๋ณ": 241,
"๋ณด": 242,
"๋ณต": 243,
"๋ณธ": 244,
"๋ณผ": 245,
"๋ด": 246,
"๋ดค": 247,
"๋ถ€": 248,
"๋ถ„": 249,
"๋ถˆ": 250,
"๋ถ“": 251,
"๋ถ™": 252,
"๋น„": 253,
"๋น ": 254,
"๋นผ": 255,
"๋บ": 256,
"๋ฟ”": 257,
"์‚ฌ": 258,
"์‚ฐ": 259,
"์‚ด": 260,
"์‚ผ": 261,
"์ƒ": 262,
"์ƒˆ": 263,
"์ƒ‰": 264,
"์ƒ": 265,
"์„œ": 266,
"์„": 267,
"์„ ": 268,
"์„ค": 269,
"์„ฏ": 270,
"์„ฑ": 271,
"์„ธ": 272,
"์…จ": 273,
"์†Œ": 274,
"์†": 275,
"์†": 276,
"์ˆ˜": 277,
"์ˆ™": 278,
"์ˆœ": 279,
"์ˆซ": 280,
"์‰ฌ": 281,
"์Šˆ": 282,
"์Š‰": 283,
"์Šค": 284,
"์Šต": 285,
"์Šท": 286,
"์‹œ": 287,
"์‹": 288,
"์‹ ": 289,
"์‹ค": 290,
"์‹ฌ": 291,
"์‹ญ": 292,
"์‹ถ": 293,
"์‹น": 294,
"์Œ": 295,
"์จ": 296,
"์Ÿ": 297,
"์“ฐ": 298,
"์“ด": 299,
"์“ธ": 300,
"์”Œ": 301,
"์”จ": 302,
"์”ฉ": 303,
"์”ฌ": 304,
"์•„": 305,
"์•ˆ": 306,
"์•Š": 307,
"์•Œ": 308,
"์•˜": 309,
"์•ž": 310,
"์• ": 311,
"์•ผ": 312,
"์•ฝ": 313,
"์–‡": 314,
"์–‘": 315,
"์–•": 316,
"์–˜": 317,
"์–ด": 318,
"์–ต": 319,
"์–ธ": 320,
"์–ผ": 321,
"์—„": 322,
"์—…": 323,
"์—†": 324,
"์—ˆ": 325,
"์—": 326,
"์—‘": 327,
"์—”": 328,
"์— ": 329,
"์—ฌ": 330,
"์—ญ": 331,
"์—ฐ": 332,
"์—ด": 333,
"์˜€": 334,
"์˜": 335,
"์˜ˆ": 336,
"์˜ค": 337,
"์˜จ": 338,
"์˜ฌ": 339,
"์˜ฎ": 340,
"์˜ท": 341,
"์™€": 342,
"์™„": 343,
"์™”": 344,
"์™œ": 345,
"์™ธ": 346,
"์™ผ": 347,
"์š”": 348,
"์šฉ": 349,
"์šฐ": 350,
"์šด": 351,
"์šธ": 352,
"์›€": 353,
"์›": 354,
"์›Œ": 355,
"์›": 356,
"์› ": 357,
"์œ„": 358,
"์œ ": 359,
"์œก": 360,
"์œจ": 361,
"์œผ": 362,
"์€": 363,
"์„": 364,
"์Œ": 365,
"์‘": 366,
"์˜": 367,
"์ด": 368,
"์ต": 369,
"์ธ": 370,
"์ผ": 371,
"์ฝ": 372,
"์ž„": 373,
"์ž…": 374,
"์žˆ": 375,
"์žŠ": 376,
"์ž": 377,
"์ž‘": 378,
"์ž–": 379,
"์ž˜": 380,
"์ž ": 381,
"์žก": 382,
"์žฅ": 383,
"์žฌ": 384,
"์ €": 385,
"์ ": 386,
"์ „": 387,
"์ ˆ": 388,
"์ ": 389,
"์ ‘": 390,
"์ •": 391,
"์ œ": 392,
"์ ธ": 393,
"์กŒ": 394,
"์กฐ": 395,
"์กฑ": 396,
"์กด": 397,
"์กธ": 398,
"์ข€": 399,
"์ข…": 400,
"์ข‹": 401,
"์ขŒ": 402,
"์ฃ ": 403,
"์ฃผ": 404,
"์ค€": 405,
"์ค„": 406,
"์ค‘": 407,
"์ฆ": 408,
"์ง€": 409,
"์ง": 410,
"์ง„": 411,
"์งˆ": 412,
"์ง": 413,
"์ง‘": 414,
"์ง“": 415,
"์ง•": 416,
"์งœ": 417,
"์ง": 418,
"์งง": 419,
"์งฑ": 420,
"์งธ": 421,
"์จŒ": 422,
"์ฉœ": 423,
"์ชผ": 424,
"์ชฝ": 425,
"์ซ™": 426,
"์ญ‰": 427,
"์ฏค": 428,
"์ฐŒ": 429,
"์ฐ": 430,
"์ฐจ": 431,
"์ฐฎ": 432,
"์ฐธ": 433,
"์ฑ„": 434,
"์ฑ…": 435,
"์ฒ˜": 436,
"์ฒœ": 437,
"์ฒ ": 438,
"์ฒซ": 439,
"์ฒญ": 440,
"์ฒด": 441,
"์ณค": 442,
"์ดˆ": 443,
"์ด": 444,
"์ตœ": 445,
"์ถ”": 446,
"์ถ•": 447,
"์ถœ": 448,
"์ถฉ": 449,
"์ถฐ": 450,
"์น˜": 451,
"์น™": 452,
"์นœ": 453,
"์น ": 454,
"์นจ": 455,
"์นด": 456,
"์นธ": 457,
"์ปค": 458,
"์ปด": 459,
"์ผ€": 460,
"์ฝ”": 461,
"ํ€„": 462,
"ํฌ": 463,
"ํฐ": 464,
"ํด": 465,
"ํผ": 466,
"ํ‚ฌ": 467,
"ํƒ€": 468,
"ํƒœ": 469,
"ํƒ": 470,
"ํ„ฐ": 471,
"ํ…Œ": 472,
"ํ…": 473,
"ํ† ": 474,
"ํ†ต": 475,
"ํˆฌ": 476,
"ํŠธ": 477,
"ํŠน": 478,
"ํŠผ": 479,
"ํ‹€": 480,
"ํ‹ฐ": 481,
"ํŒŒ": 482,
"ํŒ”": 483,
"ํŽ˜": 484,
"ํŽด": 485,
"ํŽธ": 486,
"ํŽผ": 487,
"ํ…": 488,
"ํ‰": 489,
"ํฌ": 490,
"ํด": 491,
"ํ‘œ": 492,
"ํ‘ธ": 493,
"ํ‘ผ": 494,
"ํ’€": 495,
"ํ”„": 496,
"ํ”Œ": 497,
"ํ”ผ": 498,
"ํ•„": 499,
"ํ•˜": 500,
"ํ•™": 501,
"ํ•œ": 502,
"ํ• ": 503,
"ํ•จ": 504,
"ํ•ฉ": 505,
"ํ•ญ": 506,
"ํ•ด": 507,
"ํ–ˆ": 508,
"ํ–‰": 509,
"ํ—˜": 510,
"ํ˜€": 511,
"ํ˜„": 512,
"ํ˜•": 513,
"ํ˜ธ": 514,
"ํ™”": 515,
"ํ™•": 516,
"ํ™˜": 517,
"ํ™œ": 518,
"ํšŒ": 519,
"ํš": 520,
"ํšจ": 521,
"ํ›„": 522,
"ํ›Œ": 523,
"ํ›จ": 524,
"ํœ˜": 525,
"ํžˆ": 526,
"ํž˜": 527
}