add tokenizer

Browse files

Files changed (4) hide show

added_tokens.json +4 -0
special_tokens_map.json +22 -0
tokenizer_config.json +12 -0
vocab.json +510 -0

added_tokens.json ADDED Viewed

	@@ -0,0 +1,4 @@

+{
+  "</s>": 509,
+  "<s>": 508
+}

special_tokens_map.json ADDED Viewed

	@@ -0,0 +1,22 @@

+{
+  "additional_special_tokens": [
+    {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false
+    }
+  ],
+  "bos_token": "<s>",
+  "eos_token": "</s>",
+  "pad_token": "[PAD]",
+  "unk_token": "[UNK]"
+}

tokenizer_config.json ADDED Viewed

	@@ -0,0 +1,12 @@

+{
+  "bos_token": "<s>",
+  "do_lower_case": false,
+  "eos_token": "</s>",
+  "name_or_path": "./",
+  "pad_token": "[PAD]",
+  "replace_word_delimiter_char": " ",
+  "special_tokens_map_file": null,
+  "tokenizer_class": "Wav2Vec2CTCTokenizer",
+  "unk_token": "[UNK]",
+  "word_delimiter_token": "|"
+}

vocab.json ADDED Viewed

	@@ -0,0 +1,510 @@

+{
+  "[PAD]": 507,
+  "[UNK]": 506,
+  "|": 0,
+  "ㄱ": 1,
+  "ㄴ": 2,
+  "ㄷ": 3,
+  "ㄹ": 4,
+  "ㅁ": 5,
+  "ㅂ": 6,
+  "가": 7,
+  "각": 8,
+  "간": 9,
+  "갈": 10,
+  "감": 11,
+  "갑": 12,
+  "값": 13,
+  "갔": 14,
+  "강": 15,
+  "같": 16,
+  "개": 17,
+  "갯": 18,
+  "거": 19,
+  "건": 20,
+  "걸": 21,
+  "검": 22,
+  "겁": 23,
+  "것": 24,
+  "게": 25,
+  "겠": 26,
+  "겨": 27,
+  "결": 28,
+  "경": 29,
+  "계": 30,
+  "고": 31,
+  "곡": 32,
+  "곤": 33,
+  "골": 34,
+  "곱": 35,
+  "곳": 36,
+  "공": 37,
+  "과": 38,
+  "관": 39,
+  "굉": 40,
+  "교": 41,
+  "구": 42,
+  "국": 43,
+  "굳": 44,
+  "궁": 45,
+  "귀": 46,
+  "귓": 47,
+  "규": 48,
+  "그": 49,
+  "근": 50,
+  "귿": 51,
+  "금": 52,
+  "급": 53,
+  "기": 54,
+  "긴": 55,
+  "길": 56,
+  "김": 57,
+  "까": 58,
+  "깐": 59,
+  "깥": 60,
+  "꺼": 61,
+  "껏": 62,
+  "께": 63,
+  "꼭": 64,
+  "꼼": 65,
+  "꾸": 66,
+  "꾼": 67,
+  "꿔": 68,
+  "뀌": 69,
+  "끊": 70,
+  "끓": 71,
+  "끝": 72,
+  "끼": 73,
+  "나": 74,
+  "난": 75,
+  "날": 76,
+  "납": 77,
+  "낫": 78,
+  "났": 79,
+  "낮": 80,
+  "내": 81,
+  "낸": 82,
+  "낼": 83,
+  "냄": 84,
+  "냈": 85,
+  "냐": 86,
+  "너": 87,
+  "넣": 88,
+  "네": 89,
+  "넷": 90,
+  "념": 91,
+  "녕": 92,
+  "녹": 93,
+  "농": 94,
+  "높": 95,
+  "놓": 96,
+  "놨": 97,
+  "누": 98,
+  "눈": 99,
+  "눌": 100,
+  "느": 101,
+  "는": 102,
+  "늘": 103,
+  "니": 104,
+  "닌": 105,
+  "님": 106,
+  "다": 107,
+  "단": 108,
+  "달": 109,
+  "담": 110,
+  "답": 111,
+  "닷": 112,
+  "당": 113,
+  "대": 114,
+  "더": 115,
+  "던": 116,
+  "덩": 117,
+  "데": 118,
+  "델": 119,
+  "도": 120,
+  "돌": 121,
+  "동": 122,
+  "돼": 123,
+  "됐": 124,
+  "되": 125,
+  "된": 126,
+  "될": 127,
+  "됩": 128,
+  "두": 129,
+  "둔": 130,
+  "둘": 131,
+  "둬": 132,
+  "뒷": 133,
+  "드": 134,
+  "든": 135,
+  "들": 136,
+  "등": 137,
+  "디": 138,
+  "따": 139,
+  "땀": 140,
+  "때": 141,
+  "떠": 142,
+  "떤": 143,
+  "떨": 144,
+  "떻": 145,
+  "또": 146,
+  "똑": 147,
+  "뜨": 148,
+  "뜻": 149,
+  "띄": 150,
+  "라": 151,
+  "란": 152,
+  "람": 153,
+  "랐": 154,
+  "랑": 155,
+  "래": 156,
+  "랜": 157,
+  "랬": 158,
+  "량": 159,
+  "러": 160,
+  "런": 161,
+  "럴": 162,
+  "럼": 163,
+  "렀": 164,
+  "렇": 165,
+  "렌": 166,
+  "려": 167,
+  "력": 168,
+  "련": 169,
+  "렸": 170,
+  "례": 171,
+  "로": 172,
+  "록": 173,
+  "론": 174,
+  "료": 175,
+  "루": 176,
+  "류": 177,
+  "르": 178,
+  "른": 179,
+  "를": 180,
+  "름": 181,
+  "리": 182,
+  "린": 183,
+  "림": 184,
+  "립": 185,
+  "마": 186,
+  "막": 187,
+  "만": 188,
+  "많": 189,
+  "말": 190,
+  "맑": 191,
+  "맞": 192,
+  "맡": 193,
+  "매": 194,
+  "맺": 195,
+  "머": 196,
+  "먹": 197,
+  "먼": 198,
+  "멀": 199,
+  "멉": 200,
+  "며": 201,
+  "면": 202,
+  "명": 203,
+  "몇": 204,
+  "모": 205,
+  "몰": 206,
+  "못": 207,
+  "무": 208,
+  "묶": 209,
+  "문": 210,
+  "묻": 211,
+  "물": 212,
+  "뭐": 213,
+  "미": 214,
+  "밀": 215,
+  "밑": 216,
+  "바": 217,
+  "밖": 218,
+  "반": 219,
+  "받": 220,
+  "발": 221,
+  "방": 222,
+  "배": 223,
+  "백": 224,
+  "번": 225,
+  "법": 226,
+  "벗": 227,
+  "벳": 228,
+  "벼": 229,
+  "벽": 230,
+  "변": 231,
+  "볕": 232,
+  "보": 233,
+  "본": 234,
+  "볼": 235,
+  "봅": 236,
+  "봉": 237,
+  "봐": 238,
+  "봤": 239,
+  "부": 240,
+  "분": 241,
+  "불": 242,
+  "붙": 243,
+  "브": 244,
+  "비": 245,
+  "빈": 246,
+  "빠": 247,
+  "빨": 248,
+  "뺄": 249,
+  "뽀": 250,
+  "뿐": 251,
+  "뿜": 252,
+  "사": 253,
+  "삭": 254,
+  "산": 255,
+  "살": 256,
+  "삼": 257,
+  "상": 258,
+  "새": 259,
+  "색": 260,
+  "샘": 261,
+  "생": 262,
+  "샤": 263,
+  "서": 264,
+  "석": 265,
+  "섞": 266,
+  "선": 267,
+  "설": 268,
+  "섭": 269,
+  "섯": 270,
+  "성": 271,
+  "세": 272,
+  "셔": 273,
+  "셨": 274,
+  "소": 275,
+  "속": 276,
+  "손": 277,
+  "송": 278,
+  "수": 279,
+  "순": 280,
+  "술": 281,
+  "쉬": 282,
+  "쉽": 283,
+  "스": 284,
+  "슨": 285,
+  "슬": 286,
+  "습": 287,
+  "승": 288,
+  "시": 289,
+  "식": 290,
+  "신": 291,
+  "실": 292,
+  "심": 293,
+  "십": 294,
+  "썼": 295,
+  "쓴": 296,
+  "쓸": 297,
+  "씨": 298,
+  "씩": 299,
+  "아": 300,
+  "안": 301,
+  "않": 302,
+  "알": 303,
+  "암": 304,
+  "압": 305,
+  "았": 306,
+  "애": 307,
+  "액": 308,
+  "앤": 309,
+  "야": 310,
+  "약": 311,
+  "양": 312,
+  "얘": 313,
+  "어": 314,
+  "억": 315,
+  "언": 316,
+  "얻": 317,
+  "얼": 318,
+  "없": 319,
+  "엇": 320,
+  "었": 321,
+  "에": 322,
+  "엔": 323,
+  "여": 324,
+  "역": 325,
+  "연": 326,
+  "열": 327,
+  "염": 328,
+  "였": 329,
+  "영": 330,
+  "예": 331,
+  "오": 332,
+  "온": 333,
+  "올": 334,
+  "옳": 335,
+  "옷": 336,
+  "와": 337,
+  "왔": 338,
+  "왜": 339,
+  "외": 340,
+  "요": 341,
+  "용": 342,
+  "우": 343,
+  "운": 344,
+  "울": 345,
+  "움": 346,
+  "워": 347,
+  "원": 348,
+  "월": 349,
+  "웠": 350,
+  "위": 351,
+  "유": 352,
+  "육": 353,
+  "융": 354,
+  "으": 355,
+  "은": 356,
+  "을": 357,
+  "음": 358,
+  "읍": 359,
+  "응": 360,
+  "의": 361,
+  "이": 362,
+  "인": 363,
+  "일": 364,
+  "읽": 365,
+  "임": 366,
+  "입": 367,
+  "있": 368,
+  "잉": 369,
+  "잎": 370,
+  "자": 371,
+  "작": 372,
+  "잘": 373,
+  "잠": 374,
+  "장": 375,
+  "재": 376,
+  "저": 377,
+  "적": 378,
+  "전": 379,
+  "절": 380,
+  "점": 381,
+  "정": 382,
+  "젖": 383,
+  "제": 384,
+  "져": 385,
+  "졌": 386,
+  "조": 387,
+  "존": 388,
+  "좀": 389,
+  "종": 390,
+  "좋": 391,
+  "죠": 392,
+  "주": 393,
+  "준": 394,
+  "줄": 395,
+  "중": 396,
+  "즉": 397,
+  "즐": 398,
+  "증": 399,
+  "지": 400,
+  "직": 401,
+  "진": 402,
+  "질": 403,
+  "집": 404,
+  "짚": 405,
+  "짝": 406,
+  "짭": 407,
+  "짱": 408,
+  "째": 409,
+  "쨍": 410,
+  "쪼": 411,
+  "쫙": 412,
+  "쬐": 413,
+  "쭉": 414,
+  "차": 415,
+  "찬": 416,
+  "찰": 417,
+  "창": 418,
+  "찾": 419,
+  "책": 420,
+  "처": 421,
+  "철": 422,
+  "첫": 423,
+  "체": 424,
+  "쳐": 425,
+  "쵸": 426,
+  "추": 427,
+  "축": 428,
+  "출": 429,
+  "충": 430,
+  "측": 431,
+  "층": 432,
+  "치": 433,
+  "칙": 434,
+  "친": 435,
+  "칠": 436,
+  "칭": 437,
+  "캘": 438,
+  "커": 439,
+  "컵": 440,
+  "컷": 441,
+  "케": 442,
+  "켜": 443,
+  "켰": 444,
+  "코": 445,
+  "큐": 446,
+  "크": 447,
+  "큼": 448,
+  "키": 449,
+  "킨": 450,
+  "킬": 451,
+  "타": 452,
+  "탄": 453,
+  "탈": 454,
+  "탕": 455,
+  "태": 456,
+  "택": 457,
+  "터": 458,
+  "텐": 459,
+  "톤": 460,
+  "통": 461,
+  "투": 462,
+  "특": 463,
+  "틀": 464,
+  "티": 465,
+  "파": 466,
+  "판": 467,
+  "팔": 468,
+  "팽": 469,
+  "퍼": 470,
+  "펴": 471,
+  "편": 472,
+  "포": 473,
+  "표": 474,
+  "풀": 475,
+  "풍": 476,
+  "프": 477,
+  "피": 478,
+  "필": 479,
+  "하": 480,
+  "학": 481,
+  "한": 482,
+  "할": 483,
+  "함": 484,
+  "합": 485,
+  "항": 486,
+  "해": 487,
+  "햇": 488,
+  "했": 489,
+  "향": 490,
+  "험": 491,
+  "현": 492,
+  "형": 493,
+  "화": 494,
+  "확": 495,
+  "활": 496,
+  "황": 497,
+  "회": 498,
+  "횟": 499,
+  "후": 500,
+  "흘": 501,
+  "흡": 502,
+  "히": 503,
+  "힌": 504,
+  "힘": 505
+}