seny1004
/

wav2vec2-large-mms-korean-colab

Model card Files Files and versions Community

seny1004 commited on Jul 17, 2023

Commit

af70e45

•

1 Parent(s): 1622ebb

Upload tokenizer

Browse files

Files changed (4) hide show

added_tokens.json +4 -0
special_tokens_map.json +22 -0
tokenizer_config.json +13 -0
vocab.json +359 -0

added_tokens.json ADDED Viewed

	@@ -0,0 +1,4 @@

+{
+  "</s>": 356,
+  "<s>": 355
+}

special_tokens_map.json ADDED Viewed

	@@ -0,0 +1,22 @@

+{
+  "additional_special_tokens": [
+    {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false
+    }
+  ],
+  "bos_token": "<s>",
+  "eos_token": "</s>",
+  "pad_token": "[PAD]",
+  "unk_token": "[UNK]"
+}

tokenizer_config.json ADDED Viewed

	@@ -0,0 +1,13 @@

+{
+  "bos_token": "<s>",
+  "clean_up_tokenization_spaces": true,
+  "do_lower_case": false,
+  "eos_token": "</s>",
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "[PAD]",
+  "replace_word_delimiter_char": " ",
+  "target_lang": "kor",
+  "tokenizer_class": "Wav2Vec2CTCTokenizer",
+  "unk_token": "[UNK]",
+  "word_delimiter_token": "|"
+}

vocab.json ADDED Viewed

	@@ -0,0 +1,359 @@

+{
+  "kor": {
+    "*": 1,
+    "0": 2,
+    "1": 3,
+    "2": 4,
+    "3": 5,
+    "4": 6,
+    "5": 7,
+    "6": 8,
+    "7": 9,
+    "8": 10,
+    "9": 11,
+    "[PAD]": 354,
+    "[UNK]": 353,
+    "a": 12,
+    "b": 13,
+    "c": 14,
+    "d": 15,
+    "e": 16,
+    "f": 17,
+    "g": 18,
+    "h": 19,
+    "i": 20,
+    "j": 21,
+    "k": 22,
+    "l": 23,
+    "m": 24,
+    "n": 25,
+    "o": 26,
+    "p": 27,
+    "q": 28,
+    "r": 29,
+    "s": 30,
+    "t": 31,
+    "u": 32,
+    "v": 33,
+    "w": 34,
+    "x": 35,
+    "y": 36,
+    "|": 0,
+    "ㄱ": 37,
+    "ㄲ": 38,
+    "ㄳ": 39,
+    "ㄴ": 40,
+    "ㄵ": 41,
+    "ㄶ": 42,
+    "ㄷ": 43,
+    "ㄹ": 44,
+    "ㄺ": 45,
+    "ㄻ": 46,
+    "ㄼ": 47,
+    "ㄾ": 48,
+    "ㅀ": 49,
+    "ㅁ": 50,
+    "ㅂ": 51,
+    "ㅄ": 52,
+    "ㅅ": 53,
+    "ㅆ": 54,
+    "ㅇ": 55,
+    "ㅈ": 56,
+    "ㅊ": 57,
+    "ㅋ": 58,
+    "ㅌ": 59,
+    "ㅍ": 60,
+    "ㅎ": 61,
+    "가": 62,
+    "개": 63,
+    "갸": 64,
+    "걔": 65,
+    "거": 66,
+    "게": 67,
+    "겨": 68,
+    "계": 69,
+    "고": 70,
+    "과": 71,
+    "괘": 72,
+    "괴": 73,
+    "교": 74,
+    "구": 75,
+    "궈": 76,
+    "궤": 77,
+    "귀": 78,
+    "규": 79,
+    "그": 80,
+    "기": 81,
+    "까": 82,
+    "깨": 83,
+    "꺠": 84,
+    "꺼": 85,
+    "께": 86,
+    "껴": 87,
+    "꼐": 88,
+    "꼬": 89,
+    "꽈": 90,
+    "꽤": 91,
+    "꾀": 92,
+    "꾸": 93,
+    "꿔": 94,
+    "꿰": 95,
+    "뀌": 96,
+    "끄": 97,
+    "끼": 98,
+    "나": 99,
+    "내": 100,
+    "냐": 101,
+    "냬": 102,
+    "너": 103,
+    "네": 104,
+    "녀": 105,
+    "녜": 106,
+    "노": 107,
+    "놔": 108,
+    "뇌": 109,
+    "뇨": 110,
+    "누": 111,
+    "눠": 112,
+    "뉘": 113,
+    "뉴": 114,
+    "느": 115,
+    "늬": 116,
+    "니": 117,
+    "다": 118,
+    "대": 119,
+    "더": 120,
+    "데": 121,
+    "뎌": 122,
+    "도": 123,
+    "돼": 124,
+    "되": 125,
+    "두": 126,
+    "둬": 127,
+    "뒤": 128,
+    "듀": 129,
+    "드": 130,
+    "디": 131,
+    "따": 132,
+    "때": 133,
+    "떄": 134,
+    "떠": 135,
+    "떼": 136,
+    "뗘": 137,
+    "또": 138,
+    "뚜": 139,
+    "뛰": 140,
+    "뜨": 141,
+    "띄": 142,
+    "띠": 143,
+    "라": 144,
+    "래": 145,
+    "랴": 146,
+    "러": 147,
+    "레": 148,
+    "려": 149,
+    "례": 150,
+    "로": 151,
+    "뢰": 152,
+    "료": 153,
+    "루": 154,
+    "뤄": 155,
+    "류": 156,
+    "르": 157,
+    "리": 158,
+    "마": 159,
+    "매": 160,
+    "머": 161,
+    "메": 162,
+    "며": 163,
+    "모": 164,
+    "묘": 165,
+    "무": 166,
+    "뭐": 167,
+    "뮤": 168,
+    "므": 169,
+    "미": 170,
+    "바": 171,
+    "배": 172,
+    "버": 173,
+    "베": 174,
+    "벼": 175,
+    "보": 176,
+    "봐": 177,
+    "봬": 178,
+    "뵈": 179,
+    "부": 180,
+    "뷔": 181,
+    "뷰": 182,
+    "브": 183,
+    "비": 184,
+    "빠": 185,
+    "빼": 186,
+    "뻐": 187,
+    "뼈": 188,
+    "뽀": 189,
+    "뾰": 190,
+    "뿌": 191,
+    "쁘": 192,
+    "삐": 193,
+    "사": 194,
+    "새": 195,
+    "샤": 196,
+    "섀": 197,
+    "서": 198,
+    "세": 199,
+    "셔": 200,
+    "셰": 201,
+    "소": 202,
+    "쇄": 203,
+    "쇠": 204,
+    "쇼": 205,
+    "수": 206,
+    "숴": 207,
+    "쉐": 208,
+    "쉬": 209,
+    "슈": 210,
+    "스": 211,
+    "시": 212,
+    "싸": 213,
+    "쌔": 214,
+    "쌰": 215,
+    "써": 216,
+    "쎄": 217,
+    "쏘": 218,
+    "쏴": 219,
+    "쐐": 220,
+    "쑤": 221,
+    "쓰": 222,
+    "씌": 223,
+    "씨": 224,
+    "아": 225,
+    "애": 226,
+    "야": 227,
+    "얘": 228,
+    "어": 229,
+    "에": 230,
+    "여": 231,
+    "예": 232,
+    "오": 233,
+    "와": 234,
+    "왜": 235,
+    "외": 236,
+    "요": 237,
+    "우": 238,
+    "워": 239,
+    "웨": 240,
+    "위": 241,
+    "유": 242,
+    "으": 243,
+    "의": 244,
+    "이": 245,
+    "자": 246,
+    "재": 247,
+    "쟈": 248,
+    "쟤": 249,
+    "저": 250,
+    "제": 251,
+    "져": 252,
+    "조": 253,
+    "좌": 254,
+    "죄": 255,
+    "죠": 256,
+    "주": 257,
+    "줘": 258,
+    "쥐": 259,
+    "쥬": 260,
+    "즈": 261,
+    "지": 262,
+    "짜": 263,
+    "째": 264,
+    "쨰": 265,
+    "쩌": 266,
+    "쩨": 267,
+    "쪄": 268,
+    "쪼": 269,
+    "쫘": 270,
+    "쬐": 271,
+    "쭈": 272,
+    "쭤": 273,
+    "쯔": 274,
+    "찌": 275,
+    "차": 276,
+    "채": 277,
+    "처": 278,
+    "체": 279,
+    "쳐": 280,
+    "초": 281,
+    "촤": 282,
+    "최": 283,
+    "쵸": 284,
+    "추": 285,
+    "춰": 286,
+    "췌": 287,
+    "취": 288,
+    "츄": 289,
+    "츠": 290,
+    "치": 291,
+    "카": 292,
+    "캐": 293,
+    "캬": 294,
+    "커": 295,
+    "케": 296,
+    "켜": 297,
+    "코": 298,
+    "콰": 299,
+    "쾌": 300,
+    "쿄": 301,
+    "쿠": 302,
+    "쿼": 303,
+    "퀘": 304,
+    "퀴": 305,
+    "큐": 306,
+    "크": 307,
+    "키": 308,
+    "타": 309,
+    "태": 310,
+    "터": 311,
+    "테": 312,
+    "텨": 313,
+    "토": 314,
+    "퇴": 315,
+    "투": 316,
+    "튀": 317,
+    "튜": 318,
+    "트": 319,
+    "티": 320,
+    "파": 321,
+    "패": 322,
+    "퍼": 323,
+    "페": 324,
+    "펴": 325,
+    "폐": 326,
+    "포": 327,
+    "표": 328,
+    "푸": 329,
+    "퓨": 330,
+    "프": 331,
+    "피": 332,
+    "하": 333,
+    "해": 334,
+    "햐": 335,
+    "햬": 336,
+    "허": 337,
+    "헤": 338,
+    "혀": 339,
+    "혜": 340,
+    "호": 341,
+    "화": 342,
+    "홰": 343,
+    "회": 344,
+    "효": 345,
+    "후": 346,
+    "훠": 347,
+    "휘": 348,
+    "휴": 349,
+    "흐": 350,
+    "희": 351,
+    "히": 352
+  }
+}