jaeyeon commited on
Commit
b280fc8
โ€ข
1 Parent(s): 14c9785

add tokenizer

Browse files
Files changed (4) hide show
  1. added_tokens.json +4 -0
  2. special_tokens_map.json +22 -0
  3. tokenizer_config.json +12 -0
  4. vocab.json +532 -0
added_tokens.json ADDED
@@ -0,0 +1,4 @@
 
 
 
 
 
1
+ {
2
+ "</s>": 531,
3
+ "<s>": 530
4
+ }
special_tokens_map.json ADDED
@@ -0,0 +1,22 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "additional_special_tokens": [
3
+ {
4
+ "content": "<s>",
5
+ "lstrip": false,
6
+ "normalized": true,
7
+ "rstrip": false,
8
+ "single_word": false
9
+ },
10
+ {
11
+ "content": "</s>",
12
+ "lstrip": false,
13
+ "normalized": true,
14
+ "rstrip": false,
15
+ "single_word": false
16
+ }
17
+ ],
18
+ "bos_token": "<s>",
19
+ "eos_token": "</s>",
20
+ "pad_token": "[PAD]",
21
+ "unk_token": "[UNK]"
22
+ }
tokenizer_config.json ADDED
@@ -0,0 +1,12 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "bos_token": "<s>",
3
+ "do_lower_case": false,
4
+ "eos_token": "</s>",
5
+ "name_or_path": "./",
6
+ "pad_token": "[PAD]",
7
+ "replace_word_delimiter_char": " ",
8
+ "special_tokens_map_file": null,
9
+ "tokenizer_class": "Wav2Vec2CTCTokenizer",
10
+ "unk_token": "[UNK]",
11
+ "word_delimiter_token": "|"
12
+ }
vocab.json ADDED
@@ -0,0 +1,532 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "[PAD]": 529,
3
+ "[UNK]": 528,
4
+ "|": 0,
5
+ "๊ฐ€": 1,
6
+ "๊ฐ": 2,
7
+ "๊ฐ„": 3,
8
+ "๊ฐˆ": 4,
9
+ "๊ฐ": 5,
10
+ "๊ฐ‘": 6,
11
+ "๊ฐ’": 7,
12
+ "๊ฐ•": 8,
13
+ "๊ฐ–": 9,
14
+ "๊ฐ™": 10,
15
+ "๊ฐœ": 11,
16
+ "๊ฑฐ": 12,
17
+ "๊ฑฑ": 13,
18
+ "๊ฑด": 14,
19
+ "๊ฑธ": 15,
20
+ "๊ฒ": 16,
21
+ "๊ฒƒ": 17,
22
+ "๊ฒŒ": 18,
23
+ "๊ฒ ": 19,
24
+ "๊ฒจ": 20,
25
+ "๊ฒฉ": 21,
26
+ "๊ฒฐ": 22,
27
+ "๊ฒน": 23,
28
+ "๊ฒฝ": 24,
29
+ "๊ณ„": 25,
30
+ "๊ณ ": 26,
31
+ "๊ณก": 27,
32
+ "๊ณฑ": 28,
33
+ "๊ณต": 29,
34
+ "๊ณผ": 30,
35
+ "๊ด€": 31,
36
+ "๊ด„": 32,
37
+ "๊ดœ": 33,
38
+ "๊ต‰": 34,
39
+ "๊ต": 35,
40
+ "๊ตฌ": 36,
41
+ "๊ตญ": 37,
42
+ "๊ถค": 38,
43
+ "๊ทœ": 39,
44
+ "๊ทธ": 40,
45
+ "๊ทน": 41,
46
+ "๊ทผ": 42,
47
+ "๊ธ€": 43,
48
+ "๊ธˆ": 44,
49
+ "๊ธฐ": 45,
50
+ "๊ธด": 46,
51
+ "๊ธธ": 47,
52
+ "๊น€": 48,
53
+ "๊น": 49,
54
+ "๊นŠ": 50,
55
+ "๊นŒ": 51,
56
+ "๊น": 52,
57
+ "๊ผญ": 53,
58
+ "๊ผด": 54,
59
+ "๊พธ": 55,
60
+ "๊ฟฐ": 56,
61
+ "๋€Œ": 57,
62
+ "๋€": 58,
63
+ "๋ˆ": 59,
64
+ "๋Š": 60,
65
+ "๋": 61,
66
+ "๋ผ": 62,
67
+ "๋‚Œ": 63,
68
+ "๋‚˜": 64,
69
+ "๋‚™": 65,
70
+ "๋‚œ": 66,
71
+ "๋‚ ": 67,
72
+ "๋‚ฉ": 68,
73
+ "๋‚ฌ": 69,
74
+ "๋‚ด": 70,
75
+ "๋ƒˆ": 71,
76
+ "๋ƒ": 72,
77
+ "๋ƒฅ": 73,
78
+ "๋„ˆ": 74,
79
+ "๋„“": 75,
80
+ "๋„˜": 76,
81
+ "๋„ฃ": 77,
82
+ "๋„ค": 78,
83
+ "๋…„": 79,
84
+ "๋…": 80,
85
+ "๋…ธ": 81,
86
+ "๋…ผ": 82,
87
+ "๋†’": 83,
88
+ "๋†“": 84,
89
+ "๋†จ": 85,
90
+ "๋ˆ„": 86,
91
+ "๋ˆˆ": 87,
92
+ "๋ˆ ": 88,
93
+ "๋‰œ": 89,
94
+ "๋‰ด": 90,
95
+ "๋Š": 91,
96
+ "๋Š”": 92,
97
+ "๋Š˜": 93,
98
+ "๋Šฅ": 94,
99
+ "๋‹ˆ": 95,
100
+ "๋‹Œ": 96,
101
+ "๋‹": 97,
102
+ "๋‹˜": 98,
103
+ "๋‹™": 99,
104
+ "๋‹ค": 100,
105
+ "๋‹จ": 101,
106
+ "๋‹ฌ": 102,
107
+ "๋‹ฎ": 103,
108
+ "๋‹ด": 104,
109
+ "๋‹ต": 105,
110
+ "๋‹น": 106,
111
+ "๋Œ€": 107,
112
+ "๋Œˆ": 108,
113
+ "๋Œ“": 109,
114
+ "๋”": 110,
115
+ "๋˜": 111,
116
+ "๋Ÿ": 112,
117
+ "๋ฐ": 113,
118
+ "๋„": 114,
119
+ "๋…": 115,
120
+ "๋Œ": 116,
121
+ "๋™": 117,
122
+ "๋ผ": 118,
123
+ "๋": 119,
124
+ "๋˜": 120,
125
+ "๋œ": 121,
126
+ "๋ ": 122,
127
+ "๋ฉ": 123,
128
+ "๋‘": 124,
129
+ "๋‘˜": 125,
130
+ "๋‘ฅ": 126,
131
+ "๋’ค": 127,
132
+ "๋“œ": 128,
133
+ "๋“ ": 129,
134
+ "๋“ฃ": 130,
135
+ "๋“ค": 131,
136
+ "๋“ฌ": 132,
137
+ "๋“ญ": 133,
138
+ "๋“ฑ": 134,
139
+ "๋””": 135,
140
+ "๋”ฐ": 136,
141
+ "๋”ฑ": 137,
142
+ "๋•Œ": 138,
143
+ "๋– ": 139,
144
+ "๋–ค": 140,
145
+ "๋–จ": 141,
146
+ "๋–ป": 142,
147
+ "๋˜": 143,
148
+ "๋˜‘": 144,
149
+ "๋˜ฅ": 145,
150
+ "๋šฑ": 146,
151
+ "๋œป": 147,
152
+ "๋ผ": 148,
153
+ "๋ž€": 149,
154
+ "๋ž„": 150,
155
+ "๋žŒ": 151,
156
+ "๋ž": 152,
157
+ "๋ž‘": 153,
158
+ "๋ž˜": 154,
159
+ "๋žœ": 155,
160
+ "๋žต": 156,
161
+ "๋Ÿ‰": 157,
162
+ "๋Ÿฌ": 158,
163
+ "๋Ÿฐ": 159,
164
+ "๋Ÿด": 160,
165
+ "๋Ÿผ": 161,
166
+ "๋Ÿฝ": 162,
167
+ "๋ ‡": 163,
168
+ "๋ ˆ": 164,
169
+ "๋ ค": 165,
170
+ "๋ ฅ": 166,
171
+ "๋ จ": 167,
172
+ "๋ ต": 168,
173
+ "๋ ธ": 169,
174
+ "๋ น": 170,
175
+ "๋ก€": 171,
176
+ "๋กœ": 172,
177
+ "๋ก ": 173,
178
+ "๋กญ": 174,
179
+ "๋ฃŒ": 175,
180
+ "๋ฃจ": 176,
181
+ "๋ฅ˜": 177,
182
+ "๋ฅ ": 178,
183
+ "๋ฅจ": 179,
184
+ "๋ฅญ": 180,
185
+ "๋ฅด": 181,
186
+ "๋ฅต": 182,
187
+ "๋ฅธ": 183,
188
+ "๋ฅผ": 184,
189
+ "๋ฆ„": 185,
190
+ "๋ฆ…": 186,
191
+ "๋ฆฌ": 187,
192
+ "๋ฆญ": 188,
193
+ "๋ฆฐ": 189,
194
+ "๋ฆผ": 190,
195
+ "๋ฆฝ": 191,
196
+ "๋งˆ": 192,
197
+ "๋ง‰": 193,
198
+ "๋งŒ": 194,
199
+ "๋งŽ": 195,
200
+ "๋ง": 196,
201
+ "๋ง˜": 197,
202
+ "๋งž": 198,
203
+ "๋งฃ": 199,
204
+ "๋งฅ": 200,
205
+ "๋จธ": 201,
206
+ "๋จน": 202,
207
+ "๋จผ": 203,
208
+ "๋ฉˆ": 204,
209
+ "๋ฉ”": 205,
210
+ "๋ฉ˜": 206,
211
+ "๋ฉฉ": 207,
212
+ "๋ฉด": 208,
213
+ "๋ช…": 209,
214
+ "๋ช‡": 210,
215
+ "๋ชจ": 211,
216
+ "๋ชฉ": 212,
217
+ "๋ชฐ": 213,
218
+ "๋ชป": 214,
219
+ "๋ฌด": 215,
220
+ "๋ฌธ": 216,
221
+ "๋ฌผ": 217,
222
+ "๋ญ‰": 218,
223
+ "๋ญ": 219,
224
+ "๋ญ˜": 220,
225
+ "๋ญก": 221,
226
+ "๋ฏธ": 222,
227
+ "๋ฏผ": 223,
228
+ "๋ฐ€": 224,
229
+ "๋ฐ”": 225,
230
+ "๋ฐ–": 226,
231
+ "๋ฐ˜": 227,
232
+ "๋ฐ›": 228,
233
+ "๋ฐœ": 229,
234
+ "๋ฐฉ": 230,
235
+ "๋ฐฐ": 231,
236
+ "๋ฐฑ": 232,
237
+ "๋ฒ„": 233,
238
+ "๋ฒˆ": 234,
239
+ "๋ฒŒ": 235,
240
+ "๋ฒ”": 236,
241
+ "๋ฒก": 237,
242
+ "๋ฒฝ": 238,
243
+ "๋ณ€": 239,
244
+ "๋ณ„": 240,
245
+ "๋ณ": 241,
246
+ "๋ณด": 242,
247
+ "๋ณต": 243,
248
+ "๋ณธ": 244,
249
+ "๋ณผ": 245,
250
+ "๋ด": 246,
251
+ "๋ดค": 247,
252
+ "๋ถ€": 248,
253
+ "๋ถ„": 249,
254
+ "๋ถˆ": 250,
255
+ "๋ถ“": 251,
256
+ "๋ถ™": 252,
257
+ "๋น„": 253,
258
+ "๋น ": 254,
259
+ "๋นผ": 255,
260
+ "๋บ": 256,
261
+ "๋ฟ”": 257,
262
+ "์‚ฌ": 258,
263
+ "์‚ฐ": 259,
264
+ "์‚ด": 260,
265
+ "์‚ผ": 261,
266
+ "์ƒ": 262,
267
+ "์ƒˆ": 263,
268
+ "์ƒ‰": 264,
269
+ "์ƒ": 265,
270
+ "์„œ": 266,
271
+ "์„": 267,
272
+ "์„ ": 268,
273
+ "์„ค": 269,
274
+ "์„ฏ": 270,
275
+ "์„ฑ": 271,
276
+ "์„ธ": 272,
277
+ "์…จ": 273,
278
+ "์†Œ": 274,
279
+ "์†": 275,
280
+ "์†": 276,
281
+ "์ˆ˜": 277,
282
+ "์ˆ™": 278,
283
+ "์ˆœ": 279,
284
+ "์ˆซ": 280,
285
+ "์‰ฌ": 281,
286
+ "์Šˆ": 282,
287
+ "์Š‰": 283,
288
+ "์Šค": 284,
289
+ "์Šต": 285,
290
+ "์Šท": 286,
291
+ "์‹œ": 287,
292
+ "์‹": 288,
293
+ "์‹ ": 289,
294
+ "์‹ค": 290,
295
+ "์‹ฌ": 291,
296
+ "์‹ญ": 292,
297
+ "์‹ถ": 293,
298
+ "์‹น": 294,
299
+ "์Œ": 295,
300
+ "์จ": 296,
301
+ "์Ÿ": 297,
302
+ "์“ฐ": 298,
303
+ "์“ด": 299,
304
+ "์“ธ": 300,
305
+ "์”Œ": 301,
306
+ "์”จ": 302,
307
+ "์”ฉ": 303,
308
+ "์”ฌ": 304,
309
+ "์•„": 305,
310
+ "์•ˆ": 306,
311
+ "์•Š": 307,
312
+ "์•Œ": 308,
313
+ "์•˜": 309,
314
+ "์•ž": 310,
315
+ "์• ": 311,
316
+ "์•ผ": 312,
317
+ "์•ฝ": 313,
318
+ "์–‡": 314,
319
+ "์–‘": 315,
320
+ "์–•": 316,
321
+ "์–˜": 317,
322
+ "์–ด": 318,
323
+ "์–ต": 319,
324
+ "์–ธ": 320,
325
+ "์–ผ": 321,
326
+ "์—„": 322,
327
+ "์—…": 323,
328
+ "์—†": 324,
329
+ "์—ˆ": 325,
330
+ "์—": 326,
331
+ "์—‘": 327,
332
+ "์—”": 328,
333
+ "์— ": 329,
334
+ "์—ฌ": 330,
335
+ "์—ญ": 331,
336
+ "์—ฐ": 332,
337
+ "์—ด": 333,
338
+ "์˜€": 334,
339
+ "์˜": 335,
340
+ "์˜ˆ": 336,
341
+ "์˜ค": 337,
342
+ "์˜จ": 338,
343
+ "์˜ฌ": 339,
344
+ "์˜ฎ": 340,
345
+ "์˜ท": 341,
346
+ "์™€": 342,
347
+ "์™„": 343,
348
+ "์™”": 344,
349
+ "์™œ": 345,
350
+ "์™ธ": 346,
351
+ "์™ผ": 347,
352
+ "์š”": 348,
353
+ "์šฉ": 349,
354
+ "์šฐ": 350,
355
+ "์šด": 351,
356
+ "์šธ": 352,
357
+ "์›€": 353,
358
+ "์›": 354,
359
+ "์›Œ": 355,
360
+ "์›": 356,
361
+ "์› ": 357,
362
+ "์œ„": 358,
363
+ "์œ ": 359,
364
+ "์œก": 360,
365
+ "์œจ": 361,
366
+ "์œผ": 362,
367
+ "์€": 363,
368
+ "์„": 364,
369
+ "์Œ": 365,
370
+ "์‘": 366,
371
+ "์˜": 367,
372
+ "์ด": 368,
373
+ "์ต": 369,
374
+ "์ธ": 370,
375
+ "์ผ": 371,
376
+ "์ฝ": 372,
377
+ "์ž„": 373,
378
+ "์ž…": 374,
379
+ "์žˆ": 375,
380
+ "์žŠ": 376,
381
+ "์ž": 377,
382
+ "์ž‘": 378,
383
+ "์ž–": 379,
384
+ "์ž˜": 380,
385
+ "์ž ": 381,
386
+ "์žก": 382,
387
+ "์žฅ": 383,
388
+ "์žฌ": 384,
389
+ "์ €": 385,
390
+ "์ ": 386,
391
+ "์ „": 387,
392
+ "์ ˆ": 388,
393
+ "์ ": 389,
394
+ "์ ‘": 390,
395
+ "์ •": 391,
396
+ "์ œ": 392,
397
+ "์ ธ": 393,
398
+ "์กŒ": 394,
399
+ "์กฐ": 395,
400
+ "์กฑ": 396,
401
+ "์กด": 397,
402
+ "์กธ": 398,
403
+ "์ข€": 399,
404
+ "์ข…": 400,
405
+ "์ข‹": 401,
406
+ "์ขŒ": 402,
407
+ "์ฃ ": 403,
408
+ "์ฃผ": 404,
409
+ "์ค€": 405,
410
+ "์ค„": 406,
411
+ "์ค‘": 407,
412
+ "์ฆ": 408,
413
+ "์ง€": 409,
414
+ "์ง": 410,
415
+ "์ง„": 411,
416
+ "์งˆ": 412,
417
+ "์ง": 413,
418
+ "์ง‘": 414,
419
+ "์ง“": 415,
420
+ "์ง•": 416,
421
+ "์งœ": 417,
422
+ "์ง": 418,
423
+ "์งง": 419,
424
+ "์งฑ": 420,
425
+ "์งธ": 421,
426
+ "์จŒ": 422,
427
+ "์ฉœ": 423,
428
+ "์ชผ": 424,
429
+ "์ชฝ": 425,
430
+ "์ซ™": 426,
431
+ "์ญ‰": 427,
432
+ "์ฏค": 428,
433
+ "์ฐŒ": 429,
434
+ "์ฐ": 430,
435
+ "์ฐจ": 431,
436
+ "์ฐฎ": 432,
437
+ "์ฐธ": 433,
438
+ "์ฑ„": 434,
439
+ "์ฑ…": 435,
440
+ "์ฒ˜": 436,
441
+ "์ฒœ": 437,
442
+ "์ฒ ": 438,
443
+ "์ฒซ": 439,
444
+ "์ฒญ": 440,
445
+ "์ฒด": 441,
446
+ "์ณค": 442,
447
+ "์ดˆ": 443,
448
+ "์ด": 444,
449
+ "์ตœ": 445,
450
+ "์ถ”": 446,
451
+ "์ถ•": 447,
452
+ "์ถœ": 448,
453
+ "์ถฉ": 449,
454
+ "์ถฐ": 450,
455
+ "์น˜": 451,
456
+ "์น™": 452,
457
+ "์นœ": 453,
458
+ "์น ": 454,
459
+ "์นจ": 455,
460
+ "์นด": 456,
461
+ "์นธ": 457,
462
+ "์ปค": 458,
463
+ "์ปด": 459,
464
+ "์ผ€": 460,
465
+ "์ฝ”": 461,
466
+ "ํ€„": 462,
467
+ "ํฌ": 463,
468
+ "ํฐ": 464,
469
+ "ํด": 465,
470
+ "ํผ": 466,
471
+ "ํ‚ฌ": 467,
472
+ "ํƒ€": 468,
473
+ "ํƒœ": 469,
474
+ "ํƒ": 470,
475
+ "ํ„ฐ": 471,
476
+ "ํ…Œ": 472,
477
+ "ํ…": 473,
478
+ "ํ† ": 474,
479
+ "ํ†ต": 475,
480
+ "ํˆฌ": 476,
481
+ "ํŠธ": 477,
482
+ "ํŠน": 478,
483
+ "ํŠผ": 479,
484
+ "ํ‹€": 480,
485
+ "ํ‹ฐ": 481,
486
+ "ํŒŒ": 482,
487
+ "ํŒ”": 483,
488
+ "ํŽ˜": 484,
489
+ "ํŽด": 485,
490
+ "ํŽธ": 486,
491
+ "ํŽผ": 487,
492
+ "ํ…": 488,
493
+ "ํ‰": 489,
494
+ "ํฌ": 490,
495
+ "ํด": 491,
496
+ "ํ‘œ": 492,
497
+ "ํ‘ธ": 493,
498
+ "ํ‘ผ": 494,
499
+ "ํ’€": 495,
500
+ "ํ”„": 496,
501
+ "ํ”Œ": 497,
502
+ "ํ”ผ": 498,
503
+ "ํ•„": 499,
504
+ "ํ•˜": 500,
505
+ "ํ•™": 501,
506
+ "ํ•œ": 502,
507
+ "ํ• ": 503,
508
+ "ํ•จ": 504,
509
+ "ํ•ฉ": 505,
510
+ "ํ•ญ": 506,
511
+ "ํ•ด": 507,
512
+ "ํ–ˆ": 508,
513
+ "ํ–‰": 509,
514
+ "ํ—˜": 510,
515
+ "ํ˜€": 511,
516
+ "ํ˜„": 512,
517
+ "ํ˜•": 513,
518
+ "ํ˜ธ": 514,
519
+ "ํ™”": 515,
520
+ "ํ™•": 516,
521
+ "ํ™˜": 517,
522
+ "ํ™œ": 518,
523
+ "ํšŒ": 519,
524
+ "ํš": 520,
525
+ "ํšจ": 521,
526
+ "ํ›„": 522,
527
+ "ํ›Œ": 523,
528
+ "ํ›จ": 524,
529
+ "ํœ˜": 525,
530
+ "ํžˆ": 526,
531
+ "ํž˜": 527
532
+ }