Matthijs commited on
Commit
b2d3929
1 Parent(s): 4933329

Upload tokenizer

Browse files
Files changed (3) hide show
  1. special_tokens_map.json +4 -0
  2. tokenizer_config.json +9 -0
  3. vocab.json +180 -0
special_tokens_map.json ADDED
@@ -0,0 +1,4 @@
 
 
 
 
 
1
+ {
2
+ "pad_token": "<pad>",
3
+ "unk_token": "<unk>"
4
+ }
tokenizer_config.json ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "add_blank": true,
3
+ "clean_up_tokenization_spaces": true,
4
+ "language": null,
5
+ "model_max_length": 1000000000000000019884624838656,
6
+ "pad_token": "<pad>",
7
+ "tokenizer_class": "VitsMmsTokenizer",
8
+ "unk_token": "<unk>"
9
+ }
vocab.json ADDED
@@ -0,0 +1,180 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "": 16,
3
+ "!": 5,
4
+ "\"": 11,
5
+ "'": 176,
6
+ ".": 4,
7
+ ":": 2,
8
+ ";": 1,
9
+ "<pad>": 176,
10
+ "<unk>": 177,
11
+ "?": 6,
12
+ "A": 17,
13
+ "B": 18,
14
+ "C": 19,
15
+ "D": 20,
16
+ "E": 21,
17
+ "F": 22,
18
+ "G": 23,
19
+ "H": 24,
20
+ "I": 25,
21
+ "J": 26,
22
+ "K": 27,
23
+ "L": 28,
24
+ "M": 29,
25
+ "N": 30,
26
+ "O": 31,
27
+ "P": 32,
28
+ "Q": 33,
29
+ "R": 34,
30
+ "S": 35,
31
+ "T": 36,
32
+ "U": 37,
33
+ "V": 38,
34
+ "W": 39,
35
+ "X": 40,
36
+ "Y": 41,
37
+ "Z": 42,
38
+ "_": 0,
39
+ "a": 43,
40
+ "b": 44,
41
+ "c": 45,
42
+ "d": 46,
43
+ "e": 47,
44
+ "f": 48,
45
+ "g": 49,
46
+ "h": 50,
47
+ "i": 51,
48
+ "j": 52,
49
+ "k": 53,
50
+ "l": 54,
51
+ "m": 55,
52
+ "n": 56,
53
+ "o": 57,
54
+ "p": 58,
55
+ "q": 59,
56
+ "r": 60,
57
+ "s": 61,
58
+ "t": 62,
59
+ "u": 63,
60
+ "v": 64,
61
+ "w": 65,
62
+ "x": 66,
63
+ "y": 67,
64
+ "z": 68,
65
+ "¡": 7,
66
+ "«": 12,
67
+ "»": 13,
68
+ "¿": 8,
69
+ "æ": 72,
70
+ "ç": 78,
71
+ "ð": 81,
72
+ "ø": 116,
73
+ "ħ": 98,
74
+ "ŋ": 112,
75
+ "œ": 120,
76
+ "ǀ": 152,
77
+ "ǁ": 153,
78
+ "ǂ": 154,
79
+ "ǃ": 155,
80
+ "ɐ": 70,
81
+ "ɑ": 69,
82
+ "ɒ": 71,
83
+ "ɓ": 73,
84
+ "ɔ": 76,
85
+ "ɕ": 77,
86
+ "ɖ": 80,
87
+ "ɗ": 79,
88
+ "ɘ": 84,
89
+ "ə": 83,
90
+ "ɚ": 85,
91
+ "ɛ": 86,
92
+ "ɜ": 87,
93
+ "ɝ": 88,
94
+ "ɞ": 89,
95
+ "ɟ": 90,
96
+ "ɠ": 93,
97
+ "ɡ": 92,
98
+ "ɢ": 94,
99
+ "ɣ": 139,
100
+ "ɤ": 140,
101
+ "ɥ": 99,
102
+ "ɦ": 96,
103
+ "ɧ": 97,
104
+ "ɨ": 101,
105
+ "ɪ": 102,
106
+ "ɫ": 106,
107
+ "ɬ": 105,
108
+ "ɭ": 104,
109
+ "ɮ": 107,
110
+ "ɯ": 110,
111
+ "ɰ": 111,
112
+ "ɱ": 109,
113
+ "ɲ": 114,
114
+ "ɳ": 113,
115
+ "ɴ": 115,
116
+ "ɵ": 117,
117
+ "ɶ": 121,
118
+ "ɸ": 118,
119
+ "ɹ": 123,
120
+ "ɺ": 124,
121
+ "ɻ": 126,
122
+ "ɽ": 129,
123
+ "ɾ": 125,
124
+ "ʀ": 127,
125
+ "ʁ": 128,
126
+ "ʂ": 130,
127
+ "ʃ": 131,
128
+ "ʄ": 91,
129
+ "ʈ": 132,
130
+ "ʉ": 134,
131
+ "ʊ": 135,
132
+ "ʋ": 136,
133
+ "ʌ": 138,
134
+ "ʍ": 141,
135
+ "ʎ": 143,
136
+ "ʏ": 144,
137
+ "ʐ": 146,
138
+ "ʑ": 145,
139
+ "ʒ": 147,
140
+ "ʔ": 148,
141
+ "ʕ": 150,
142
+ "ʘ": 122,
143
+ "ʙ": 74,
144
+ "ʛ": 95,
145
+ "ʜ": 100,
146
+ "ʝ": 103,
147
+ "ʟ": 108,
148
+ "ʡ": 149,
149
+ "ʢ": 151,
150
+ "ʤ": 82,
151
+ "ʧ": 133,
152
+ "ʰ": 162,
153
+ "ʱ": 163,
154
+ "ʲ": 164,
155
+ "ʴ": 161,
156
+ "ʷ": 165,
157
+ "ʼ": 160,
158
+ "ˈ": 156,
159
+ "ˌ": 157,
160
+ "ː": 158,
161
+ "ˑ": 159,
162
+ "˞": 168,
163
+ "ˠ": 166,
164
+ "ˤ": 167,
165
+ "̩": 175,
166
+ "β": 75,
167
+ "θ": 119,
168
+ "χ": 142,
169
+ "ᵻ": 177,
170
+ "—": 9,
171
+ "“": 14,
172
+ "”": 15,
173
+ "…": 10,
174
+ "↑": 170,
175
+ "→": 171,
176
+ "↓": 169,
177
+ "↗": 172,
178
+ "↘": 173,
179
+ "ⱱ": 137
180
+ }