ndeclarke commited on
Commit
7effc69
1 Parent(s): 774231b

Upload tokenizer

Browse files
Files changed (4) hide show
  1. README.md +3 -3
  2. added_tokens.json +2 -2
  3. tokenizer_config.json +5 -6
  4. vocab.json +56 -0
README.md CHANGED
@@ -1,11 +1,11 @@
1
  ---
 
 
 
2
  library_name: transformers
3
  license: cc-by-nc-4.0
4
- base_model: facebook/mms-1b-all
5
  tags:
6
  - generated_from_trainer
7
- datasets:
8
- - common_voice_17_0
9
  model-index:
10
  - name: wav2vec2-mms-1b-CV17.0
11
  results: []
 
1
  ---
2
+ base_model: facebook/mms-1b-all
3
+ datasets:
4
+ - common_voice_17_0
5
  library_name: transformers
6
  license: cc-by-nc-4.0
 
7
  tags:
8
  - generated_from_trainer
 
 
9
  model-index:
10
  - name: wav2vec2-mms-1b-CV17.0
11
  results: []
added_tokens.json CHANGED
@@ -1,4 +1,4 @@
1
  {
2
- "</s>": 54,
3
- "<s>": 53
4
  }
 
1
  {
2
+ "</s>": 55,
3
+ "<s>": 54
4
  }
tokenizer_config.json CHANGED
@@ -1,6 +1,6 @@
1
  {
2
  "added_tokens_decoder": {
3
- "51": {
4
  "content": "[UNK]",
5
  "lstrip": true,
6
  "normalized": false,
@@ -8,7 +8,7 @@
8
  "single_word": false,
9
  "special": false
10
  },
11
- "52": {
12
  "content": "[PAD]",
13
  "lstrip": true,
14
  "normalized": false,
@@ -16,7 +16,7 @@
16
  "single_word": false,
17
  "special": false
18
  },
19
- "53": {
20
  "content": "<s>",
21
  "lstrip": false,
22
  "normalized": false,
@@ -24,7 +24,7 @@
24
  "single_word": false,
25
  "special": true
26
  },
27
- "54": {
28
  "content": "</s>",
29
  "lstrip": false,
30
  "normalized": false,
@@ -39,9 +39,8 @@
39
  "eos_token": "</s>",
40
  "model_max_length": 1000000000000000019884624838656,
41
  "pad_token": "[PAD]",
42
- "processor_class": "Wav2Vec2Processor",
43
  "replace_word_delimiter_char": " ",
44
- "target_lang": "tam",
45
  "tokenizer_class": "Wav2Vec2CTCTokenizer",
46
  "unk_token": "[UNK]",
47
  "word_delimiter_token": "|"
 
1
  {
2
  "added_tokens_decoder": {
3
+ "52": {
4
  "content": "[UNK]",
5
  "lstrip": true,
6
  "normalized": false,
 
8
  "single_word": false,
9
  "special": false
10
  },
11
+ "53": {
12
  "content": "[PAD]",
13
  "lstrip": true,
14
  "normalized": false,
 
16
  "single_word": false,
17
  "special": false
18
  },
19
+ "54": {
20
  "content": "<s>",
21
  "lstrip": false,
22
  "normalized": false,
 
24
  "single_word": false,
25
  "special": true
26
  },
27
+ "55": {
28
  "content": "</s>",
29
  "lstrip": false,
30
  "normalized": false,
 
39
  "eos_token": "</s>",
40
  "model_max_length": 1000000000000000019884624838656,
41
  "pad_token": "[PAD]",
 
42
  "replace_word_delimiter_char": " ",
43
+ "target_lang": "tel",
44
  "tokenizer_class": "Wav2Vec2CTCTokenizer",
45
  "unk_token": "[UNK]",
46
  "word_delimiter_token": "|"
vocab.json CHANGED
@@ -128,5 +128,61 @@
128
  "’": 48,
129
  "●": 49,
130
  "◯": 50
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
131
  }
132
  }
 
128
  "’": 48,
129
  "●": 49,
130
  "◯": 50
131
+ },
132
+ "tel": {
133
+ "[PAD]": 53,
134
+ "[UNK]": 52,
135
+ "|": 0,
136
+ "ం": 1,
137
+ "అ": 2,
138
+ "ఆ": 3,
139
+ "ఇ": 4,
140
+ "ఈ": 5,
141
+ "ఉ": 6,
142
+ "ఊ": 7,
143
+ "ఎ": 8,
144
+ "ఏ": 9,
145
+ "ఒ": 10,
146
+ "క": 11,
147
+ "ఖ": 12,
148
+ "గ": 13,
149
+ "ఘ": 14,
150
+ "చ": 15,
151
+ "జ": 16,
152
+ "ట": 17,
153
+ "డ": 18,
154
+ "ణ": 19,
155
+ "త": 20,
156
+ "థ": 21,
157
+ "ద": 22,
158
+ "ధ": 23,
159
+ "న": 24,
160
+ "ప": 25,
161
+ "ఫ": 26,
162
+ "బ": 27,
163
+ "భ": 28,
164
+ "మ": 29,
165
+ "య": 30,
166
+ "ర": 31,
167
+ "ల": 32,
168
+ "ళ": 33,
169
+ "వ": 34,
170
+ "శ": 35,
171
+ "ష": 36,
172
+ "స": 37,
173
+ "హ": 38,
174
+ "ా": 39,
175
+ "ి": 40,
176
+ "ీ": 41,
177
+ "ు": 42,
178
+ "ూ": 43,
179
+ "ృ": 44,
180
+ "ె": 45,
181
+ "ే": 46,
182
+ "ై": 47,
183
+ "ొ": 48,
184
+ "ో": 49,
185
+ "ౌ": 50,
186
+ "్": 51
187
  }
188
  }