update tokenizer

added_tokens.json

@@ -0,0 +1 @@
+{"<s>": 8065, "</s>": 8066}

feature_extractor_config.json

@@ -1,13 +0,0 @@
-{
-    "do_normalize": true,
-    "feature_size": 80,
-    "sampling_rate": 16000,
-    "padding_value": 0.0,
-    "hop_length": 10,
-    "win_length": 25,
-    "win_function": "hamming_window",
-    "normalize_means": true,
-    "normalize_vars": true,    
-    "return_attention_mask": false
-  }
-  

preprocessor_config.json

@@ -1,12 +1,21 @@
 {
+  "K": 257,
   "do_normalize": true,
+  "feature_extractor_type": "MCTCFeatureExtractor",
   "feature_size": 80,
-  "sampling_rate": 16000,
-  "padding_value": 0.0,
+  "frame_signal_scale": 32768.0,
   "hop_length": 10,
-  "win_length": 25,
-  "win_function": "hamming_window",
+  "mel_floor": 1.0,
+  "n_fft": 512,
   "normalize_means": true,
-  "normalize_vars": true,    
-  "return_attention_mask": false
+  "normalize_vars": true,
+  "padding_side": "right",
+  "padding_value": 0.0,
+  "preemphasis_coeff": 0.97,
+  "return_attention_mask": false,
+  "sample_size": 400,
+  "sample_stride": 160,
+  "sampling_rate": 16000,
+  "win_function": "hamming_window",
+  "win_length": 25
 }

special_tokens_map.json

@@ -1 +1 @@
-{"bos_token": "<s>", "eos_token": "</s>", "unk_token": "<unk>", "pad_token": "<pad>"}
+{"bos_token": "<s>", "eos_token": "</s>", "unk_token": "<unk>", "pad_token": "<pad>", "additional_special_tokens": [{"content": "<s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, {"content": "</s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}]}

tokenizer_config.json

@@ -1 +1 @@
-{"unk_token": "<unk>", "bos_token": "<s>", "eos_token": "</s>", "pad_token": "<pad>", "do_lower_case": false, "return_attention_mask": false, "do_normalize": true}
+{"unk_token": "<unk>", "bos_token": "<s>", "eos_token": "</s>", "pad_token": "<pad>", "do_lower_case": false, "word_delimiter_token": "|", "replace_word_delimiter_char": " ", "return_attention_mask": false, "do_normalize": true, "special_tokens_map_file": "./mctc-large/special_tokens_map.json", "name_or_path": "./mctc-large", "tokenizer_class": "Wav2Vec2CTCTokenizer"}