attempt to remove bos/eos
Browse files- special_tokens_map.json +1 -1
- tokenizer_config.json +1 -1
special_tokens_map.json
CHANGED
@@ -1 +1 @@
|
|
1 |
-
{"
|
|
|
1 |
+
{"unk_token": "[UNK]", "pad_token": "[PAD]"}
|
tokenizer_config.json
CHANGED
@@ -1 +1 @@
|
|
1 |
-
{"unk_token": "[UNK]", "
|
|
|
1 |
+
{"unk_token": "[UNK]", "pad_token": "[PAD]", "do_lower_case": false, "word_delimiter_token": " ", "replace_word_delimiter_char": " ", "tokenizer_class": "Wav2Vec2CTCTokenizer", "processor_class": "Wav2Vec2Processor"}
|