Shubham09 commited on
Commit
056bd8c
1 Parent(s): 82e01d6

Upload tokenizer

Browse files
Files changed (2) hide show
  1. special_tokens_map.json +0 -23
  2. tokenizer_config.json +0 -3
special_tokens_map.json CHANGED
@@ -1,27 +1,4 @@
1
  {
2
- "additional_special_tokens": [
3
- {
4
- "content": "<s>",
5
- "lstrip": false,
6
- "normalized": true,
7
- "rstrip": false,
8
- "single_word": false
9
- },
10
- {
11
- "content": "</s>",
12
- "lstrip": false,
13
- "normalized": true,
14
- "rstrip": false,
15
- "single_word": false
16
- },
17
- {
18
- "content": "[PAD]",
19
- "lstrip": false,
20
- "normalized": true,
21
- "rstrip": false,
22
- "single_word": false
23
- }
24
- ],
25
  "bos_token": "<s>",
26
  "eos_token": "</s>",
27
  "pad_token": "[PAD]",
 
1
  {
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2
  "bos_token": "<s>",
3
  "eos_token": "</s>",
4
  "pad_token": "[PAD]",
tokenizer_config.json CHANGED
@@ -2,11 +2,8 @@
2
  "bos_token": "<s>",
3
  "do_lower_case": false,
4
  "eos_token": "</s>",
5
- "name_or_path": "./",
6
  "pad_token": "[PAD]",
7
- "processor_class": "Wav2Vec2Processor",
8
  "replace_word_delimiter_char": " ",
9
- "special_tokens_map_file": null,
10
  "tokenizer_class": "Wav2Vec2CTCTokenizer",
11
  "unk_token": "[UNK]",
12
  "word_delimiter_token": "|"
 
2
  "bos_token": "<s>",
3
  "do_lower_case": false,
4
  "eos_token": "</s>",
 
5
  "pad_token": "[PAD]",
 
6
  "replace_word_delimiter_char": " ",
 
7
  "tokenizer_class": "Wav2Vec2CTCTokenizer",
8
  "unk_token": "[UNK]",
9
  "word_delimiter_token": "|"