Kushala commited on
Commit
a6e62d0
1 Parent(s): 64e9fe7

add tokenizer

Browse files
Files changed (2) hide show
  1. tokenizer_config.json +1 -1
  2. vocab.json +1 -1
tokenizer_config.json CHANGED
@@ -1 +1 @@
1
- {"unk_token": "[UNK]", "bos_token": "<s>", "eos_token": "</s>", "pad_token": "[PAD]", "do_lower_case": false, "word_delimiter_token": "|", "tokenizer_class": "Wav2Vec2CTCTokenizer"}
 
1
+ {"unk_token": "[UNK]", "bos_token": "<s>", "eos_token": "</s>", "pad_token": "[PAD]", "do_lower_case": false, "word_delimiter_token": "|", "replace_word_delimiter_char": " ", "tokenizer_class": "Wav2Vec2CTCTokenizer", "processor_class": "Wav2Vec2Processor"}
vocab.json CHANGED
@@ -1 +1 @@
1
- {"w": 0, "b": 1, "4": 2, "u": 3, "y": 4, ")": 5, "v": 6, "x": 7, "i": 8, "s": 9, "a": 10, "f": 11, "2": 12, "k": 13, "j": 14, "n": 15, "t": 16, "0": 17, "o": 18, "1": 19, "(": 20, "m": 21, "d": 22, "r": 23, "h": 24, "5": 25, "c": 26, "e": 27, "p": 28, "g": 29, "=": 30, "l": 31, "/": 33, "|": 32, "[UNK]": 34, "[PAD]": 35}
 
1
+ {"0": 0, "n": 1, "4": 2, "z": 3, "7": 4, "m": 5, "r": 6, "(": 7, "c": 8, "2": 9, "a": 10, "j": 11, "o": 12, "y": 13, "g": 14, "q": 16, ")": 17, "e": 18, "i": 19, "5": 20, "v": 21, "s": 22, "=": 23, "b": 24, "f": 25, "w": 26, "6": 27, "k": 28, "h": 29, "d": 30, "p": 31, "x": 32, "/": 33, "1": 34, "u": 35, "l": 36, "t": 37, "|": 15, "[UNK]": 38, "[PAD]": 39}